heal: isObjectDangling should return false when it cannot decide (#14053)

In a multi-pool setup when disks are coming up, or in a single pool
setup let's say with 100's of erasure sets with a slow network.

It's possible when healing is attempted on `.minio.sys/config`
folder, it can lead to healing unexpectedly deleting some policy
files as dangling due to a mistake in understanding when `isObjectDangling`
is considered to be 'true'.

This issue happened in commit 30135eed86
when we assumed the validMeta with empty ErasureInfo is considered
to be fully dangling. This implementation issue gets exposed when
the server is starting up.

This is most easily seen with multiple-pool setups because of the
disconnected fashion pools that come up. The decision to purge the
object as dangling is taken incorrectly prior to the correct state
being achieved on each pool, when the corresponding drive let's say
returns 'errDiskNotFound', a 'delete' is triggered. At this point,
the 'drive' comes online because this is part of the startup sequence
as drives can come online lazily.

This kind of situation exists because we allow (totalDisks/2) number
of drives to be online when the server is being restarted.

Implementation made an incorrect assumption here leading to policies
getting deleted.

Added tests to capture the implementation requirements.

.github/workflows/go-healing.yml

@@ -0,0 +1,48 @@
+name: Healing Functional Tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
+          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
+          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
+          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_AUTO_ENCRYPTION: on
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make verify-healing

.github/workflows/go.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   build:
-    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -46,4 +46,3 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify
-          make verify-healing

.github/workflows/iam-integrations.yaml

@@ -85,3 +85,11 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-iam
+      - name: Test LDAP for automatic site replication
+        if: matrix.ldap == 'localhost:389'
+        run: |
+          make test-site-replication-ldap
+      - name: Test OIDC for automatic site replication
+        if: matrix.openid == 'http://127.0.0.1:5556/dex'
+        run: |
+          make test-site-replication-oidc

.github/workflows/replication.yaml

@@ -37,5 +37,10 @@ jobs:
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
-          go install -v github.com/minio/mc@latest
           make test-replication
+      - name: Test MinIO IDP for automatic site replication
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-site-replication-minio
+

.gitignore

@@ -21,4 +21,15 @@ prime/
 stage/
 .sia_temp/
 config.json
-node_modules/
+node_modules/
+mc.*
+s3-check-md5*
+xl-meta*
+healing-*
+inspect*
+200M*
+hash-set
+minio.RELEASE*
+mc
+nancy
+inspects/*

.golangci.yml

@@ -24,6 +24,15 @@ linters:
     - unconvert
     - varcheck
     - gocritic
+    - gofumpt
+
+linters-settings:
+  gofumpt:
+    lang-version: "1.17"
+
+    # Choose whether or not to use the extra rules that are disabled
+    # by default
+    extra-rules: false
 
 issues:
   exclude-use-default: false

COMPLIANCE.md

@@ -0,0 +1,6 @@
+# AGPLv3 Compliance
+We have designed MinIO as an Open Source software for the Open Source software community. This requires applications to consider whether their usage of MinIO is in compliance with the GNU AGPLv3 [license](https://github.com/minio/minio/blob/master/LICENSE).
+
+MinIO cannot make the determination as to whether your application's usage of MinIO is in compliance with the AGPLv3 license requirements. You should instead rely on your own legal counsel or licensing specialists to audit and ensure your application is in compliance with the licenses of MinIO and all other open-source projects with which your application integrates or interacts. We understand that AGPLv3 licensing is complex and nuanced. It is for that reason we strongly encourage using experts in licensing to make any such determinations around compliance instead of relying on apocryphal or anecdotal advice.
+
+[MinIO Commercial Licensing](https://min.io/pricing) is the best option for applications that trigger AGPLv3 obligations (e.g. open sourcing your application). Applications using MinIO - or any other OSS-licensed code - without validating their usage do so at their own risk.

CREDITS

@@ -2412,6 +2412,33 @@ https://github.com/briandowns/spinner
 
 ================================================================
 
+github.com/buger/jsonparser
+https://github.com/buger/jsonparser
+----------------------------------------------------------------
+MIT License
+
+Copyright (c) 2016 Leonid Bugaev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 github.com/cespare/xxhash
 https://github.com/cespare/xxhash
 ----------------------------------------------------------------
@@ -2928,6 +2955,214 @@ https://github.com/coredns/coredns
 
 ================================================================
 
+github.com/coreos/go-oidc
+https://github.com/coreos/go-oidc
+----------------------------------------------------------------
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+================================================================
+
 github.com/coreos/go-semver
 https://github.com/coreos/go-semver
 ----------------------------------------------------------------
@@ -23378,6 +23613,214 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 ================================================================
 
+github.com/pquerna/cachecontrol
+https://github.com/pquerna/cachecontrol
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 github.com/prometheus/client_golang
 https://github.com/prometheus/client_golang
 ----------------------------------------------------------------
@@ -25523,6 +25966,37 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 ================================================================
 
+github.com/zeebo/xxh3
+https://github.com/zeebo/xxh3
+----------------------------------------------------------------
+xxHash Library
+Copyright (c) 2012-2014, Yann Collet
+Copyright (c) 2019, Jeff Wendling
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 go.etcd.io/bbolt
 https://go.etcd.io/bbolt
 ----------------------------------------------------------------
@@ -29211,6 +29685,214 @@ https://gopkg.in/jcmturner/rpc.v1
 
 ================================================================
 
+gopkg.in/square/go-jose.v2
+https://gopkg.in/square/go-jose.v2
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 gopkg.in/yaml.v2
 https://gopkg.in/yaml.v2
 ----------------------------------------------------------------

Dockerfile

@@ -1,5 +1,8 @@
 FROM minio/minio:latest
 
+COPY ./minio /opt/bin/minio
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]

Dockerfile.dev

@@ -1,17 +1,7 @@
-FROM minio/minio
+FROM minio/minio:latest
 
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-ENV PATH=/opt/bin:$PATH
-
-COPY minio /opt/bin
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
-
-RUN mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-    chmod +x /opt/bin/minio && \
-    chmod +x /usr/bin/docker-entrypoint.sh
-
-EXPOSE 9000
+COPY ./minio /opt/bin/minio
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Dockerfile.hotfix

@@ -0,0 +1,50 @@
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4
+
+ARG RELEASE
+
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="${RELEASE}" \
+      release="${RELEASE}" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
+
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
+    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=/opt/bin:$PATH
+
+COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
+
+RUN \
+     microdnf clean all && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+     microdnf install minisign --nodocs && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
+     microdnf clean all && \
+     chmod +x /opt/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/verify-minio.sh && \
+     /usr/bin/verify-minio.sh && \
+     microdnf clean all
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Makefile

@@ -34,13 +34,12 @@ check-gen: ## check for updated autogenerated files
 
 lint: ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
-	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
-	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
+	@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
 
 check: test
 test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
+	@CGO_ENABLED=0 go test -tags kqueue ./...
 
 test-upgrade: build
 	@echo "Running minio upgrade tests"
@@ -57,36 +56,61 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
 test-replication: install ## verify multi site replication
-	@echo "Running tests for Replication three sites"
+	@echo "Running tests for replicating three sites"
 	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
 
+test-site-replication-ldap: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with LDAP)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
+
+test-site-replication-oidc: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with OIDC)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
+
+test-site-replication-minio: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
+
 verify: ## verify minio various setups
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
+	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
 build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
-	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
 hotfix-vars:
 	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
-	$(eval TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)")
-hotfix: hotfix-vars install ## builds minio binary with hotfix tags
+	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
+	$(eval TAG := "minio/minio:$(VERSION)")
 
-docker-hotfix: hotfix checks ## builds minio docker container with hotfix tags
+hotfix: hotfix-vars install ## builds minio binary with hotfix tags
+	@mv -f ./minio ./minio.$(VERSION)
+	@minisign -qQSm ./minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
+	@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum
+
+hotfix-push: hotfix
+	@scp -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+
+docker-hotfix-push: docker-hotfix
+	@docker push $(TAG)
+
+docker-hotfix: hotfix-push checks ## builds minio docker container with hotfix tags
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) . -f Dockerfile.dev
+	@docker build -t $(TAG) --build-arg RELEASE=$(VERSION) . -f Dockerfile.hotfix
 
 docker: build checks ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) . -f Dockerfile.dev
+	@docker build -t $(TAG) . -f Dockerfile
 
 install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"

README.md

@@ -5,7 +5,7 @@
 
 MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
 
-This README provides quickstart instructions on running MinIO on baremetal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
+This README provides quickstart instructions on running MinIO on bare metal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
 
 # Container Installation
 
@@ -237,13 +237,11 @@ mc admin update <minio alias, e.g., myminio>
 
 ## Important things to remember during MinIO upgrades
 
+- Container environments are advised to update the container images instead of updating binaries inside the container.
+- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
 - `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
-- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
-- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
 - If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
-- If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
-- If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
 # Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
@@ -257,5 +255,6 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 # License
-MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
-MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
+- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)

buildscripts/minio-upgrade.sh

@@ -43,6 +43,9 @@ add_alias() {
             echo "...waiting... for 5secs" && sleep 5
         done
     done
+
+    echo "Sleeping for nginx"
+    sleep 20
 }
 
 __init__() {

buildscripts/unaligned-healing.sh

@@ -0,0 +1,171 @@
+#!/bin/bash -e
+#
+
+set -E
+set -o pipefail
+set -x
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+function download_old_release() {
+    if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
+	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
+	chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
+    fi
+}
+
+function start_minio_16drive() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
+    "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
+    "${WORK_DIR}/mc" cp \
+		     "${WORK_DIR}/unaligned" \
+		     minio/healing-shard-bucket/unaligned \
+		     --disable-multipart --quiet
+
+    ## "unaligned" object name gets consistently distributed
+    ## to disks in following distribution order
+    ##
+    ## NOTE: if you change the name make sure to change the
+    ## distribution order present here
+    ##
+    ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+
+    ## make sure to remove the "last" data shard
+    rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
+    sleep 10
+    ## Heal the shard
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+    ## then remove any other data shard let's pick first disk
+    ## - 1st data shard.
+    rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
+    sleep 10
+
+    go build ./docs/debugging/s3-check-md5/
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+}
+
+function main() {
+    download_old_release
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    start_minio_16drive ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/verify-build.sh

@@ -66,7 +66,7 @@ function start_minio_pool_erasure_sets_ipv6()
 {
     export MINIO_ROOT_USER=$ACCESS_KEY
     export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
+    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
     "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
     "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
 

buildscripts/verify-healing.sh

@@ -18,54 +18,73 @@ function start_minio_3_node() {
     export MINIO_ROOT_PASSWORD=minio123
     export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
-    start_port=$(shuf -i 10000-65000 -n 1)
+    start_port=$2
     args=""
     for i in $(seq 1 3); do
-        args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
+	args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
     done
 
     "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    disown $!
+    pid1=$!
+    disown ${pid1}
 
     "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    disown $!
+    pid2=$!
+    disown $pid2
 
     "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    disown $!
+    pid3=$!
+    disown $pid3
 
     sleep "$1"
-    if [ "$(pgrep -c minio)" -ne 3 ]; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+
+    if ! ps -p $pid1 1>&2 > /dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/dist-minio-server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
+
+    if ! ps -p $pid2 1>&2 > /dev/null; then
+	echo "server2 log:"
+	cat "${WORK_DIR}/dist-minio-server2.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ps -p $pid3 1>&2 > /dev/null; then
+	echo "server3 log:"
+	cat "${WORK_DIR}/dist-minio-server3.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
     if ! pkill minio; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+	for i in $(seq 1 3); do
+	    echo "server$i log:"
+	    cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
 
     sleep 1;
     if pgrep minio; then
-        # forcibly killing, to proceed further properly.
-        if ! pkill -9 minio; then
-            echo "no minio process running anymore, proceed."
-        fi
+	# forcibly killing, to proceed further properly.
+	if ! pkill -9 minio; then
+	    echo "no minio process running anymore, proceed."
+	fi
     fi
 }
 
 
 function check_online() {
     if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
-        echo "1"
+	echo "1"
     fi
 }
 
@@ -85,33 +104,36 @@ function __init__()
 }
 
 function perform_test() {
-    start_minio_3_node 120
+    start_minio_3_node 120 $2
 
     echo "Testing Distributed Erasure setup healing of drives"
     echo "Remove the contents of the disks belonging to '${1}' erasure set"
 
     rm -rf ${WORK_DIR}/${1}/*/
 
-    start_minio_3_node 120
+    start_minio_3_node 120 $2
 
     rv=$(check_online)
     if [ "$rv" == "1" ]; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        pkill -9 minio
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+	for i in $(seq 1 3); do
+	    echo "server$i log:"
+	    cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	pkill -9 minio
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
 }
 
 function main()
 {
-    perform_test "2"
-    perform_test "1"
-    perform_test "3"
+    # use same ports for all tests
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    perform_test "2" ${start_port}
+    perform_test "1" ${start_port}
+    perform_test "3" ${start_port}
 }
 
 ( __init__ "$@" && main "$@" )

cmd/admin-bucket-handlers.go

@@ -155,7 +155,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		return
 	}
 	var target madmin.BucketTarget
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(reqBytes, &target); err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return

cmd/admin-handler-utils.go

@@ -86,6 +86,8 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			Description:    e.Message,
 			HTTPStatusCode: e.StatusCode,
 		}
+	case SRError:
+		apiErr = errorCodes.ToAPIErrWithErr(e.Code, e.Cause)
 	default:
 		switch {
 		case errors.Is(err, errConfigNotFound):

cmd/admin-handlers-config-kv.go

@@ -170,7 +170,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	cfg := globalServerConfig.Clone()
 	vars := mux.Vars(r)
-	var buf = &bytes.Buffer{}
+	buf := &bytes.Buffer{}
 	cw := config.NewConfigWriteTo(cfg, vars["key"])
 	if _, err := cw.WriteTo(buf); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)

cmd/admin-handlers-site-replication.go

@@ -45,16 +45,16 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	}
 
 	var sites []madmin.PeerSite
-	errCode := readJSONBody(ctx, r.Body, &sites, cred.SecretKey)
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	status, errInfo := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
-	if errInfo.Code != ErrNone {
-		logger.LogIf(ctx, errInfo)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
@@ -67,12 +67,12 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, body)
 }
 
-// SRInternalJoin - PUT /minio/admin/v3/site-replication/join
+// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
 //
 // used internally to tell current cluster to enable SR with
 // the provided peer clusters and service account.
-func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalJoin")
+func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerJoin")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -81,24 +81,22 @@ func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	var joinArg madmin.SRInternalJoinReq
-	errCode := readJSONBody(ctx, r.Body, &joinArg, cred.SecretKey)
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	var joinArg madmin.SRPeerJoinReq
+	if err := parseJSONBody(ctx, r.Body, &joinArg, cred.SecretKey); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	errInfo := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg)
-	if errInfo.Code != ErrNone {
-		logger.LogIf(ctx, errInfo)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+	if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
 
-// SRInternalBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
-func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalBucketOps")
+// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
+func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerBucketOps")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -113,6 +111,8 @@ func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Req
 
 	var err error
 	switch operation {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.MakeWithVersioningBktOp:
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
@@ -128,21 +128,17 @@ func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Req
 		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
 	case madmin.ForceDeleteBucketBktOp:
 		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
-		return
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
 }
 
-// SRInternalReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
-func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
+func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateIAMItem")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -152,45 +148,51 @@ func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *h
 	}
 
 	var item madmin.SRIAMItem
-	errCode := readJSONBody(ctx, r.Body, &item, "")
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	var err error
 	switch item.Type {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRIAMItemPolicy:
-		var policy *iampolicy.Policy
-		if len(item.Policy) > 0 {
-			policy, err = iampolicy.ParseConfig(bytes.NewReader(item.Policy))
-			if err != nil {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		if item.Policy == nil {
+			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+		} else {
+			policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
+			if perr != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
 				return
 			}
+			if policy.IsEmpty() {
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+			} else {
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+			}
 		}
-		err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
 	case madmin.SRIAMItemSvcAcc:
-		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, *item.SvcAccChange)
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange)
 	case madmin.SRIAMItemPolicyMapping:
-		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, *item.PolicyMapping)
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping)
 	case madmin.SRIAMItemSTSAcc:
-		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, *item.STSCredential)
-
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
-		return
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential)
+	case madmin.SRIAMItemIAMUser:
+		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser)
+	case madmin.SRIAMItemGroupInfo:
+		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
 
-// SRInternalReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
-func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateBucketItem")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -200,38 +202,40 @@ func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r
 	}
 
 	var item madmin.SRBucketMeta
-	errCode := readJSONBody(ctx, r.Body, &item, "")
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	var err error
 	switch item.Type {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRBucketMetaTypePolicy:
-		var bktPolicy *policy.Policy
-		if len(item.Policy) > 0 {
-			bktPolicy, err = policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
-			if err != nil {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		if item.Policy == nil {
+			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+		} else {
+			bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
+			if berr != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
 				return
 			}
+			if bktPolicy.IsEmpty() {
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+			} else {
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+			}
 		}
-		err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
 	case madmin.SRBucketMetaTypeTags:
 		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
 	case madmin.SRBucketMetaTypeObjectLockConfig:
 		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
 	case madmin.SRBucketMetaTypeSSEConfig:
 		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
-
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
-		return
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
@@ -271,7 +275,7 @@ func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Req
 	}
 }
 
-func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -288,24 +292,71 @@ func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *htt
 	}
 }
 
-func readJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) APIErrorCode {
+func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
 	data, err := ioutil.ReadAll(body)
 	if err != nil {
-		return ErrInvalidRequest
+		return SRError{
+			Cause: err,
+			Code:  ErrSiteReplicationInvalidRequest,
+		}
 	}
 
 	if encryptionKey != "" {
 		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
 		if err != nil {
 			logger.LogIf(ctx, err)
-			return ErrInvalidRequest
+			return SRError{
+				Cause: err,
+				Code:  ErrSiteReplicationInvalidRequest,
+			}
 		}
 	}
 
-	err = json.Unmarshal(data, v)
-	if err != nil {
-		return ErrAdminConfigBadJSON
+	return json.Unmarshal(data, v)
+}
+
+// SiteReplicationStatus - GET /minio/admin/v3/site-replication/status
+func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationStatus")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	if objectAPI == nil {
+		return
 	}
 
-	return ErrNone
+	info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(info); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo
+func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationMetaInfo")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	if objectAPI == nil {
+		return
+	}
+
+	info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(info); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }

cmd/admin-handlers-users.go

@@ -58,17 +58,20 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := globalIAMSys.DeleteUser(ctx, accessKey); err != nil {
+	if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to delete user.
-	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemIAMUser,
+		IAMUser: &madmin.SRIAMUser{
+			AccessKey:   accessKey,
+			IsDeleteReq: true,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 }
 
@@ -164,31 +167,24 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	accessKey := cred.ParentUser
-	if accessKey == "" {
-		accessKey = cred.AccessKey
+	checkDenyOnly := false
+	if name == cred.AccessKey {
+		// Check that there is no explicit deny - otherwise it's allowed
+		// to view one's own info.
+		checkDenyOnly = true
 	}
 
-	// For temporary credentials always
-	// the temporary credentials to check
-	// policy without implicit permissions.
-	if cred.IsTemp() && cred.ParentUser == globalActiveCred.AccessKey {
-		accessKey = cred.AccessKey
-	}
-
-	implicitPerm := name == accessKey
-	if !implicitPerm {
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     accessKey,
-			Groups:          cred.Groups,
-			Action:          iampolicy.GetUserAdminAction,
-			ConditionValues: getConditionValues(r, "", accessKey, claims),
-			IsOwner:         owner,
-			Claims:          claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+	if !globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          iampolicy.GetUserAdminAction,
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		IsOwner:         owner,
+		Claims:          claims,
+		DenyOnly:        checkDenyOnly,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
 	}
 
 	userInfo, err := globalIAMSys.GetUserInfo(ctx, name)
@@ -235,20 +231,19 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 	} else {
 		err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
 	}
-
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to load group.
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemGroupInfo,
+		GroupInfo: &madmin.SRGroupInfo{
+			UpdateReq: updReq,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 }
 
@@ -335,16 +330,6 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to reload user.
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadGroup(group) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
 }
 
 // SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
@@ -372,16 +357,6 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to reload user.
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
 }
 
 // AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
@@ -412,6 +387,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	userCred, exists := globalIAMSys.GetUser(ctx, accessKey)
+	if exists && (userCred.IsTemp() || userCred.IsServiceAccount()) {
+		// Updating STS credential is not allowed, and this API does not
+		// support updating service accounts.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
 	if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
 		// Incoming access key matches parent user then we should
 		// reject password change requests.
@@ -419,39 +402,21 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	implicitPerm := accessKey == cred.AccessKey
-	if !implicitPerm {
-		parentUser := cred.ParentUser
-		if parentUser == "" {
-			parentUser = cred.AccessKey
-		}
-		// For temporary credentials always
-		// the temporary credentials to check
-		// policy without implicit permissions.
-		if cred.IsTemp() && cred.ParentUser == globalActiveCred.AccessKey {
-			parentUser = cred.AccessKey
-		}
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     parentUser,
-			Groups:          cred.Groups,
-			Action:          iampolicy.CreateUserAdminAction,
-			ConditionValues: getConditionValues(r, "", parentUser, claims),
-			IsOwner:         owner,
-			Claims:          claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+	checkDenyOnly := false
+	if accessKey == cred.AccessKey {
+		// Check that there is no explicit deny - otherwise it's allowed
+		// to change one's own password.
+		checkDenyOnly = true
 	}
 
-	if implicitPerm && !globalIAMSys.IsAllowed(iampolicy.Args{
-		AccountName:     accessKey,
+	if !globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.CreateUserAdminAction,
-		ConditionValues: getConditionValues(r, "", accessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 		IsOwner:         owner,
 		Claims:          claims,
-		DenyOnly:        true, // check if changing password is explicitly denied.
+		DenyOnly:        checkDenyOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
@@ -471,26 +436,28 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var uinfo madmin.UserInfo
-	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+	var ureq madmin.AddOrUpdateUserReq
+	if err = json.Unmarshal(configBytes, &ureq); err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
 
-	if err = globalIAMSys.CreateUser(ctx, accessKey, uinfo); err != nil {
+	if err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other Minio peers to reload user
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemIAMUser,
+		IAMUser: &madmin.SRIAMUser{
+			AccessKey:   accessKey,
+			IsDeleteReq: false,
+			UserReq:     &ureq,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 }
 
@@ -571,6 +538,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// if there is no deny statement this call is implicitly enabled.
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     requestorUser,
+			Groups:          requestorGroups,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
@@ -604,10 +572,12 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// user <> to the request sender
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     requestorUser,
+			Groups:          requestorGroups,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
 			Claims:          claims,
+			DenyOnly:        true,
 		}) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
@@ -647,24 +617,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Notify all other Minio peers to reload user the service account
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
-	// Call hook for cluster-replication.
-	//
-	// FIXME: This wont work in an OpenID situation as the parent credential
-	// may not be present on peer clusters to provide inherited policies.
-	// Also, we should not be replicating root user's service account - as
-	// they are not authenticated by a common external IDP, so we skip when
-	// opts.ldapUser == "".
-	if _, isLDAPAccount := opts.claims[ldapUserN]; isLDAPAccount {
+	// Call hook for cluster-replication if the service account is not for a
+	// root user.
+	if newCred.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -685,7 +640,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	var createResp = madmin.AddServiceAccountResp{
+	createResp := madmin.AddServiceAccountResp{
 		Credentials: madmin.Credentials{
 			AccessKey: newCred.AccessKey,
 			SecretKey: newCred.SecretKey,
@@ -788,24 +743,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Notify all other Minio peers to reload user the service account
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadServiceAccount(accessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
-	// Call site replication hook. Only LDAP accounts are supported for
-	// replication operations.
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+	// Call site replication hook - non-root user accounts are replicated.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -822,6 +761,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 			return
 		}
 	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -896,7 +836,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	var infoResp = madmin.InfoServiceAccountResp{
+	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
 		AccountStatus: svcAccount.Status,
 		ImpliedPolicy: impliedPolicy,
@@ -973,7 +913,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 		serviceAccountsNames = append(serviceAccountsNames, svc.AccessKey)
 	}
 
-	var listResp = madmin.ListServiceAccountsResp{
+	listResp := madmin.ListServiceAccountsResp{
 		Accounts: serviceAccountsNames,
 	}
 
@@ -1045,28 +985,14 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	// Save svc acc claims before deletion (for site replication hook).
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, serviceAccount)
-	if err != nil && err != errNoSuchServiceAccount {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount)
+	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	for _, nerr := range globalNotificationSys.DeleteServiceAccount(serviceAccount) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 
-	// Call site replication hook. Only LDAP accounts are supported for
-	// replication operations.
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+	// Call site replication hook - non-root user accounts are replicated.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -1181,19 +1107,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	accountName := cred.AccessKey
-	var policies []string
-	switch globalIAMSys.usersSysType {
-	case MinIOUsersSysType:
-		policies, err = globalIAMSys.PolicyDBGet(accountName, false)
-	case LDAPUsersSysType:
-		parentUser := accountName
-		if cred.ParentUser != "" {
-			parentUser = cred.ParentUser
-		}
-		policies, err = globalIAMSys.PolicyDBGet(parentUser, false, cred.Groups...)
-	default:
-		err = errors.New("should never happen")
+	if cred.IsTemp() || cred.IsServiceAccount() {
+		// For derived credentials, check the parent user's permissions.
+		accountName = cred.ParentUser
 	}
+	policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1272,6 +1190,16 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 }
 
 // InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName}
+//
+// Newer API response with policy timestamps is returned with query parameter
+// `v=2` like:
+//
+// GET /minio/admin/v3/info-canned-policy?name={policyName}&v=2
+//
+// The newer API will eventually become the default (and only) one. The older
+// response is to return only the policy JSON. The newer response returns
+// timestamps along with the policy JSON. Both versions are supported for now,
+// for smooth transition to new API.
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
@@ -1282,13 +1210,36 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	name := mux.Vars(r)["name"]
+	policies := newMappedPolicy(name).toSlice()
+	if len(policies) != 1 {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL)
+		return
+	}
+
+	policyDoc, err := globalIAMSys.InfoPolicy(name)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	buf, err := json.MarshalIndent(policy, "", " ")
+	// Is the new API version being requested?
+	infoPolicyAPIVersion := r.Form.Get("v")
+	if infoPolicyAPIVersion == "2" {
+		buf, err := json.MarshalIndent(policyDoc, "", " ")
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		w.Write(buf)
+		return
+	} else if infoPolicyAPIVersion != "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("invalid version parameter 'v' supplied")), r.URL)
+		return
+	}
+
+	// Return the older API response value of just the policy json.
+	buf, err := json.MarshalIndent(policyDoc.Policy, "", " ")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1314,7 +1265,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	var newPolicies = make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]iampolicy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1346,7 +1297,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	var newPolicies = make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]iampolicy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1375,19 +1326,11 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	vars := mux.Vars(r)
 	policyName := vars["name"]
 
-	if err := globalIAMSys.DeletePolicy(ctx, policyName); err != nil {
+	if err := globalIAMSys.DeletePolicy(ctx, policyName, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to delete policy
-	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-
 	// Call cluster-replication policy creation hook to replicate policy deletion to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
@@ -1448,16 +1391,6 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	// Notify all other MinIO peers to reload policy
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	// Call cluster-replication policy creation hook to replicate policy to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
@@ -1503,16 +1436,6 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Notify all other MinIO peers to reload policy
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{

cmd/admin-handlers-users_test.go

@@ -18,8 +18,15 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"testing"
@@ -29,7 +36,9 @@ import (
 	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/internal/auth"
 )
 
@@ -177,6 +186,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
 
 				suite.SetUpSuite(c)
 				suite.TestUserCreate(c)
+				suite.TestUserPolicyEscalationBug(c)
 				suite.TestPolicyCreate(c)
 				suite.TestCannedPolicies(c)
 				suite.TestGroupAddRemove(c)
@@ -222,6 +232,24 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 		c.Fatalf("user could not create bucket: %v", err)
 	}
 
+	// 3.10. Check that user's password can be updated.
+	_, newSecretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to update user's secret key: %v", err)
+	}
+	// 3.10.1 Check that old password no longer works.
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err == nil {
+		c.Fatalf("user was unexpectedly able to create bucket with bad password!")
+	}
+	// 3.10.2 Check that new password works.
+	client = s.getUserClient(c, accessKey, newSecretKey, "")
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("user could not create bucket: %v", err)
+	}
+
 	// 4. Check that user can be disabled and verify it.
 	err = s.adm.SetUserStatus(ctx, accessKey, madmin.AccountDisabled)
 	if err != nil {
@@ -260,6 +288,108 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	}
 }
 
+func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// 2. Create a user, associate policy and verify access
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+	// 2.1 check that user does not have any access to the bucket
+	uClient := s.getUserClient(c, accessKey, secretKey, "")
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	// 2.2 create and associate policy to user
+	policy := "mypolicy-test-user-update"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+	// 2.3 check user has access to bucket
+	c.mustListObjects(ctx, uClient, bucket)
+	// 2.3 check that user cannot delete the bucket
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil || err.Error() != "Access Denied." {
+		c.Fatalf("bucket was deleted unexpectedly or got unexpected err: %v", err)
+	}
+
+	// 3. Craft a request to update the user's permissions
+	ep := s.adm.GetEndpointURL()
+	urlValue := url.Values{}
+	urlValue.Add("accessKey", accessKey)
+	u, err := url.Parse(fmt.Sprintf("%s://%s/minio/admin/v3/add-user?%s", ep.Scheme, ep.Host, s3utils.QueryEncode(urlValue)))
+	if err != nil {
+		c.Fatalf("unexpected url parse err: %v", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), nil)
+	if err != nil {
+		c.Fatalf("unexpected new request err: %v", err)
+	}
+	reqBodyArg := madmin.UserInfo{
+		SecretKey:  secretKey,
+		PolicyName: "consoleAdmin",
+		Status:     madmin.AccountEnabled,
+	}
+	buf, err := json.Marshal(reqBodyArg)
+	if err != nil {
+		c.Fatalf("unexpected json encode err: %v", err)
+	}
+	buf, err = madmin.EncryptData(secretKey, buf)
+	if err != nil {
+		c.Fatalf("unexpected encryption err: %v", err)
+	}
+
+	req.ContentLength = int64(len(buf))
+	sum := sha256.Sum256(buf)
+	req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:]))
+	req.Body = ioutil.NopCloser(bytes.NewReader(buf))
+	req = signer.SignV4(*req, accessKey, secretKey, "", "")
+
+	// 3.1 Execute the request.
+	resp, err := s.TestSuiteCommon.client.Do(req)
+	if err != nil {
+		c.Fatalf("unexpected request err: %v", err)
+	}
+	if resp.StatusCode != 200 {
+		c.Fatalf("got unexpected response: %#v\n", resp)
+	}
+
+	// 3.2 check that user cannot delete the bucket
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil || err.Error() != "Access Denied." {
+		c.Fatalf("User was able to escalate privileges (Err=%v)!", err)
+	}
+}
+
 func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
@@ -523,7 +653,6 @@ func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
 	if !strings.Contains(infoStr, `"s3:PutObject"`) || !strings.Contains(infoStr, ":"+bucket+"/") {
 		c.Fatalf("policy contains unexpected content!")
 	}
-
 }
 
 func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
@@ -831,6 +960,36 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
 }
 
+func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials {
+	randUser := mustGetUUID()
+	randPass := mustGetUUID()
+	err := admClnt.AddUser(ctx, randUser, randPass)
+	if err != nil {
+		c.Fatalf("should be able to create a user: %v", err)
+	}
+	return madmin.Credentials{
+		AccessKey: randUser,
+		SecretKey: randPass,
+	}
+}
+
+func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminClient, accessKey string) madmin.UserInfo {
+	ui, err := admClnt.GetUserInfo(ctx, accessKey)
+	if err != nil {
+		c.Fatalf("should be able to get user info: %v", err)
+	}
+	return ui
+}
+
+func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) {
+	randUser := mustGetUUID()
+	randPass := mustGetUUID()
+	err := admClnt.AddUser(ctx, randUser, randPass)
+	if err == nil {
+		c.Fatalf("should not be able to create a user")
+	}
+}
+
 func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) madmin.Credentials {
 	cr, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: tgtUser,

cmd/admin-handlers.go

@@ -333,7 +333,6 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
-
 }
 
 // DataUsageInfoHandler - GET /minio/admin/v3/datausage
@@ -957,6 +956,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
 	autotune := r.Form.Get("autotune") == "true"
+	storageClass := r.Form.Get("storage-class")
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -968,6 +968,10 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		concurrent = 32
 	}
 
+	if runtime.GOMAXPROCS(0) < concurrent {
+		concurrent = runtime.GOMAXPROCS(0)
+	}
+
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
 		duration = time.Second * 10
@@ -980,6 +984,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 			NoRecreate: true,
 		})
 	}
+	defer deleteBucket()
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -991,7 +996,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)
-	ch := speedTest(ctx, size, concurrent, duration, autotune)
+	ch := speedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
 	for {
 		select {
 		case <-ctx.Done():
@@ -1004,7 +1009,6 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 			w.(http.Flusher).Flush()
 		case result, ok := <-ch:
 			if !ok {
-				deleteBucket()
 				return
 			}
 			if err := enc.Encode(result); err != nil {
@@ -1327,7 +1331,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	if keyID == "" {
 		keyID = stat.DefaultKey
 	}
-	var response = madmin.KMSKeyStatus{
+	response := madmin.KMSKeyStatus{
 		KeyID: keyID,
 	}
 
@@ -1466,7 +1470,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 	return madmin.InfoMessage{
 		Mode:         string(mode),
 		Domain:       domain,
-		Region:       globalServerRegion,
+		Region:       globalSite.Region,
 		SQSARN:       globalNotificationSys.GetARNList(false),
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
@@ -1811,7 +1815,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			anonNetwork[anonEndpoint] = status
 		}
 		return anonNetwork
-
 	}
 
 	anonymizeDrives := func(drives []madmin.Disk) []madmin.Disk {
@@ -1864,6 +1867,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 
 			tls := getTLSInfo()
+			isK8s := IsKubernetes()
+			isDocker := IsDocker()
 			healthInfo.Minio.Info = madmin.MinioInfo{
 				Mode:         infoMessage.Mode,
 				Domain:       infoMessage.Domain,
@@ -1877,6 +1882,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				Backend:      infoMessage.Backend,
 				Servers:      servers,
 				TLS:          &tls,
+				IsKubernetes: &isK8s,
+				IsDocker:     &isDocker,
 			}
 			partialWrite(healthInfo)
 		}
@@ -1907,7 +1914,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			return
 		}
 	}
-
 }
 
 func getTLSInfo() madmin.TLSInfo {
@@ -2033,7 +2039,6 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
 }
 
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
-
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
 	for _, tgt := range globalConfigTargetList.Targets() {
@@ -2275,7 +2280,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		}
 		if si.Mode == 0 {
 			// Not, set it to default.
-			si.Mode = 0600
+			si.Mode = 0o600
 		}
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,

cmd/admin-handlers_test.go

@@ -41,11 +41,13 @@ type adminErasureTestBed struct {
 	erasureDirs []string
 	objLayer    ObjectLayer
 	router      *mux.Router
+	done        context.CancelFunc
 }
 
 // prepareAdminErasureTestBed - helper function that setups a single-node
 // Erasure backend for admin-handler tests.
 func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, error) {
+	ctx, cancel := context.WithCancel(ctx)
 
 	// reset global variables to start afresh.
 	resetTestGlobals()
@@ -57,11 +59,13 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 	// Initializing objectLayer for HealFormatHandler.
 	objLayer, erasureDirs, xlErr := initTestErasureObjLayer(ctx)
 	if xlErr != nil {
+		cancel()
 		return nil, xlErr
 	}
 
 	// Initialize minio server config.
 	if err := newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		cancel()
 		return nil, err
 	}
 
@@ -74,7 +78,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -84,12 +88,14 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 		erasureDirs: erasureDirs,
 		objLayer:    objLayer,
 		router:      adminRouter,
+		done:        cancel,
 	}, nil
 }
 
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
 func (atb *adminErasureTestBed) TearDown() {
+	atb.done()
 	removeRoots(atb.erasureDirs)
 	resetTestGlobals()
 }
@@ -230,8 +236,8 @@ func TestServiceRestartHandler(t *testing.T) {
 
 // buildAdminRequest - helper function to build an admin API request.
 func buildAdminRequest(queryVal url.Values, method, path string,
-	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
-
+	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error,
+) {
 	req, err := newTestRequest(method,
 		adminPathPrefix+adminAPIVersionPrefix+path+"?"+queryVal.Encode(),
 		contentLength, bodySeeker)
@@ -374,5 +380,4 @@ func TestExtractHealInitParams(t *testing.T) {
 			}
 		}
 	}
-
 }

cmd/admin-heal-ops.go

@@ -278,8 +278,8 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 // background routine to clean up heal results after the
 // aforementioned duration.
 func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLayer) (
-	respBytes []byte, apiErr APIError, errMsg string) {
-
+	respBytes []byte, apiErr APIError, errMsg string,
+) {
 	if h.forceStarted {
 		_, apiErr = ahs.stopHealSequence(pathJoin(h.bucket, h.object))
 		if apiErr.Code != "" {
@@ -338,8 +338,8 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
 // representation. The clientToken helps ensure there aren't
 // conflicting clients fetching status.
 func (ahs *allHealState) PopHealStatusJSON(hpath string,
-	clientToken string) ([]byte, APIErrorCode) {
-
+	clientToken string) ([]byte, APIErrorCode,
+) {
 	// fetch heal state for given path
 	h, exists := ahs.getHealSequence(hpath)
 	if !exists {
@@ -453,8 +453,8 @@ type healSequence struct {
 // NewHealSequence - creates healSettings, assumes bucket and
 // objPrefix are already validated.
 func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
-	hs madmin.HealOpts, forceStart bool) *healSequence {
-
+	hs madmin.HealOpts, forceStart bool,
+) *healSequence {
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
 	ctx, cancel := context.WithCancel(logger.SetReqInfo(ctx, reqInfo))
@@ -710,6 +710,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 
 	select {
 	case globalBackgroundHealRoutine.tasks <- task:
+		if serverDebugLog {
+			logger.Info("Task in the queue: %#v", task)
+		}
 	case <-h.ctx.Done():
 		return nil
 	}
@@ -885,6 +888,7 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		bucket:    bucket,
 		object:    object,
 		versionID: versionID,
+		opts:      &h.settings,
 	}, madmin.HealItemObject)
 
 	// Wait and proceed if there are active requests

cmd/admin-router.go

@@ -38,7 +38,6 @@ type adminAPIHandlers struct{}
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
 func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
-
 	adminAPI := adminAPIHandlers{}
 	// Admin router
 	adminRouter := router.PathPrefix(adminPathPrefix).Subrouter()
@@ -194,11 +193,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalJoin)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateIAMItem)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateBucketItem)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalGetIDPSettings)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
+
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
 		}
 
 		if globalIsDistErasure {

cmd/api-datatypes.go

@@ -48,10 +48,15 @@ func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElem
 	return e.EncodeElement(t.Time.Format(time.RFC3339), startElement)
 }
 
-// ObjectToDelete carries key name for the object to delete.
-type ObjectToDelete struct {
+// ObjectV object version key/versionId
+type ObjectV struct {
 	ObjectName string `xml:"Key"`
 	VersionID  string `xml:"VersionId"`
+}
+
+// ObjectToDelete carries key name for the object to delete.
+type ObjectToDelete struct {
+	ObjectV
 	// Replication status of DeleteMarker
 	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"`
 	// Status of versioned delete (of object or DeleteMarker)

cmd/api-errors.go

@@ -82,6 +82,7 @@ const (
 	ErrIncompleteBody
 	ErrInternalError
 	ErrInvalidAccessKeyID
+	ErrAccessKeyDisabled
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidRange
@@ -129,6 +130,7 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
+	ErrReplicationDenyEditError
 	ErrReplicationNoMatchingRuleError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -396,10 +398,10 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
 	if err != nil {
 		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
 	}
-	if globalServerRegion != "" {
+	if globalSite.Region != "" {
 		switch errCode {
 		case ErrAuthorizationHeaderMalformed:
-			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
+			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 			return apiErr
 		}
 	}
@@ -513,6 +515,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrAccessKeyDisabled: {
+		Code:           "InvalidAccessKeyId",
+		Description:    "Your account is disabled; please contact your administrator.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
@@ -680,7 +687,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
-		Description:    "All access to this bucket has been disabled.",
+		Description:    "All access to this resource has been disabled.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMalformedPolicy: {
@@ -888,6 +895,11 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching replication rule found for this object prefix",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrReplicationDenyEditError: {
+		Code:           "XMinioReplicationDenyEdit",
+		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
@@ -2113,7 +2125,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 		return noError
 	}
 
-	var apiErr = errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
+	apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
 	e, ok := err.(dns.ErrInvalidBucketName)
 	if ok {
 		code := toAPIErrorCode(ctx, e)
@@ -2226,7 +2238,6 @@ func toAPIError(ctx context.Context, err error) APIError {
 			// since S3 only sends one Error XML response.
 			if len(e.Errors) >= 1 {
 				apiErr.Code = e.Errors[0].Reason
-
 			}
 		case azblob.StorageError:
 			apiErr = APIError{
@@ -2286,7 +2297,7 @@ func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
 		Resource:   resource,
-		Region:     globalServerRegion,
+		Region:     globalSite.Region,
 		RequestID:  requestID,
 		HostID:     hostID,
 	}

cmd/api-headers.go

@@ -52,7 +52,7 @@ func setCommonHeaders(w http.ResponseWriter) {
 
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
-	if region := globalServerRegion; region != "" {
+	if region := globalSite.Region; region != "" {
 		w.Header().Set(xhttp.AmzBucketRegion, region)
 	}
 	w.Header().Set(xhttp.AcceptRanges, "bytes")

cmd/api-headers_test.go

@@ -23,7 +23,7 @@ import (
 
 func TestNewRequestID(t *testing.T) {
 	// Ensure that it returns an alphanumeric result of length 16.
-	var id = mustGetRequestID(UTCNow())
+	id := mustGetRequestID(UTCNow())
 
 	if len(id) != 16 {
 		t.Fail()

cmd/api-response.go

@@ -268,7 +268,6 @@ type StringMap map[string]string
 
 // MarshalXML - StringMap marshals into XML.
 func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-
 	tokens := []xml.Token{start}
 
 	for key, value := range s {
@@ -417,8 +416,8 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 // serialized to match XML and JSON API spec output.
 func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	listbuckets := make([]Bucket, 0, len(buckets))
-	var data = ListBucketsResponse{}
-	var owner = Owner{
+	data := ListBucketsResponse{}
+	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
@@ -439,14 +438,14 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
-	var owner = Owner{
+	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
-	var data = ListVersionsResponse{}
+	data := ListVersionsResponse{}
 
 	for _, object := range resp.Objects {
-		var content = ObjectVersion{}
+		content := ObjectVersion{}
 		if object.Name == "" {
 			continue
 		}
@@ -486,7 +485,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 
 	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
-		var prefixItem = CommonPrefix{}
+		prefixItem := CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
 		prefixes = append(prefixes, prefixItem)
 	}
@@ -497,14 +496,14 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	contents := make([]Object, 0, len(resp.Objects))
-	var owner = Owner{
+	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
-	var data = ListObjectsResponse{}
+	data := ListObjectsResponse{}
 
 	for _, object := range resp.Objects {
-		var content = Object{}
+		content := Object{}
 		if object.Name == "" {
 			continue
 		}
@@ -535,7 +534,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 
 	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
-		var prefixItem = CommonPrefix{}
+		prefixItem := CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
 		prefixes = append(prefixes, prefixItem)
 	}
@@ -546,14 +545,14 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
 func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
-	var owner = Owner{
+	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
-	var data = ListObjectsV2Response{}
+	data := ListObjectsV2Response{}
 
 	for _, object := range objects {
-		var content = Object{}
+		content := Object{}
 		if object.Name == "" {
 			continue
 		}
@@ -608,7 +607,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 
 	commonPrefixes := make([]CommonPrefix, 0, len(prefixes))
 	for _, prefix := range prefixes {
-		var prefixItem = CommonPrefix{}
+		prefixItem := CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
 		commonPrefixes = append(commonPrefixes, prefixItem)
 	}
@@ -787,9 +786,9 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "120")
 	case "InvalidRegion":
-		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalServerRegion)
+		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region)
 	case "AuthorizationHeaderMalformed":
-		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
+		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 	}
 
 	// Generate error response.
@@ -821,8 +820,8 @@ func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIE
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)
 func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError,
-	errBody string, reqURL *url.URL) {
-
+	errBody string, reqURL *url.URL,
+) {
 	reqInfo := logger.GetReqInfo(ctx)
 	errorResponse := APIErrorResponse{
 		Code:       err.Code,

cmd/api-router.go

@@ -484,7 +484,6 @@ func registerAPIRouter(router *mux.Router) {
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
 	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
-
 }
 
 // corsHandler handler for CORS (Cross Origin Resource Sharing)

cmd/api-utils_test.go

@@ -44,7 +44,6 @@ func TestS3EncodeName(t *testing.T) {
 			if testCase.expectedOutput != outputText {
 				t.Errorf("Expected `%s`, got `%s`", testCase.expectedOutput, outputText)
 			}
-
 		})
 	}
 }

cmd/auth-handler.go

@@ -238,19 +238,6 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
 
-	// If LDAP claim key is set, return here.
-	if _, ok := claims.MapClaims[ldapUser]; ok {
-		return claims.Map(), nil
-	}
-
-	// Session token must have a policy, reject requests without policy
-	// claim.
-	_, pokOpenID := claims.MapClaims[iamPolicyClaimNameOpenID()]
-	_, pokSA := claims.MapClaims[iamPolicyClaimNameSA()]
-	if !pokOpenID && !pokSA {
-		return nil, errAuthentication
-	}
-
 	return claims.Map(), nil
 }
 
@@ -299,7 +286,7 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
@@ -529,7 +516,7 @@ func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool,
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypePresigned, authTypeSigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
 			return cred, owner, s3Err
 		}
@@ -596,7 +583,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {

cmd/auth-handler_test.go

@@ -369,7 +369,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -397,7 +397,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerRegion, serviceS3)
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region, serviceS3)
 		if s3Error != testCase.s3Error {
 			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
 				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
@@ -435,7 +435,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}
@@ -459,7 +459,7 @@ func TestValidateAdminSignature(t *testing.T) {
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {

cmd/background-heal-ops.go

@@ -115,7 +115,6 @@ func newHealRoutine() *healRoutine {
 		tasks:   make(chan healTask),
 		workers: workers,
 	}
-
 }
 
 // healDiskFormat - heals format.json, return value indicates if a

cmd/background-newdisks-heal-ops.go

@@ -305,7 +305,6 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 		}
 	}
 	return disksToHeal
-
 }
 
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
@@ -375,15 +374,13 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 
 			buckets, _ := z.ListBuckets(ctx)
 
-			buckets = append(buckets, BucketInfo{
-				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
-			})
-
 			// Buckets data are dispersed in multiple zones/sets, make
 			// sure to heal all bucket metadata configuration.
-			buckets = append(buckets, []BucketInfo{
-				{Name: pathJoin(minioMetaBucket, bucketMetaPrefix)},
-			}...)
+			buckets = append(buckets, BucketInfo{
+				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+			}, BucketInfo{
+				Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+			})
 
 			// Heal latest buckets first.
 			sort.Slice(buckets, func(i, j int) bool {
@@ -441,10 +438,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 								continue
 							}
 
-							logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
+							logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
+								humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
 							logger.Info("Summary:\n")
 							tracker.printTo(os.Stdout)
 							logger.LogIf(ctx, tracker.delete(ctx))
+							logger.Info("\n")
 
 							// Only upon success pop the healed disk.
 							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())

cmd/bitrot.go

@@ -212,7 +212,7 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
 // bitrotSelfTest tries to catch any issue in the bitrot implementation
 // early instead of silently corrupting data.
 func bitrotSelfTest() {
-	var checksums = map[BitrotAlgorithm]string{
+	checksums := map[BitrotAlgorithm]string{
 		SHA256:          "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004",
 		BLAKE2b512:      "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669",
 		HighwayHash256:  "39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313",

cmd/bootstrap-peer-server.go

@@ -86,14 +86,28 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 		}
 	}
 	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
-		return fmt.Errorf("Expected same MINIO_ environment variables and values")
+		var missing []string
+		var mismatching []string
+		for k, v := range s1.MinioEnv {
+			ev, ok := s2.MinioEnv[k]
+			if !ok {
+				missing = append(missing, k)
+			} else if v != ev {
+				mismatching = append(mismatching, k)
+			}
+		}
+		if len(mismatching) > 0 {
+			return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching)
+		}
+		return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing)
 	}
 	return nil
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":        {},
-	"MINIO_CERT_PASSWD": {},
+	"MINIO_OPTS":         {},
+	"MINIO_CERT_PASSWD":  {},
+	"MINIO_SERVER_DEBUG": {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {

cmd/bucket-handlers.go

@@ -211,7 +211,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
 	// Get current region.
-	region := globalServerRegion
+	region := globalSite.Region
 	if region != globalMinioDefaultRegion {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
 			Location: region,
@@ -422,11 +422,16 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	objects := make([]ObjectV, len(deleteObjectsReq.Objects))
 	// Convert object name delete objects if it has `/` in the beginning.
 	for i := range deleteObjectsReq.Objects {
 		deleteObjectsReq.Objects[i].ObjectName = trimLeadingSlash(deleteObjectsReq.Objects[i].ObjectName)
+		objects[i] = deleteObjectsReq.Objects[i].ObjectV
 	}
 
+	// Make sure to update context to print ObjectNames for multi objects.
+	ctx = updateReqContext(ctx, objects...)
+
 	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
 	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
@@ -449,7 +454,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var objectsToDelete = map[ObjectToDelete]int{}
+	objectsToDelete := map[ObjectToDelete]int{}
 	getObjectInfoFn := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfoFn = api.CacheAPI().GetObjectInfo
@@ -527,8 +532,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 		if replicateDeletes {
 			dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
-				ObjectName: object.ObjectName,
-				VersionID:  object.VersionID,
+				ObjectV: ObjectV{
+					ObjectName: object.ObjectName,
+					VersionID:  object.VersionID,
+				},
 			}, goi, opts, gerr)
 			if dsc.ReplicateAny() {
 				if object.VersionID != "" {
@@ -569,6 +576,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Disable timeouts and cancellation
+	ctx = bgContext(ctx)
+
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
 		Versioned:        versioned,
@@ -581,8 +591,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// created during DeleteMarker creation when client didn't
 		// specify a versionID.
 		objToDel := ObjectToDelete{
-			ObjectName:                    dObjects[i].ObjectName,
-			VersionID:                     dObjects[i].VersionID,
+			ObjectV: ObjectV{
+				ObjectName: dObjects[i].ObjectName,
+				VersionID:  dObjects[i].VersionID,
+			},
 			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus(),
 			VersionPurgeStatuses:          dObjects[i].ReplicationState.VersionPurgeStatusInternal,
 			DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal,
@@ -606,8 +618,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	// Generate response
-	var deleteErrors = make([]DeleteError, 0, len(deleteObjectsReq.Objects))
-	var deletedObjects = make([]DeletedObject, 0, len(deleteObjectsReq.Objects))
+	deleteErrors := make([]DeleteError, 0, len(deleteObjectsReq.Objects))
+	deletedObjects := make([]DeletedObject, 0, len(deleteObjectsReq.Objects))
 	for _, deleteResult := range deleteResults {
 		if deleteResult.errInfo.Code != "" {
 			deleteErrors = append(deleteErrors, deleteResult.errInfo)
@@ -1210,7 +1222,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	writeSuccessResponseHeadersOnly(w)
+	writeResponse(w, http.StatusOK, nil, mimeXML)
 }
 
 // DeleteBucketHandler - Delete bucket
@@ -1280,7 +1292,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 		if globalDNSConfig != nil {
 			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, pl1ease fix it manually", err2))
+				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
 			}
 		}
 		writeErrorResponse(ctx, w, apiErr, r.URL)
@@ -1586,7 +1598,10 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
 		return
@@ -1688,6 +1703,10 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if err := globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1799,7 +1818,8 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 	tgtArns := config.FilterTargetArns(
 		replication.ObjectOpts{
 			OpType:    replication.ResyncReplicationType,
-			TargetArn: arn})
+			TargetArn: arn,
+		})
 
 	if len(tgtArns) == 0 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{

cmd/bucket-handlers_test.go

@@ -81,8 +81,8 @@ func TestGetBucketLocationHandler(t *testing.T) {
 }
 
 func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
+	credentials auth.Credentials, t *testing.T,
+) {
 	// test cases with sample input and expected output.
 	testCases := []struct {
 		bucketName string
@@ -163,7 +163,6 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2(http.MethodGet, getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
-
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -210,7 +209,6 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	nilBucket := "dummy-bucket"
 	nilReq, err := newTestRequest(http.MethodGet, getBucketLocationURL("", nilBucket), 0, nil)
-
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}
@@ -225,8 +223,8 @@ func TestHeadBucketHandler(t *testing.T) {
 }
 
 func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
+	credentials auth.Credentials, t *testing.T,
+) {
 	// test cases with sample input and expected output.
 	testCases := []struct {
 		bucketName string
@@ -282,7 +280,6 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2(http.MethodHead, getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
-
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -297,7 +294,6 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 
 	// Test for Anonymous/unsigned http request.
 	anonReq, err := newTestRequest(http.MethodHead, getHEADBucketURL("", bucketName), 0, nil)
-
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
 			instanceType, bucketName, err)
@@ -315,7 +311,6 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 
 	nilBucket := "dummy-bucket"
 	nilReq, err := newTestRequest(http.MethodHead, getHEADBucketURL("", nilBucket), 0, nil)
-
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}
@@ -331,8 +326,8 @@ func TestListMultipartUploadsHandler(t *testing.T) {
 
 // testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
 func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
+	credentials auth.Credentials, t *testing.T,
+) {
 	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
 	// and success responses.
 	testCases := []struct {
@@ -552,7 +547,6 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 		testCases[6].uploadIDMarker, testCases[6].delimiter, testCases[6].maxUploads)
 
 	nilReq, err := newTestRequest(http.MethodGet, url, 0, nil)
-
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}
@@ -568,8 +562,8 @@ func TestListBucketsHandler(t *testing.T) {
 
 // testListBucketsHandler - Tests validate listing of buckets.
 func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
+	credentials auth.Credentials, t *testing.T,
+) {
 	testCases := []struct {
 		bucketName         string
 		accessKey          string
@@ -615,7 +609,6 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 
 		// verify response for V2 signed HTTP request.
 		reqV2, err := newTestSignedRequestV2(http.MethodGet, getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
-
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -630,7 +623,6 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	// Test for Anonymous/unsigned http request.
 	// ListBucketsHandler doesn't support bucket policies, setting the policies shouldn't make a difference.
 	anonReq, err := newTestRequest(http.MethodGet, getListBucketURL(""), 0, nil)
-
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request.", instanceType)
 	}
@@ -646,7 +638,6 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
 
 	nilReq, err := newTestRequest(http.MethodGet, getListBucketURL(""), 0, nil)
-
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}
@@ -661,8 +652,8 @@ func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
 }
 
 func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
+	credentials auth.Credentials, t *testing.T,
+) {
 	var err error
 
 	contentBytes := []byte("hello")
@@ -707,7 +698,9 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) {
 		for _, objectName := range objectNames {
 			objectList = append(objectList, ObjectToDelete{
-				ObjectName: objectName,
+				ObjectV: ObjectV{
+					ObjectName: objectName,
+				},
 			})
 		}
 
@@ -726,10 +719,21 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		return deleteErrorList
 	}
 
+	objects := []ObjectToDelete{}
+	objects = append(objects, ObjectToDelete{
+		ObjectV: ObjectV{
+			ObjectName: "private/object",
+		},
+	})
+	objects = append(objects, ObjectToDelete{
+		ObjectV: ObjectV{
+			ObjectName: "public/object",
+		},
+	})
 	requestList := []DeleteObjectsRequest{
 		{Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])},
 		{Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])},
-		{Quiet: false, Objects: []ObjectToDelete{{ObjectName: "private/object"}, {ObjectName: "public/object"}}},
+		{Quiet: false, Objects: objects},
 	}
 
 	// generate multi objects delete response.

cmd/bucket-lifecycle-handlers.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/bucket/object/lock"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
@@ -80,17 +79,6 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Disallow MaxNoncurrentVersions if bucket has object locking enabled
-	var rCfg lock.Retention
-	if rCfg, err = globalBucketObjectLockSys.Get(bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	if rCfg.LockEnabled && bucketLifecycle.HasMaxNoncurrentVersions() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidLifecycleWithObjectLock), r.URL)
-		return
-	}
-
 	// Validate the transition storage ARNs
 	if err = validateTransitionTier(bucketLifecycle); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)

cmd/bucket-lifecycle-handlers_test.go

@@ -150,8 +150,8 @@ func TestBucketLifecycle(t *testing.T) {
 // Simple tests of bucket lifecycle: PUT, GET, DELETE.
 // Tests are related and the order is important.
 func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	creds auth.Credentials, t *testing.T) {
-
+	creds auth.Credentials, t *testing.T,
+) {
 	// test cases with sample input and expected output.
 	testCases := []struct {
 		method     string
@@ -266,8 +266,8 @@ func testBucketLifecycle(obj ObjectLayer, instanceType, bucketName string, apiRo
 		lifecycleResponse  []byte
 		errorResponse      APIErrorResponse
 		shouldPass         bool
-	}) {
-
+	},
+) {
 	for i, testCase := range testCases {
 		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()

cmd/bucket-lifecycle.go

@@ -81,21 +81,21 @@ type expiryTask struct {
 }
 
 type expiryState struct {
-	once              sync.Once
-	byDaysCh          chan expiryTask
-	byMaxNoncurrentCh chan maxNoncurrentTask
+	once                sync.Once
+	byDaysCh            chan expiryTask
+	byNewerNoncurrentCh chan newerNoncurrentTask
 }
 
 // PendingTasks returns the number of pending ILM expiry tasks.
 func (es *expiryState) PendingTasks() int {
-	return len(es.byDaysCh) + len(es.byMaxNoncurrentCh)
+	return len(es.byDaysCh) + len(es.byNewerNoncurrentCh)
 }
 
 // close closes work channels exactly once.
 func (es *expiryState) close() {
 	es.once.Do(func() {
 		close(es.byDaysCh)
-		close(es.byMaxNoncurrentCh)
+		close(es.byNewerNoncurrentCh)
 	})
 }
 
@@ -109,13 +109,13 @@ func (es *expiryState) enqueueByDays(oi ObjectInfo, restoredObject bool, rmVersi
 	}
 }
 
-// enqueueByMaxNoncurrent enqueues object versions expired by
-// MaxNoncurrentVersions limit for expiry.
-func (es *expiryState) enqueueByMaxNoncurrent(bucket string, versions []ObjectToDelete) {
+// enqueueByNewerNoncurrent enqueues object versions expired by
+// NewerNoncurrentVersions limit for expiry.
+func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byMaxNoncurrentCh <- maxNoncurrentTask{bucket: bucket, versions: versions}:
+	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions}:
 	default:
 	}
 }
@@ -124,8 +124,8 @@ var globalExpiryState *expiryState
 
 func newExpiryState() *expiryState {
 	return &expiryState{
-		byDaysCh:          make(chan expiryTask, 10000),
-		byMaxNoncurrentCh: make(chan maxNoncurrentTask, 10000),
+		byDaysCh:            make(chan expiryTask, 10000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
 	}
 }
 
@@ -141,15 +141,15 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 		}
 	}()
 	go func() {
-		for t := range globalExpiryState.byMaxNoncurrentCh {
+		for t := range globalExpiryState.byNewerNoncurrentCh {
 			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
 		}
 	}()
 }
 
-// maxNoncurrentTask encapsulates arguments required by worker to expire objects
-// by MaxNoncurrentVersions
-type maxNoncurrentTask struct {
+// newerNoncurrentTask encapsulates arguments required by worker to expire objects
+// by NewerNoncurrentVersions
+type newerNoncurrentTask struct {
 	bucket   string
 	versions []ObjectToDelete
 }
@@ -178,9 +178,7 @@ func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
 	}
 }
 
-var (
-	globalTransitionState *transitionState
-)
+var globalTransitionState *transitionState
 
 func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
 	return &transitionState{
@@ -466,9 +464,7 @@ func (sp *SelectParameters) IsEmpty() bool {
 	return sp == nil
 }
 
-var (
-	selectParamsXMLName = "SelectParameters"
-)
+var selectParamsXMLName = "SelectParameters"
 
 // UnmarshalXML - decodes XML data.
 func (sp *SelectParameters) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {

cmd/bucket-metadata.go

@@ -120,7 +120,7 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
 		return errors.New("bucket name cannot be empty")
 	}
-	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
+	configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)
 	if err != nil {
 		return err
@@ -277,7 +277,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 	}
 
 	for _, legacyFile := range legacyConfigs {
-		configFile := path.Join(bucketConfigPrefix, b.Name, legacyFile)
+		configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile)
 
 		configData, err := readConfig(ctx, objectAPI, configFile)
 		if err != nil {
@@ -338,7 +338,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 	}
 
 	for legacyFile := range configs {
-		configFile := path.Join(bucketConfigPrefix, b.Name, legacyFile)
+		configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile)
 		if err := deleteConfig(ctx, objectAPI, configFile); err != nil && !errors.Is(err, errConfigNotFound) {
 			logger.LogIf(ctx, err)
 		}
@@ -365,7 +365,7 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 		return err
 	}
 
-	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
+	configFile := path.Join(bucketMetaPrefix, b.Name, bucketMetadataFile)
 	return saveConfig(ctx, api, configFile, data)
 }
 
@@ -377,7 +377,7 @@ func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string)
 		bucketMetadataFile,
 	}
 	for _, metaFile := range metadataFiles {
-		configFile := path.Join(bucketConfigPrefix, bucket, metaFile)
+		configFile := path.Join(bucketMetaPrefix, bucket, metaFile)
 		if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
 			return err
 		}

cmd/bucket-notification-handlers.go

@@ -30,7 +30,6 @@ import (
 )
 
 const (
-	bucketConfigPrefix       = "buckets"
 	bucketNotificationConfig = "notification.xml"
 )
 
@@ -72,8 +71,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	config.SetRegion(globalServerRegion)
-	if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
+	config.SetRegion(globalSite.Region)
+	if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
 		if ok {
 			for i, queue := range config.QueueList {
@@ -145,7 +144,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerRegion, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {

cmd/bucket-object-lock.go

@@ -58,6 +58,10 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
 // enforceRetentionForDeletion checks if it is appropriate to remove an
 // object according to locking configuration when this is lifecycle/ bucket quota asking.
 func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locked bool) {
+	if objInfo.DeleteMarker {
+		return false
+	}
+
 	lhold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
 	if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn {
 		return true

cmd/bucket-policy-handlers_test.go

@@ -105,7 +105,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
 
 	const n = 100
-	var start = make(chan struct{})
+	start := make(chan struct{})
 	var ok, errs int
 	var wg sync.WaitGroup
 	var mu sync.Mutex
@@ -147,8 +147,8 @@ func TestPutBucketPolicyHandler(t *testing.T) {
 
 // testPutBucketPolicyHandler - Test for Bucket policy end point.
 func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
+	credentials auth.Credentials, t *testing.T,
+) {
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
 	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
 		t.Fatal(err)
@@ -333,7 +333,6 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// create unsigned HTTP request for PutBucketPolicyHandler.
 	anonReq, err := newTestRequest(http.MethodPut, getPutPolicyURL("", bucketName),
 		int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)))
-
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
 			instanceType, bucketName, err)
@@ -352,14 +351,12 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 	nilReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", nilBucket),
 		0, nil, "", "", nil)
-
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}
 	// execute the object layer set to `nil` test.
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
-
 }
 
 // Wrapper for calling Get Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup.
@@ -465,7 +462,6 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", testCase.bucketName),
 			0, nil, testCase.accessKey, testCase.secretKey, nil)
-
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -540,7 +536,6 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
 	// create unsigned HTTP request for PutBucketPolicyHandler.
 	anonReq, err := newTestRequest(http.MethodGet, getPutPolicyURL("", bucketName), 0, nil)
-
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
 			instanceType, bucketName, err)
@@ -559,7 +554,6 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 	nilReq, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", nilBucket),
 		0, nil, "", "", nil)
-
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}
@@ -575,8 +569,8 @@ func TestDeleteBucketPolicyHandler(t *testing.T) {
 
 // testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
 func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
+	credentials auth.Credentials, t *testing.T,
+) {
 	// template for constructing HTTP request body for PUT bucket policy.
 	bucketPolicyTemplate := `{
     "Version": "2012-10-17",
@@ -743,7 +737,6 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
 	// create unsigned HTTP request for PutBucketPolicyHandler.
 	anonReq, err := newTestRequest(http.MethodDelete, getPutPolicyURL("", bucketName), 0, nil)
-
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
 			instanceType, bucketName, err)
@@ -762,7 +755,6 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 
 	nilReq, err := newTestSignedRequestV4(http.MethodDelete, getDeletePolicyURL("", nilBucket),
 		0, nil, "", "", nil)
-
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}

cmd/bucket-policy.go

@@ -200,7 +200,7 @@ func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.Buc
 	}
 
 	var policyInfo miniogopolicy.BucketAccessPolicy
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(data, &policyInfo); err != nil {
 		// This should not happen because data is valid to JSON data.
 		return nil, err
@@ -218,7 +218,7 @@ func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*
 	}
 
 	var bucketPolicy policy.Policy
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(data, &bucketPolicy); err != nil {
 		// This should not happen because data is valid to JSON data.
 		return nil, err

cmd/bucket-quota.go

@@ -154,7 +154,7 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 	// ModTime to find the oldest objects in bucket to delete. In
 	// the context of bucket quota enforcement - number of hits are
 	// irrelevant.
-	scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
+	scorer, err := newFileScorer(toFree, UTCNow().Unix(), 1)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return
@@ -162,11 +162,6 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 
 	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
 	for obj := range objInfoCh {
-		if obj.DeleteMarker {
-			// Delete markers are automatically added for FIFO purge.
-			scorer.addFileWithObjInfo(obj, 1)
-			continue
-		}
 		// skip objects currently under retention
 		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			continue
@@ -178,6 +173,7 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 	if scorer.seenBytes <= cfg.Quota {
 		return
 	}
+
 	// Calculate how much we want to delete now.
 	toFreeNow := scorer.seenBytes - cfg.Quota
 	// We were less over quota than we thought. Adjust so we delete less.
@@ -193,8 +189,10 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 	numKeys := len(scorer.fileObjInfos())
 	for i, obj := range scorer.fileObjInfos() {
 		objects = append(objects, ObjectToDelete{
-			ObjectName: obj.Name,
-			VersionID:  obj.VersionID,
+			ObjectV: ObjectV{
+				ObjectName: obj.Name,
+				VersionID:  obj.VersionID,
+			},
 		})
 		if len(objects) < maxDeleteList && (i < numKeys-1) {
 			// skip deletion until maxDeleteList or end of slice
@@ -209,9 +207,9 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 		_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
 			Versioned: versioned,
 		})
-		for i := range deleteErrs {
-			if deleteErrs[i] != nil {
-				logger.LogIf(ctx, deleteErrs[i])
+		for j := range deleteErrs {
+			if deleteErrs[j] != nil {
+				logger.LogIf(ctx, deleteErrs[j])
 				continue
 			}
 

cmd/bucket-replication-stats.go

@@ -51,8 +51,10 @@ func (r *ReplicationStats) Delete(bucket string) {
 	r.Lock()
 	defer r.Unlock()
 	delete(r.Cache, bucket)
-	delete(r.UsageCache, bucket)
 
+	r.ulock.Lock()
+	defer r.ulock.Unlock()
+	delete(r.UsageCache, bucket)
 }
 
 // UpdateReplicaStat updates in-memory replica statistics with new values.
@@ -82,10 +84,12 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 	bs, ok := r.Cache[bucket]
 	if !ok {
 		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		r.Cache[bucket] = bs
 	}
 	b, ok := bs.Stats[arn]
 	if !ok {
 		b = &BucketReplicationStat{}
+		bs.Stats[arn] = b
 	}
 	switch status {
 	case replication.Completed:
@@ -115,9 +119,6 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 			b.ReplicaSize += n
 		}
 	}
-
-	bs.Stats[arn] = b
-	r.Cache[bucket] = bs
 }
 
 // GetInitialUsage get replication metrics available at the time of cluster initialization
@@ -128,10 +129,10 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 	r.ulock.RLock()
 	defer r.ulock.RUnlock()
 	st, ok := r.UsageCache[bucket]
-	if ok {
-		return st.Clone()
+	if !ok {
+		return BucketReplicationStats{}
 	}
-	return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+	return st.Clone()
 }
 
 // Get replication metrics for a bucket from this node since this node came up.
@@ -145,7 +146,7 @@ func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 
 	st, ok := r.Cache[bucket]
 	if !ok {
-		return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		return BucketReplicationStats{}
 	}
 	return st.Clone()
 }
@@ -162,41 +163,46 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
 	rTimer := time.NewTimer(time.Minute * 1)
 	defer rTimer.Stop()
+	var (
+		dui DataUsageInfo
+		err error
+	)
+outer:
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-rTimer.C:
-			dui, err := loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
+			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
 			if err != nil {
 				continue
 			}
-			// data usage has not captured any data yet.
-			if dui.LastUpdate.IsZero() {
-				continue
+			// If LastUpdate is set, data usage is available.
+			if !dui.LastUpdate.IsZero() {
+				break outer
 			}
-			m := make(map[string]*BucketReplicationStats)
-			for bucket, usage := range dui.BucketsUsage {
-				b := &BucketReplicationStats{
-					Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
-				}
-				for arn, uinfo := range usage.ReplicationInfo {
-					b.Stats[arn] = &BucketReplicationStat{
-						FailedSize:     int64(uinfo.ReplicationFailedSize),
-						ReplicatedSize: int64(uinfo.ReplicatedSize),
-						ReplicaSize:    int64(uinfo.ReplicaSize),
-						FailedCount:    int64(uinfo.ReplicationFailedCount),
-					}
-				}
-				b.ReplicaSize += int64(usage.ReplicaSize)
-				if b.hasReplicationUsage() {
-					m[bucket] = b
-				}
-			}
-			r.ulock.Lock()
-			defer r.ulock.Unlock()
-			r.UsageCache = m
-			return
 		}
 	}
+
+	m := make(map[string]*BucketReplicationStats)
+	for bucket, usage := range dui.BucketsUsage {
+		b := &BucketReplicationStats{
+			Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
+		}
+		for arn, uinfo := range usage.ReplicationInfo {
+			b.Stats[arn] = &BucketReplicationStat{
+				FailedSize:     int64(uinfo.ReplicationFailedSize),
+				ReplicatedSize: int64(uinfo.ReplicatedSize),
+				ReplicaSize:    int64(uinfo.ReplicaSize),
+				FailedCount:    int64(uinfo.ReplicationFailedCount),
+			}
+		}
+		b.ReplicaSize += int64(usage.ReplicaSize)
+		if b.hasReplicationUsage() {
+			m[bucket] = b
+		}
+	}
+	r.ulock.Lock()
+	defer r.ulock.Unlock()
+	r.UsageCache = m
 }

cmd/bucket-replication-utils.go

@@ -502,8 +502,10 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 	var tgtStatuses map[string]replication.StatusType
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
-			ObjectName: oi.Name,
-			VersionID:  oi.VersionID,
+			ObjectV: ObjectV{
+				ObjectName: oi.Name,
+				VersionID:  oi.VersionID,
+			},
 		}, oi, ObjectOptions{}, nil)
 	} else {
 		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{

cmd/bucket-replication-utils_test.go

@@ -77,7 +77,8 @@ var replicatedInfosTests = []struct {
 				ReplicationStatus:     replication.Failed,
 				OpType:                replication.ObjectReplicationType,
 				ReplicationAction:     replicateAll,
-			}},
+			},
+		},
 		expectedCompletedSize:             249,
 		expectedReplicationStatusInternal: "arn1=COMPLETED;arn2=FAILED;",
 		expectedReplicationStatus:         replication.Failed,
@@ -102,7 +103,8 @@ var replicatedInfosTests = []struct {
 				ReplicationStatus:     replication.Failed,
 				OpType:                replication.ObjectReplicationType,
 				ReplicationAction:     replicateAll,
-			}},
+			},
+		},
 		expectedCompletedSize:             0,
 		expectedReplicationStatusInternal: "arn1=PENDING;arn2=FAILED;",
 		expectedReplicationStatus:         replication.Failed,
@@ -182,7 +184,6 @@ var parseReplicationDecisionTest = []struct {
 func TestParseReplicateDecision(t *testing.T) {
 	for i, test := range parseReplicationDecisionTest {
 		dsc, err := parseReplicateDecision(test.expDsc.String())
-
 		if err != nil {
 			if test.expErr != err {
 				t.Errorf("Test%d (%s): Expected parse error got %t , want %t", i+1, test.name, err, test.expErr)

cmd/bucket-replication.go

@@ -62,6 +62,8 @@ const (
 	ObjectLockRetentionTimestamp = "objectlock-retention-timestamp"
 	// ObjectLockLegalHoldTimestamp - the last time a legal hold metadata modification happened on this cluster for this object version
 	ObjectLockLegalHoldTimestamp = "objectlock-legalhold-timestamp"
+	// ReplicationWorkerMultiplier is suggested worker multiplier if traffic exceeds replication worker capacity
+	ReplicationWorkerMultiplier = 1.5
 )
 
 // gets replication config associated to a given bucket name.
@@ -137,6 +139,7 @@ func (o mustReplicateOptions) ReplicationStatus() (s replication.StatusType) {
 	}
 	return s
 }
+
 func (o mustReplicateOptions) isExistingObjectReplication() bool {
 	return o.opType == replication.ExistingObjectReplicationType
 }
@@ -144,6 +147,7 @@ func (o mustReplicateOptions) isExistingObjectReplication() bool {
 func (o mustReplicateOptions) isMetadataReplication() bool {
 	return o.opType == replication.MetadataReplicationType
 }
+
 func getMustReplicateOptions(o ObjectInfo, op replication.Type, opts ObjectOptions) mustReplicateOptions {
 	if !op.Valid() {
 		op = replication.ObjectReplicationType
@@ -439,7 +443,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		}
 	}
 
-	var eventName = event.ObjectReplicationComplete
+	eventName := event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
 	}
@@ -515,19 +519,18 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		}
 		return
 	}
-	// early return if already replicated delete marker for existing object replication
-	if dobj.DeleteMarkerVersionID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
+	// early return if already replicated delete marker for existing object replication/ healing delete markers
+	if dobj.DeleteMarkerVersionID != "" && (dobj.OpType == replication.ExistingObjectReplicationType || dobj.OpType == replication.HealReplicationType) {
 		if _, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, miniogo.StatObjectOptions{
 			VersionID: versionID,
 			Internal: miniogo.AdvancedGetOptions{
 				ReplicationProxyRequest: "false",
-			}}); isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
+			},
+		}); isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
 			if dobj.VersionID == "" {
 				rinfo.ReplicationStatus = replication.Completed
-			} else {
-				rinfo.VersionPurgeStatus = Complete
+				return
 			}
-			return
 		}
 	}
 
@@ -902,7 +905,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	// FIXME: add support for missing replication events
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
-	var eventName = event.ObjectReplicationComplete
+	eventName := event.ObjectReplicationComplete
 	if rinfos.ReplicationStatus() == replication.Failed {
 		eventName = event.ObjectReplicationFailed
 	}
@@ -1058,7 +1061,8 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 		VersionID: objInfo.VersionID,
 		Internal: miniogo.AdvancedGetOptions{
 			ReplicationProxyRequest: "false",
-		}})
+		},
+	})
 	if cerr == nil {
 		rAction = getReplicationAction(objInfo, oi, ri.OpType)
 		rinfo.ReplicationStatus = replication.Completed
@@ -1117,7 +1121,8 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 			Internal: miniogo.AdvancedPutOptions{
 				SourceVersionID:    objInfo.VersionID,
 				ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
-			}}
+			},
+		}
 		if _, err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); err != nil {
 			rinfo.ReplicationStatus = replication.Failed
 			logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
@@ -1213,7 +1218,8 @@ func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket,
 			SourceMTime: objInfo.ModTime,
 			// always set this to distinguish between `mc mirror` replication and serverside
 			ReplicationRequest: true,
-		}})
+		},
+	})
 	return err
 }
 
@@ -1357,7 +1363,6 @@ func (p *ReplicationPool) AddWorker() {
 			return
 		}
 	}
-
 }
 
 // AddExistingObjectReplicateWorker adds a worker to queue existing objects that need to be sync'd
@@ -1412,6 +1417,14 @@ func (p *ReplicationPool) ResizeFailedWorkers(n int) {
 	}
 }
 
+// suggestedWorkers recommends an increase in number of workers to meet replication load.
+func (p *ReplicationPool) suggestedWorkers(failQueue bool) int {
+	if failQueue {
+		return int(float64(p.mrfWorkerSize) * ReplicationWorkerMultiplier)
+	}
+	return int(float64(p.workerSize) * ReplicationWorkerMultiplier)
+}
+
 func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
@@ -1425,6 +1438,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication failed workers could not keep up with healing failures - consider increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
 	}
 }
 
@@ -1450,6 +1464,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		})
 	case ch <- ri:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1486,6 +1501,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1660,6 +1676,7 @@ type replicationConfig struct {
 func (c replicationConfig) Empty() bool {
 	return c.Config == nil
 }
+
 func (c replicationConfig) Replicate(opts replication.ObjectOpts) bool {
 	return c.Config.Replicate(opts)
 }
@@ -1683,7 +1700,8 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli
 			DeleteMarker:   oi.DeleteMarker,
 			VersionID:      oi.VersionID,
 			OpType:         replication.DeleteReplicationType,
-			ExistingObject: true}
+			ExistingObject: true,
+		}
 
 		tgtArns := c.Config.FilterTargetArns(opts)
 		// indicates no matching target with Existing object replication enabled.
@@ -1797,22 +1815,15 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 	// add initial usage stat to cluster stats
 	usageStat := globalReplicationStats.GetInitialUsage(bucket)
 	totReplicaSize += usageStat.ReplicaSize
-	if usageStat.Stats != nil {
-		for arn, stat := range usageStat.Stats {
-			st := stats[arn]
-			if st == nil {
-				st = &BucketReplicationStat{
-					ReplicatedSize: stat.ReplicatedSize,
-					FailedSize:     stat.FailedSize,
-					FailedCount:    stat.FailedCount,
-				}
-			} else {
-				st.ReplicatedSize += stat.ReplicatedSize
-				st.FailedSize += stat.FailedSize
-				st.FailedCount += stat.FailedCount
-			}
+	for arn, stat := range usageStat.Stats {
+		st, ok := stats[arn]
+		if !ok {
+			st = &BucketReplicationStat{}
 			stats[arn] = st
 		}
+		st.ReplicatedSize += stat.ReplicatedSize
+		st.FailedSize += stat.FailedSize
+		st.FailedCount += stat.FailedCount
 	}
 	s = BucketReplicationStats{
 		Stats: make(map[string]*BucketReplicationStat, len(stats)),

cmd/bucket-replication_test.go

@@ -75,7 +75,8 @@ var replicationConfigTests = []struct {
 	},
 	{ // 4. existing object replication enabled, versioning enabled; no reset in progress
 		name: "existing object replication enabled, versioning enabled; no reset in progress",
-		info: ObjectInfo{Size: 100,
+		info: ObjectInfo{
+			Size:              100,
 			ReplicationStatus: replication.Completed,
 			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
@@ -93,174 +94,192 @@ func TestReplicationResync(t *testing.T) {
 	}
 }
 
-var start = UTCNow().AddDate(0, 0, -1)
-var replicationConfigTests2 = []struct {
-	info         ObjectInfo
-	name         string
-	rcfg         replicationConfig
-	dsc          ReplicateDecision
-	tgtStatuses  map[string]replication.StatusType
-	expectedSync bool
-}{
-	{ // Cases 1-4: existing object replication enabled, versioning enabled, no reset - replication status varies
-		// 1: Pending replication
-		name: "existing object replication on object in Pending replication status",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:PENDING;",
-			ReplicationStatus:         replication.Pending,
-			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
-		},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn: "arn1",
-		}}}},
-		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		expectedSync: true,
-	},
-
-	{ // 2. replication status Failed
-		name: "existing object replication on object in Failed replication status",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:FAILED",
-			ReplicationStatus:         replication.Failed,
-			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
-		},
-		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn: "arn1",
-		}}}},
-		expectedSync: true,
-	},
-	{ // 3. replication status unset
-		name: "existing object replication on pre-existing unreplicated object",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.StatusType(""),
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
-		},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn: "arn1",
-		}}}},
-		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		expectedSync: true,
-	},
-	{ // 4. replication status Complete
-		name: "existing object replication on object in Completed replication status",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:COMPLETED",
-			ReplicationStatus:         replication.Completed,
-			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
-		},
-		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", false, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn: "arn1",
-		}}}},
-		expectedSync: false,
-	},
-	{ // 5. existing object replication enabled, versioning enabled, replication status Pending & reset ID present
-		name: "existing object replication with reset in progress and object in Pending status",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:PENDING;",
-			ReplicationStatus:         replication.Pending,
-			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
-		},
-		expectedSync: true,
-		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn:             "arn1",
-			ResetID:         "xyz",
-			ResetBeforeDate: UTCNow(),
-		}}},
-		},
-	},
-	{ // 6. existing object replication enabled, versioning enabled, replication status Failed & reset ID present
-		name: "existing object replication with reset in progress and object in Failed status",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:FAILED;",
-			ReplicationStatus:         replication.Failed,
-			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
-		},
-		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn:             "arn1",
-			ResetID:         "xyz",
-			ResetBeforeDate: UTCNow(),
-		}}},
-		},
-		expectedSync: true,
-	},
-	{ // 7. existing object replication enabled, versioning enabled, replication status unset & reset ID present
-		name: "existing object replication with reset in progress and object never replicated before",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.StatusType(""),
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
-		},
-		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn:             "arn1",
-			ResetID:         "xyz",
-			ResetBeforeDate: UTCNow(),
-		}}},
+var (
+	start                   = UTCNow().AddDate(0, 0, -1)
+	replicationConfigTests2 = []struct {
+		info         ObjectInfo
+		name         string
+		rcfg         replicationConfig
+		dsc          ReplicateDecision
+		tgtStatuses  map[string]replication.StatusType
+		expectedSync bool
+	}{
+		{ // Cases 1-4: existing object replication enabled, versioning enabled, no reset - replication status varies
+			// 1: Pending replication
+			name: "existing object replication on object in Pending replication status",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:PENDING;",
+				ReplicationStatus:         replication.Pending,
+				VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			},
+			rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+				Arn: "arn1",
+			}}}},
+			dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			expectedSync: true,
 		},
 
-		expectedSync: true,
-	},
+		{ // 2. replication status Failed
+			name: "existing object replication on object in Failed replication status",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:FAILED",
+				ReplicationStatus:         replication.Failed,
+				VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			},
+			dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+				Arn: "arn1",
+			}}}},
+			expectedSync: true,
+		},
+		{ // 3. replication status unset
+			name: "existing object replication on pre-existing unreplicated object",
+			info: ObjectInfo{
+				Size:              100,
+				ReplicationStatus: replication.StatusType(""),
+				VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			},
+			rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+				Arn: "arn1",
+			}}}},
+			dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			expectedSync: true,
+		},
+		{ // 4. replication status Complete
+			name: "existing object replication on object in Completed replication status",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:COMPLETED",
+				ReplicationStatus:         replication.Completed,
+				VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			},
+			dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", false, false)}},
+			rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+				Arn: "arn1",
+			}}}},
+			expectedSync: false,
+		},
+		{ // 5. existing object replication enabled, versioning enabled, replication status Pending & reset ID present
+			name: "existing object replication with reset in progress and object in Pending status",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:PENDING;",
+				ReplicationStatus:         replication.Pending,
+				VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+				UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
+			},
+			expectedSync: true,
+			dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			rcfg: replicationConfig{
+				remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+					Arn:             "arn1",
+					ResetID:         "xyz",
+					ResetBeforeDate: UTCNow(),
+				}}},
+			},
+		},
+		{ // 6. existing object replication enabled, versioning enabled, replication status Failed & reset ID present
+			name: "existing object replication with reset in progress and object in Failed status",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:FAILED;",
+				ReplicationStatus:         replication.Failed,
+				VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+				UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
+			},
+			dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			rcfg: replicationConfig{
+				remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+					Arn:             "arn1",
+					ResetID:         "xyz",
+					ResetBeforeDate: UTCNow(),
+				}}},
+			},
+			expectedSync: true,
+		},
+		{ // 7. existing object replication enabled, versioning enabled, replication status unset & reset ID present
+			name: "existing object replication with reset in progress and object never replicated before",
+			info: ObjectInfo{
+				Size:              100,
+				ReplicationStatus: replication.StatusType(""),
+				VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+				UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
+			},
+			dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			rcfg: replicationConfig{
+				remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+					Arn:             "arn1",
+					ResetID:         "xyz",
+					ResetBeforeDate: UTCNow(),
+				}}},
+			},
 
-	{ // 8. existing object replication enabled, versioning enabled, replication status Complete & reset ID present
-		name: "existing object replication enabled - reset in progress for an object in Completed status",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:COMPLETED;",
-			ReplicationStatus:         replication.Completed,
-			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df8",
-			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
+			expectedSync: true,
 		},
-		expectedSync: true,
-		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn:             "arn1",
-			ResetID:         "xyz",
-			ResetBeforeDate: UTCNow(),
-		}}},
-		},
-	},
-	{ // 9. existing object replication enabled, versioning enabled, replication status Pending & reset ID different
-		name: "existing object replication enabled, newer reset in progress on object in Pending replication status",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:PENDING;",
 
-			ReplicationStatus: replication.Pending,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().AddDate(0, 0, -1).Format(http.TimeFormat), "abc")},
-			ModTime:           UTCNow().AddDate(0, 0, -2),
+		{ // 8. existing object replication enabled, versioning enabled, replication status Complete & reset ID present
+			name: "existing object replication enabled - reset in progress for an object in Completed status",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:COMPLETED;",
+				ReplicationStatus:         replication.Completed,
+				VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df8",
+				UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
+			},
+			expectedSync: true,
+			dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			rcfg: replicationConfig{
+				remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+					Arn:             "arn1",
+					ResetID:         "xyz",
+					ResetBeforeDate: UTCNow(),
+				}}},
+			},
 		},
-		expectedSync: true,
-		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn:             "arn1",
-			ResetID:         "xyz",
-			ResetBeforeDate: UTCNow(),
-		}}},
+		{ // 9. existing object replication enabled, versioning enabled, replication status Pending & reset ID different
+			name: "existing object replication enabled, newer reset in progress on object in Pending replication status",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:PENDING;",
+
+				ReplicationStatus: replication.Pending,
+				VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+				UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().AddDate(0, 0, -1).Format(http.TimeFormat), "abc")},
+				ModTime:           UTCNow().AddDate(0, 0, -2),
+			},
+			expectedSync: true,
+			dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			rcfg: replicationConfig{
+				remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+					Arn:             "arn1",
+					ResetID:         "xyz",
+					ResetBeforeDate: UTCNow(),
+				}}},
+			},
 		},
-	},
-	{ // 10. existing object replication enabled, versioning enabled, replication status Complete & reset done
-		name: "reset done on object in Completed Status - ineligbile for re-replication",
-		info: ObjectInfo{Size: 100,
-			ReplicationStatusInternal: "arn1:COMPLETED;",
-			ReplicationStatus:         replication.Completed,
-			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")},
+		{ // 10. existing object replication enabled, versioning enabled, replication status Complete & reset done
+			name: "reset done on object in Completed Status - ineligbile for re-replication",
+			info: ObjectInfo{
+				Size:                      100,
+				ReplicationStatusInternal: "arn1:COMPLETED;",
+				ReplicationStatus:         replication.Completed,
+				VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+				UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")},
+			},
+			expectedSync: false,
+			dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+			rcfg: replicationConfig{
+				remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+					Arn:             "arn1",
+					ResetID:         "xyz",
+					ResetBeforeDate: start,
+				}}},
+			},
 		},
-		expectedSync: false,
-		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
-		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
-			Arn:             "arn1",
-			ResetID:         "xyz",
-			ResetBeforeDate: start,
-		}}},
-		},
-	},
-}
+	}
+)
 
 func TestReplicationResyncwrapper(t *testing.T) {
 	for i, test := range replicationConfigTests2 {

cmd/bucket-stats.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"sync/atomic"
 	"time"
 )
 
@@ -52,13 +51,6 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
 	rl.UploadHistogram.Add(size, duration)
 }
 
-// Clone replication latency
-func (rl ReplicationLatency) clone() ReplicationLatency {
-	return ReplicationLatency{
-		UploadHistogram: rl.UploadHistogram.Clone(),
-	}
-}
-
 // BucketStats bucket statistics
 type BucketStats struct {
 	ReplicationStats BucketReplicationStats
@@ -88,29 +80,18 @@ func (brs *BucketReplicationStats) Empty() bool {
 }
 
 // Clone creates a new BucketReplicationStats copy
-func (brs BucketReplicationStats) Clone() BucketReplicationStats {
-	c := BucketReplicationStats{
-		Stats: make(map[string]*BucketReplicationStat, len(brs.Stats)),
-	}
-	// This is called only by replicationStats cache and already holds a read lock before calling Clone()
+func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) {
+	// This is called only by replicationStats cache and already holds a
+	// read lock before calling Clone()
+
+	c = brs
+	// We need to copy the map, so we do not reference the one in `brs`.
+	c.Stats = make(map[string]*BucketReplicationStat, len(brs.Stats))
 	for arn, st := range brs.Stats {
-		c.Stats[arn] = &BucketReplicationStat{
-			FailedSize:     atomic.LoadInt64(&st.FailedSize),
-			ReplicatedSize: atomic.LoadInt64(&st.ReplicatedSize),
-			ReplicaSize:    atomic.LoadInt64(&st.ReplicaSize),
-			FailedCount:    atomic.LoadInt64(&st.FailedCount),
-			PendingSize:    atomic.LoadInt64(&st.PendingSize),
-			PendingCount:   atomic.LoadInt64(&st.PendingCount),
-			Latency:        st.Latency.clone(),
-		}
+		// make a copy of `*st`
+		s := *st
+		c.Stats[arn] = &s
 	}
-	// update total counts across targets
-	c.FailedSize = atomic.LoadInt64(&brs.FailedSize)
-	c.FailedCount = atomic.LoadInt64(&brs.FailedCount)
-	c.PendingCount = atomic.LoadInt64(&brs.PendingCount)
-	c.PendingSize = atomic.LoadInt64(&brs.PendingSize)
-	c.ReplicaSize = atomic.LoadInt64(&brs.ReplicaSize)
-	c.ReplicatedSize = atomic.LoadInt64(&brs.ReplicatedSize)
 	return c
 }
 

cmd/bucket-targets.go

@@ -332,8 +332,10 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 }
 
 // getRemoteTargetInstanceTransport contains a singleton roundtripper.
-var getRemoteTargetInstanceTransport http.RoundTripper
-var getRemoteTargetInstanceTransportOnce sync.Once
+var (
+	getRemoteTargetInstanceTransport     http.RoundTripper
+	getRemoteTargetInstanceTransportOnce sync.Once
+)
 
 // Returns a minio-go Client configured to access remote host described in replication target config.
 func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {
@@ -414,7 +416,7 @@ func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.Bu
 		return nil, nil
 	}
 	data = cdata
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	if len(cmetadata) != 0 {
 		if err := json.Unmarshal(cmetadata, &meta); err != nil {
 			return nil, err

cmd/bucket-versioning-handler.go

@@ -63,6 +63,15 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "InvalidBucketState",
+			Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
+			HTTPStatusCode: http.StatusConflict,
+		}, r.URL)
+		return
+	}
+
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
@@ -135,5 +144,4 @@ func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r
 
 	// Write bucket versioning configuration to client
 	writeSuccessResponseXML(w, configData)
-
 }

cmd/common-main.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -34,6 +37,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	fcolor "github.com/fatih/color"
@@ -45,6 +49,7 @@ import (
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
 	"github.com/minio/kes"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -64,7 +69,11 @@ import (
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
-var defaultAWSCredProvider []credentials.Provider
+
+var (
+	shardDiskTimeDelta     time.Duration
+	defaultAWSCredProvider []credentials.Provider
+)
 
 func init() {
 	if runtime.GOOS == "windows" {
@@ -124,6 +133,8 @@ func init() {
 	console.SetColor("Debug", fcolor.New())
 
 	gob.Register(StorageErr(""))
+	gob.Register(madmin.TimeInfo{})
+	gob.Register(map[string]interface{}{})
 
 	defaultAWSCredProvider = []credentials.Provider{
 		&credentials.IAM{
@@ -133,6 +144,12 @@ func init() {
 		},
 	}
 
+	var err error
+	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
+	if err != nil {
+		shardDiskTimeDelta = 1 * time.Minute
+	}
+
 	// All minio-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
@@ -187,11 +204,14 @@ func minioConfigToConsoleFeatures() {
 			os.Setenv("CONSOLE_IDP_CALLBACK", getConsoleEndpoints()[0]+"/oauth_callback")
 		}
 	}
-	os.Setenv("CONSOLE_MINIO_REGION", globalServerRegion)
+	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
 	if globalSubnetConfig.License != "" {
 		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
 	}
+	if globalSubnetConfig.APIKey != "" {
+		os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
+	}
 }
 
 func initConsoleServer() (*restapi.Server, error) {
@@ -303,7 +323,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
+	logStartupMessage(prepareUpdateMessage("\nRun `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -348,7 +368,6 @@ func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() s
 }
 
 func handleCommonCmdArgs(ctx *cli.Context) {
-
 	// Get "json" flag from command line argument and
 	// enable json and quite modes if json flag is turned on.
 	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
@@ -429,7 +448,166 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
 }
 
+type envKV struct {
+	Key   string
+	Value string
+	Skip  bool
+}
+
+func (e envKV) String() string {
+	if e.Skip {
+		return ""
+	}
+	return fmt.Sprintf("%s=%s", e.Key, e.Value)
+}
+
+func parsEnvEntry(envEntry string) (envKV, error) {
+	envEntry = strings.TrimSpace(envEntry)
+	if envEntry == "" {
+		// Skip all empty lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
+	const envSeparator = "="
+	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
+	if len(envTokens) != 2 {
+		return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
+	}
+
+	key := envTokens[0]
+	val := envTokens[1]
+
+	// Remove quotes from the value if found
+	if len(val) >= 2 {
+		quote := val[0]
+		if (quote == '"' || quote == '\'') && val[len(val)-1] == quote {
+			val = val[1 : len(val)-1]
+		}
+	}
+
+	return envKV{
+		Key:   key,
+		Value: val,
+	}, nil
+}
+
+// Similar to os.Environ returns a copy of strings representing
+// the environment values from a file, in the form "key, value".
+// in a structured form.
+func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
+	f, err := os.Open(envConfigFile)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	var ekvs []envKV
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		ekv, err := parsEnvEntry(scanner.Text())
+		if err != nil {
+			return nil, err
+		}
+		if ekv.Skip {
+			// Skips empty lines
+			continue
+		}
+		ekvs = append(ekvs, ekv)
+	}
+	if err = scanner.Err(); err != nil {
+		return nil, err
+	}
+	return ekvs, nil
+}
+
+func readFromSecret(sp string) (string, error) {
+	// Supports reading path from docker secrets, filename is
+	// relative to /run/secrets/ position.
+	if isFile(pathJoin("/run/secrets/", sp)) {
+		sp = pathJoin("/run/secrets/", sp)
+	}
+	credBuf, err := ioutil.ReadFile(sp)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return "", nil
+		}
+		return "", err
+	}
+	return string(bytes.TrimSpace(credBuf)), nil
+}
+
+func loadEnvVarsFromFiles() {
+	if env.IsSet(config.EnvAccessKeyFile) {
+		accessKey, err := readFromSecret(env.Get(config.EnvAccessKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if accessKey != "" {
+			os.Setenv(config.EnvRootUser, accessKey)
+		}
+	}
+
+	if env.IsSet(config.EnvSecretKeyFile) {
+		secretKey, err := readFromSecret(env.Get(config.EnvSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if secretKey != "" {
+			os.Setenv(config.EnvRootPassword, secretKey)
+		}
+	}
+
+	if env.IsSet(config.EnvRootUserFile) {
+		rootUser, err := readFromSecret(env.Get(config.EnvRootUserFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootUser != "" {
+			os.Setenv(config.EnvRootUser, rootUser)
+		}
+	}
+
+	if env.IsSet(config.EnvRootPasswordFile) {
+		rootPassword, err := readFromSecret(env.Get(config.EnvRootPasswordFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootPassword != "" {
+			os.Setenv(config.EnvRootPassword, rootPassword)
+		}
+	}
+
+	if env.IsSet(config.EnvKMSSecretKeyFile) {
+		kmsSecret, err := readFromSecret(env.Get(config.EnvKMSSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
+		}
+		if kmsSecret != "" {
+			os.Setenv(config.EnvKMSSecretKey, kmsSecret)
+		}
+	}
+
+	if env.IsSet(config.EnvConfigEnvFile) {
+		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the config environment file")
+		}
+		for _, ekv := range ekvs {
+			os.Setenv(ekv.Key, ekv.Value)
+		}
+	}
+}
+
 func handleCommonEnvVars() {
+	loadEnvVarsFromFiles()
+
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
@@ -496,7 +674,7 @@ func handleCommonEnvVars() {
 	publicIPs := env.Get(config.EnvPublicIPs, "")
 	if len(publicIPs) != 0 {
 		minioEndpoints := strings.Split(publicIPs, config.ValueSeparator)
-		var domainIPs = set.NewStringSet()
+		domainIPs := set.NewStringSet()
 		for _, endpoint := range minioEndpoints {
 			if net.ParseIP(endpoint) == nil {
 				// Checking if the IP is a DNS entry.
@@ -526,8 +704,8 @@ func handleCommonEnvVars() {
 	// in-place update is off.
 	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, config.EnableOn), config.EnableOff)
 
-	// Check if the supported credential env vars, "MINIO_ROOT_USER" and
-	// "MINIO_ROOT_PASSWORD" are provided
+	// Check if the supported credential env vars,
+	// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
 	// Warn user if deprecated environment variables,
 	// "MINIO_ACCESS_KEY" and "MINIO_SECRET_KEY", are defined
 	// Check all error conditions first
@@ -548,25 +726,24 @@ func handleCommonEnvVars() {
 	// are defined or both are not defined.
 	// Check both cases and authenticate them if correctly defined
 	var user, password string
-	haveRootCredentials := false
-	haveAccessCredentials := false
+	var hasCredentials bool
 	//nolint:gocritic
 	if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
 		user = env.Get(config.EnvRootUser, "")
 		password = env.Get(config.EnvRootPassword, "")
-		haveRootCredentials = true
+		hasCredentials = true
 	} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
-		haveAccessCredentials = true
+		hasCredentials = true
 	}
-	if haveRootCredentials || haveAccessCredentials {
+	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
 		if err != nil {
 			logger.Fatal(config.ErrInvalidCredentials(err),
 				"Unable to validate credentials inherited from the shell environment")
 		}
-		if haveAccessCredentials {
+		if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 			msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
 				"         Please use %s and %s",
 				config.EnvAccessKey, config.EnvSecretKey,
@@ -614,7 +791,7 @@ func handleCommonEnvVars() {
 			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
-		var defaultKeyID = env.Get(config.EnvKESKeyName, "")
+		defaultKeyID := env.Get(config.EnvKESKeyName, "")
 		KMS, err := kms.NewWithConfig(kms.Config{
 			Endpoints:    endpoints,
 			DefaultKeyID: defaultKeyID,
@@ -718,6 +895,10 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		}
 	}
 	secureConn = true
+
+	// syscall.SIGHUP to reload the certs.
+	manager.ReloadOnSignal(syscall.SIGHUP)
+
 	return x509Certs, manager, secureConn, nil
 }
 
@@ -730,3 +911,32 @@ func contextCanceled(ctx context.Context) bool {
 		return false
 	}
 }
+
+// bgContext returns a context that can be used for async operations.
+// Cancellation/timeouts are removed, so parent cancellations/timeout will
+// not propagate from parent.
+// Context values are preserved.
+// This can be used for goroutines that live beyond the parent context.
+func bgContext(parent context.Context) context.Context {
+	return bgCtx{parent: parent}
+}
+
+type bgCtx struct {
+	parent context.Context
+}
+
+func (a bgCtx) Done() <-chan struct{} {
+	return nil
+}
+
+func (a bgCtx) Err() error {
+	return nil
+}
+
+func (a bgCtx) Deadline() (deadline time.Time, ok bool) {
+	return time.Time{}, false
+}
+
+func (a bgCtx) Value(key interface{}) interface{} {
+	return a.parent.Value(key)
+}

cmd/common-main_test.go

@@ -0,0 +1,161 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+	"reflect"
+	"testing"
+)
+
+func Test_readFromSecret(t *testing.T) {
+	testCases := []struct {
+		content       string
+		expectedErr   bool
+		expectedValue string
+	}{
+		{
+			"value\n",
+			false,
+			"value",
+		},
+		{
+			" \t\n Hello, Gophers \n\t\r\n",
+			false,
+			"Hello, Gophers",
+		},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			value, err := readFromSecret(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if value != testCase.expectedValue {
+				t.Errorf("Expected %s, got %s", testCase.expectedValue, value)
+			}
+		})
+	}
+}
+
+func Test_minioEnvironFromFile(t *testing.T) {
+	testCases := []struct {
+		content      string
+		expectedErr  bool
+		expectedEkvs []envKV
+	}{
+		{
+			`
+export MINIO_ROOT_USER=minio
+export MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		// Value with double quotes
+		{
+			`export MINIO_ROOT_USER="minio"`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		// Value with single quotes
+		{
+			`export MINIO_ROOT_USER='minio'`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		{
+			`
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		{
+			`
+export MINIO_ROOT_USERminio
+export MINIO_ROOT_PASSWORD=minio123`,
+			true,
+			nil,
+		},
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			ekvs, err := minioEnvironFromFile(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if !reflect.DeepEqual(ekvs, testCase.expectedEkvs) {
+				t.Errorf("expected %v, got %v", testCase.expectedEkvs, ekvs)
+			}
+		})
+	}
+}

cmd/config-common.go

@@ -29,27 +29,31 @@ import (
 
 var errConfigNotFound = errors.New("config file not found")
 
-func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
-	// Read entire content by setting size to -1
-	r, err := objAPI.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
+func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string) ([]byte, ObjectInfo, error) {
+	r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {
-			return nil, errConfigNotFound
+			return nil, ObjectInfo{}, errConfigNotFound
 		}
 
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	defer r.Close()
 
 	buf, err := ioutil.ReadAll(r)
 	if err != nil {
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	if len(buf) == 0 {
-		return nil, errConfigNotFound
+		return nil, ObjectInfo{}, errConfigNotFound
 	}
-	return buf, nil
+	return buf, r.ObjInfo, nil
+}
+
+func readConfig(ctx context.Context, store objectIO, configFile string) ([]byte, error) {
+	buf, _, err := readConfigWithMetadata(ctx, store, configFile)
+	return buf, err
 }
 
 type objectDeleter interface {
@@ -66,13 +70,13 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
 	return err
 }
 
-func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
+func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
 	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
 	if err != nil {
 		return err
 	}
 
-	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{MaxParity: true})
+	_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{MaxParity: true})
 	return err
 }
 

cmd/config-current.go

@@ -50,7 +50,7 @@ import (
 )
 
 func initHelp() {
-	var kvs = map[string]config.KVS{
+	kvs := map[string]config.KVS{
 		config.EtcdSubSys:           etcd.DefaultKVS,
 		config.CacheSubSys:          cache.DefaultKVS,
 		config.CompressionSubSys:    compress.DefaultKVS,
@@ -58,10 +58,11 @@ func initHelp() {
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
 		config.IdentityTLSSubSys:    xtls.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
+		config.SiteSubSys:           config.DefaultSiteKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
 		config.CredentialsSubSys:    config.DefaultCredentialKVS,
-		config.LoggerWebhookSubSys:  logger.DefaultKVS,
+		config.LoggerWebhookSubSys:  logger.DefaultLoggerWebhookKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditWebhookKVS,
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
 		config.HealSubSys:           heal.DefaultKVS,
@@ -77,10 +78,10 @@ func initHelp() {
 	config.RegisterDefaultKVS(kvs)
 
 	// Captures help for each sub-system
-	var helpSubSys = config.HelpKVS{
+	helpSubSys := config.HelpKVS{
 		config.HelpKV{
-			Key:         config.RegionSubSys,
-			Description: "label the location of the server",
+			Key:         config.SiteSubSys,
+			Description: "label the server and its location",
 		},
 		config.HelpKV{
 			Key:         config.CacheSubSys,
@@ -190,7 +191,7 @@ func initHelp() {
 		config.HelpKV{
 			Key:         config.SubnetSubSys,
 			Type:        "string",
-			Description: "set subnet config for the cluster e.g. license token",
+			Description: "set subnet config for the cluster e.g. api key",
 			Optional:    true,
 		},
 	}
@@ -204,8 +205,9 @@ func initHelp() {
 		}
 	}
 
-	var helpMap = map[string]config.HelpKVS{
+	helpMap := map[string]config.HelpKVS{
 		"":                          helpSubSys, // Help for all sub-systems.
+		config.SiteSubSys:           config.SiteHelp,
 		config.RegionSubSys:         config.RegionHelp,
 		config.APISubSys:            api.Help,
 		config.StorageClassSubSys:   storageclass.Help,
@@ -231,10 +233,20 @@ func initHelp() {
 		config.NotifyRedisSubSys:    notify.HelpRedis,
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
-		config.SubnetSubSys:         subnet.HelpLicense,
+		config.SubnetSubSys:         subnet.HelpSubnet,
 	}
 
 	config.RegisterHelpSubSys(helpMap)
+
+	// save top-level help for deprecated sub-systems in a separate map.
+	deprecatedHelpKVMap := map[string]config.HelpKV{
+		config.RegionSubSys: {
+			Key:         config.RegionSubSys,
+			Description: "[DEPRECATED - use `site` instead] label the location of the server",
+		},
+	}
+
+	config.RegisterHelpDeprecatedSubSys(deprecatedHelpKVMap)
 }
 
 var (
@@ -259,7 +271,7 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
-	if _, err := config.LookupRegion(s[config.RegionSubSys][config.Default]); err != nil {
+	if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -315,7 +327,7 @@ func validateConfig(s config.Config) error {
 		}
 	}
 	if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 		return err
 	}
 
@@ -437,9 +449,9 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	// but not federation.
 	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
 
-	globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
+	globalSite, err = config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
 	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
@@ -501,7 +513,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody)
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
 	}
@@ -531,7 +543,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	loggerCfg, err := logger.LookupConfig(s)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger/audit targets: %w", err))
 	}
 
 	for _, l := range loggerCfg.HTTP {
@@ -541,7 +553,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http logging
 			if err = logger.AddTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server logger HTTP target: %w", err))
 			}
 		}
 	}
@@ -553,7 +565,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http audit logging
 			if err = logger.AddAuditTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit HTTP target: %w", err))
 			}
 		}
 	}
@@ -563,7 +575,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.LogOnce = logger.LogOnceIf
 			// Enable Kafka audit logging
 			if err = logger.AddAuditTarget(kafka.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit Kafka target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit Kafka target: %w", err))
 			}
 		}
 	}
@@ -674,7 +686,10 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 
 	subSysHelp, ok := config.HelpSubSysMap[""].Lookup(subSys)
 	if !ok {
-		return Help{}, config.Errorf("unknown sub-system %s", subSys)
+		subSysHelp, ok = config.HelpDeprecatedSubSysMap[subSys]
+		if !ok {
+			return Help{}, config.Errorf("unknown sub-system %s", subSys)
+		}
 	}
 
 	h, ok := config.HelpSubSysMap[subSys]

cmd/config-current_test.go

@@ -36,18 +36,21 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 
-	if globalServerRegion != globalMinioDefaultRegion {
-		t.Errorf("Expecting region `us-east-1` found %s", globalServerRegion)
+	if globalSite.Region != globalMinioDefaultRegion {
+		t.Errorf("Expecting region `us-east-1` found %s", globalSite.Region)
 	}
 
 	// Set new region and verify.
 	config.SetRegion(globalServerConfig, "us-west-1")
-	region, err := config.LookupRegion(globalServerConfig[config.RegionSubSys][config.Default])
+	site, err := config.LookupSite(
+		globalServerConfig[config.SiteSubSys][config.Default],
+		globalServerConfig[config.RegionSubSys][config.Default],
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if region != "us-west-1" {
-		t.Errorf("Expecting region `us-west-1` found %s", globalServerRegion)
+	if site.Region != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", globalSite.Region)
 	}
 
 	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {

cmd/config-dir.go

@@ -84,7 +84,7 @@ func (dir *ConfigDir) Get() string {
 
 // Attempts to create all directories, ignores any permission denied errors.
 func mkdirAllIgnorePerm(path string) error {
-	err := os.MkdirAll(path, 0700)
+	err := os.MkdirAll(path, 0o700)
 	if err != nil {
 		// It is possible in kubernetes like deployments this directory
 		// is already mounted and is not writable, ignore any write errors.

cmd/config-migrate.go

@@ -2445,12 +2445,12 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
-	var configFiles = []string{
+	configFiles := []string{
 		getConfigFile(),
 		getConfigFile() + ".deprecated",
 		configFile,
 	}
-	var config = &serverConfigV27{}
+	config := &serverConfigV27{}
 	for _, cfgFile := range configFiles {
 		if _, err = Load(cfgFile, config); err != nil {
 			if !osIsNotExist(err) && !osIsPermission(err) {

cmd/config-migrate_test.go

@@ -51,7 +51,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	// Create a V1 config json file and store it
 	configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
 	configPath := rootPath + "/fsUsers.json"
-	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -181,7 +181,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	// Create a corrupted config file
-	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Fire a migrateConfig()
@@ -194,7 +194,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 
 	// Create a V2 config json file and store it
 	configJSON := "{ \"version\":\"2\", \"credentials\": {\"accessKeyId\":\"" + accessKey + "\", \"secretAccessKey\":\"" + secretKey + "\", \"region\":\"us-east-1\"}, \"mongoLogger\":{\"addr\":\"127.0.0.1:3543\", \"db\":\"foodb\", \"collection\":\"foo\"}, \"syslogLogger\":{\"network\":\"127.0.0.1:543\", \"addr\":\"addr\"}, \"fileLogger\":{\"filename\":\"log.out\"}}"
-	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -244,7 +244,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	// Create a corrupted config file
-	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -343,7 +343,7 @@ func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
 	for i := 3; i <= 17; i++ {
 		// Create a corrupted config file
 		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
-			0644); err != nil {
+			0o644); err != nil {
 			t.Fatal("Unexpected error: ", err)
 		}
 
@@ -354,7 +354,7 @@ func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
 	}
 
 	// Create a corrupted config file for version '2'.
-	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0644); err != nil {
+	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 

cmd/config.go

@@ -46,8 +46,8 @@ const (
 )
 
 func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) (
-	[]madmin.ConfigHistoryEntry, error) {
-
+	[]madmin.ConfigHistoryEntry, error,
+) {
 	var configHistory []madmin.ConfigHistoryEntry
 
 	// List all kvs
@@ -140,7 +140,7 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, cfg interface{})
 		return err
 	}
 
-	var configFile = path.Join(minioConfigPrefix, minioConfigFile)
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	if GlobalKMS != nil {
 		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, configFile),
@@ -153,7 +153,7 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, cfg interface{})
 }
 
 func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, error) {
-	var srvCfg = config.New()
+	srvCfg := config.New()
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	data, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
@@ -174,7 +174,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 		}
 	}
 
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(data, &srvCfg); err != nil {
 		return nil, err
 	}
@@ -186,11 +186,6 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 // ConfigSys - config system.
 type ConfigSys struct{}
 
-// Load - load config.json.
-func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
-	return sys.Init(objAPI)
-}
-
 // Init - initializes config system from config.json.
 func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {

cmd/data-scanner.go

@@ -25,7 +25,6 @@ import (
 	"io/fs"
 	"math"
 	"math/rand"
-	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -40,7 +39,6 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/heal"
 	"github.com/minio/minio/internal/event"
-	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
 )
@@ -71,7 +69,19 @@ var (
 
 // initDataScanner will start the scanner in the background.
 func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
-	go runDataScanner(ctx, objAPI)
+	go func() {
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		// Run the data scanner in a loop
+		for {
+			runDataScanner(ctx, objAPI)
+			duration := time.Duration(r.Float64() * float64(scannerCycle.Get()))
+			if duration < time.Second {
+				// Make sure to sleep atleast a second to avoid high CPU ticks.
+				duration = time.Second
+			}
+			time.Sleep(duration)
+		}
+	}()
 }
 
 type safeDuration struct {
@@ -96,36 +106,26 @@ func (s *safeDuration) Get() time.Duration {
 // There should only ever be one scanner running per cluster.
 func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 	// Make sure only 1 scanner is running on the cluster.
-	locker := objAPI.NewNSLock(minioMetaBucket, "runDataScanner.lock")
-	var ctx context.Context
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	for {
-		lkctx, err := locker.GetLock(pctx, dataScannerLeaderLockTimeout)
-		if err != nil {
-			time.Sleep(time.Duration(r.Float64() * float64(scannerCycle.Get())))
-			continue
+	locker := objAPI.NewNSLock(minioMetaBucket, "scanner/runDataScanner.lock")
+	lkctx, err := locker.GetLock(pctx, dataScannerLeaderLockTimeout)
+	if err != nil {
+		if intDataUpdateTracker.debug {
+			logger.LogIf(pctx, err)
 		}
-		ctx = lkctx.Context()
-		defer lkctx.Cancel()
-		break
-		// No unlock for "leader" lock.
+		return
 	}
+	ctx := lkctx.Context()
+	defer lkctx.Cancel()
+	// No unlock for "leader" lock.
 
 	// Load current bloom cycle
 	nextBloomCycle := intDataUpdateTracker.current() + 1
 
-	br, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageBloomName, nil, http.Header{}, readLock, ObjectOptions{})
-	if err != nil {
-		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
+	buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
+	if len(buf) >= 8 {
+		if err = binary.Read(bytes.NewReader(buf), binary.LittleEndian, &nextBloomCycle); err != nil {
 			logger.LogIf(ctx, err)
 		}
-	} else {
-		if br.ObjInfo.Size == 8 {
-			if err = binary.Read(br, binary.LittleEndian, &nextBloomCycle); err != nil {
-				logger.LogIf(ctx, err)
-			}
-		}
-		br.Close()
 	}
 
 	scannerTimer := time.NewTimer(scannerCycle.Get())
@@ -155,14 +155,7 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 				nextBloomCycle++
 				var tmp [8]byte
 				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
-				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)))
-				if err != nil {
-					logger.LogIf(ctx, err)
-					continue
-				}
-
-				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r), ObjectOptions{})
-				if !isErrBucketNotFound(err) {
+				if err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp[:]); err != nil {
 					logger.LogIf(ctx, err)
 				}
 			}
@@ -267,7 +260,6 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		defer func() {
 			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
 		}()
-
 	}
 
 	switch cache.Info.Name {
@@ -656,6 +648,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			dirQuorum: getReadQuorum(len(f.disks)),
 			objQuorum: getReadQuorum(len(f.disks)),
 			bucket:    "",
+			strict:    false,
 		}
 
 		healObjectsPrefix := color.Green("healObjects:")
@@ -864,8 +857,10 @@ func (i *scannerItem) transformMetaDir() {
 	i.objectName = split[len(split)-1]
 }
 
-var applyActionsLogPrefix = color.Green("applyActions:")
-var applyVersionActionsLogPrefix = color.Green("applyVersionActions:")
+var (
+	applyActionsLogPrefix        = color.Green("applyActions:")
+	applyVersionActionsLogPrefix = color.Green("applyVersionActions:")
+)
 
 func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi ObjectInfo) (size int64) {
 	if i.debug {
@@ -968,17 +963,16 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
 	}
-
 }
 
-// applyMaxNoncurrentVersionLimit removes noncurrent versions older than the most recent MaxNoncurrentVersions configured.
+// applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
 	if i.lifeCycle == nil {
 		return fivs, nil
 	}
 
-	lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
 	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
 		return fivs, nil
 	}
@@ -991,6 +985,7 @@ func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o Obje
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
 		obj := fi.ToObjectInfo(i.bucket, i.objectPath())
+		// skip versions with object locking enabled
 		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			if i.debug {
 				if obj.VersionID != "" {
@@ -999,22 +994,36 @@ func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o Obje
 					console.Debugf(applyVersionActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 				}
 			}
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
 			continue
 		}
+
+		// NoncurrentDays not passed yet.
+		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, days)) {
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
+			continue
+		}
+
 		toDel = append(toDel, ObjectToDelete{
-			ObjectName: fi.Name,
-			VersionID:  fi.VersionID,
+			ObjectV: ObjectV{
+				ObjectName: fi.Name,
+				VersionID:  fi.VersionID,
+			},
 		})
 	}
 
-	globalExpiryState.enqueueByMaxNoncurrent(i.bucket, toDel)
+	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
 	return fivs, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
 func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
-	return i.applyMaxNoncurrentVersionLimit(ctx, o, fivs)
+	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1076,7 +1085,6 @@ func applyTransitionRule(obj ObjectInfo) bool {
 	}
 	globalTransitionState.queueTransitionTask(obj)
 	return true
-
 }
 
 func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject bool) bool {
@@ -1245,6 +1253,7 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 				DeleteMarker:          roi.DeleteMarker,
 			},
 			Bucket: roi.Bucket,
+			OpType: replication.HealReplicationType,
 		}
 		if roi.ExistingObjResync.mustResync() {
 			doi.OpType = replication.ExistingObjectReplicationType

cmd/data-update-tracker.go

@@ -473,27 +473,32 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for in := range d.input {
-		bucket, _ := path2BucketObjectWithBasePath("", in)
-		if bucket == "" {
-			if d.debug && len(in) > 0 {
-				console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case in := <-d.input:
+			bucket, _ := path2BucketObjectWithBasePath("", in)
+			if bucket == "" {
+				if d.debug && len(in) > 0 {
+					console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
+				}
+				continue
 			}
-			continue
-		}
 
-		if isReservedOrInvalidBucket(bucket, false) {
-			continue
-		}
-		split := splitPathDeterministic(in)
+			if isReservedOrInvalidBucket(bucket, false) {
+				continue
+			}
+			split := splitPathDeterministic(in)
 
-		// Add all paths until done.
-		d.mu.Lock()
-		for i := range split {
-			d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+			// Add all paths until done.
+			d.mu.Lock()
+			for i := range split {
+				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+			}
+			d.dirty = d.dirty || len(split) > 0
+			d.mu.Unlock()
 		}
-		d.dirty = d.dirty || len(split) > 0
-		d.mu.Unlock()
 	}
 }
 

cmd/data-update-tracker_test.go

@@ -105,7 +105,7 @@ func TestDataUpdateTracker(t *testing.T) {
 	defer cancel()
 	dut.start(ctx, tmpDir)
 
-	var tests = []struct {
+	tests := []struct {
 		in    string
 		check []string // if not empty, check against these instead.
 		exist bool

cmd/data-usage-cache.go

@@ -624,7 +624,7 @@ func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compact
 	// Appears to be printed with _MINIO_SERVER_DEBUG=off
 	// console.Debugf(" %d children found, compacting %v\n", total, path)
 
-	var leaves = make([]struct {
+	leaves := make([]struct {
 		objects uint64
 		path    dataUsageHash
 	}, total)
@@ -774,7 +774,7 @@ func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
 // bucketsUsageInfo returns the buckets usage info as a map, with
 // key as bucket name
 func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]BucketUsageInfo {
-	var dst = make(map[string]BucketUsageInfo, len(buckets))
+	dst := make(map[string]BucketUsageInfo, len(buckets))
 	for _, bucket := range buckets {
 		e := d.find(bucket.Name)
 		if e == nil {
@@ -797,7 +797,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 					ReplicationPendingCount: stat.PendingCount,
 					ReplicationFailedCount:  stat.FailedCount,
 				}
-
 			}
 		}
 		dst[bucket.Name] = bui
@@ -922,6 +921,7 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 	// Abandon if more than 5 minutes, so we don't hold up scanner.
 	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 	defer cancel()
+
 	r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		switch err.(type) {

cmd/data-usage.go

@@ -18,13 +18,10 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
-	"net/http"
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -32,28 +29,24 @@ const (
 	dataUsageRoot   = SlashSeparator
 	dataUsageBucket = minioMetaBucket + SlashSeparator + bucketMetaPrefix
 
-	dataUsageObjName   = ".usage.json"
+	dataUsageObjName       = ".usage.json"
+	dataUsageObjNamePath   = bucketMetaPrefix + SlashSeparator + dataUsageObjName
+	dataUsageBloomName     = ".bloomcycle.bin"
+	dataUsageBloomNamePath = bucketMetaPrefix + SlashSeparator + dataUsageBloomName
+
 	dataUsageCacheName = ".usage-cache.bin"
-	dataUsageBloomName = ".bloomcycle.bin"
 )
 
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
 func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
 	for dataUsageInfo := range dui {
-		var json = jsoniter.ConfigCompatibleWithStandardLibrary
+		json := jsoniter.ConfigCompatibleWithStandardLibrary
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
 		}
-		size := int64(len(dataUsageJSON))
-		r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r), ObjectOptions{})
-		if !isErrBucketNotFound(err) {
+		if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
 			logger.LogIf(ctx, err)
 		}
 	}
@@ -95,18 +88,17 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
-	r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{})
+	buf, err := readConfig(ctx, objAPI, dataUsageObjNamePath)
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
 			return DataUsageInfo{}, nil
 		}
-		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
+		return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)
 	}
-	defer r.Close()
 
 	var dataUsageInfo DataUsageInfo
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
-	if err = json.NewDecoder(r).Decode(&dataUsageInfo); err != nil {
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = json.Unmarshal(buf, &dataUsageInfo); err != nil {
 		return DataUsageInfo{}, err
 	}
 	// For forward compatibility reasons, we need to add this code.

cmd/data-usage_test.go

@@ -41,7 +41,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	const bucket = "bucket"
 	defer os.RemoveAll(base)
-	var files = []usageTestFile{
+	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},
 		{name: "dir1/d1file", size: 2000},
@@ -73,7 +73,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 
 	// Test dirs
-	var want = []struct {
+	want := []struct {
 		path       string
 		isNil      bool
 		size, objs int
@@ -257,7 +257,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	scannerSleeper.Update(0, 0)
 	defer os.RemoveAll(base)
-	var files = []usageTestFile{
+	files := []usageTestFile{
 		{name: "bucket/rootfile", size: 10000},
 		{name: "bucket/rootfile2", size: 10000},
 		{name: "bucket/dir1/d1file", size: 2000},
@@ -302,7 +302,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 
 	// Test dirs
-	var want = []struct {
+	want := []struct {
 		path       string
 		isNil      bool
 		size, objs int
@@ -543,7 +543,7 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 	}
 	const bucket = "abucket"
 	defer os.RemoveAll(base)
-	var files = []usageTestFile{
+	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},
 		{name: "dir1/d1file", size: 2000},

cmd/disk-cache-backend.go

@@ -59,10 +59,13 @@ const (
 	cacheExpiryDays  = 90 * time.Hour * 24 // defaults to 90 days
 	// SSECacheEncrypted is the metadata key indicating that the object
 	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
-	SSECacheEncrypted               = "X-Minio-Internal-Encrypted-Cache"
-	cacheMultipartDir               = "multipart"
+	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
+	cacheMultipartDir = "multipart"
+	cacheWritebackDir = "writeback"
+
 	cacheStaleUploadCleanupInterval = time.Hour * 24
 	cacheStaleUploadExpiry          = time.Hour * 24
+	cacheWBStaleUploadExpiry        = time.Hour * 24 * 7
 )
 
 // CacheChecksumInfoV1 - carries checksums of individual blocks on disk.
@@ -105,15 +108,15 @@ func (r *RangeInfo) Empty() bool {
 	return r.Range == "" && r.File == "" && r.Size == 0
 }
 
-func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
+func (m *cacheMeta) ToObjectInfo() (o ObjectInfo) {
 	if len(m.Meta) == 0 {
 		m.Meta = make(map[string]string)
 		m.Stat.ModTime = timeSentinel
 	}
 
 	o = ObjectInfo{
-		Bucket:            bucket,
-		Name:              object,
+		Bucket:            m.Bucket,
+		Name:              m.Object,
 		CacheStatus:       CacheHit,
 		CacheLookupStatus: CacheHit,
 	}
@@ -187,7 +190,7 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		quotaPct = config.Quota
 	}
 
-	if err := os.MkdirAll(dir, 0777); err != nil {
+	if err := os.MkdirAll(dir, 0o777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
@@ -325,7 +328,11 @@ func (c *diskCache) purge(ctx context.Context) {
 	expiry := UTCNow().Add(-cacheExpiryDays)
 	// defaulting max hits count to 100
 	// ignore error we know what value we are passing.
-	scorer, _ := newFileScorer(toFree, time.Now().Unix(), 100)
+	scorer, err := newFileScorer(toFree, time.Now().Unix(), 100)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
 
 	// this function returns FileInfo for cached range files.
 	fiStatRangesFn := func(ranges map[string]string, pathPrefix string) map[string]os.FileInfo {
@@ -377,7 +384,7 @@ func (c *diskCache) purge(ctx context.Context) {
 		lastAtime := lastAtimeFn(meta.PartNumbers, pathJoin(c.dir, name))
 		// stat all cached file ranges.
 		cachedRngFiles := fiStatRangesFn(meta.Ranges, pathJoin(c.dir, name))
-		objInfo := meta.ToObjectInfo("", "")
+		objInfo := meta.ToObjectInfo()
 		// prevent gc from clearing un-synced commits. This metadata is present when
 		// cache writeback commit setting is enabled.
 		status, ok := objInfo.UserDefined[writeBackStatusHeader]
@@ -388,9 +395,7 @@ func (c *diskCache) purge(ctx context.Context) {
 		switch {
 		case cc != nil:
 			if cc.isStale(objInfo.ModTime) {
-				if err = removeAll(cacheDir); err != nil {
-					logger.LogIf(ctx, err)
-				}
+				removeAll(cacheDir)
 				scorer.adjustSaveBytes(-objInfo.Size)
 				// break early if sufficient disk space reclaimed.
 				if c.diskUsageLow() {
@@ -406,11 +411,12 @@ func (c *diskCache) purge(ctx context.Context) {
 		}
 
 		for fname, fi := range cachedRngFiles {
+			if fi == nil {
+				continue
+			}
 			if cc != nil {
 				if cc.isStale(objInfo.ModTime) {
-					if err = removeAll(fname); err != nil {
-						logger.LogIf(ctx, err)
-					}
+					removeAll(fname)
 					scorer.adjustSaveBytes(-fi.Size())
 
 					// break early if sufficient disk space reclaimed.
@@ -425,9 +431,11 @@ func (c *diskCache) purge(ctx context.Context) {
 		}
 		// clean up stale cache.json files for objects that never got cached but access count was maintained in cache.json
 		fi, err := os.Stat(pathJoin(cacheDir, cacheMetaJSONFile))
-		if err != nil || (fi.ModTime().Before(expiry) && len(cachedRngFiles) == 0) {
+		if err != nil || (fi != nil && fi.ModTime().Before(expiry) && len(cachedRngFiles) == 0) {
 			removeAll(cacheDir)
-			scorer.adjustSaveBytes(-fi.Size())
+			if fi != nil {
+				scorer.adjustSaveBytes(-fi.Size())
+			}
 			// Proceed to next file.
 			return nil
 		}
@@ -486,7 +494,7 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 	if partial {
 		return oi, numHits, errFileNotFound
 	}
-	oi = meta.ToObjectInfo("", "")
+	oi = meta.ToObjectInfo()
 	oi.Bucket = bucket
 	oi.Name = object
 
@@ -521,7 +529,7 @@ func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HT
 		return
 	}
 
-	oi = meta.ToObjectInfo("", "")
+	oi = meta.ToObjectInfo()
 	oi.Bucket = bucket
 	oi.Name = object
 	if !partial {
@@ -573,12 +581,16 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 	if _, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile)); err == nil {
 		partial = false
 	}
+	if writebackInProgress(meta.Meta) {
+		partial = false
+	}
 	return meta, partial, meta.Hits, nil
 }
 
 // saves object metadata to disk cache
 // incHitsOnly is true if metadata update is incrementing only the hit counter
-func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
+// finalizeWB is true only if metadata update accompanied by moving part from temp location to cache dir.
+func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly, finalizeWB bool) error {
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
 	cLock := c.NewNSLockFn(cachedPath)
 	lkctx, err := cLock.GetLock(ctx, globalOperationTimeout)
@@ -587,7 +599,18 @@ func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, met
 	}
 	ctx = lkctx.Context()
 	defer cLock.Unlock(lkctx.Cancel)
-	return c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly)
+	if err = c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly); err != nil {
+		return err
+	}
+	// move part saved in writeback directory and cache.json atomically
+	if finalizeWB {
+		wbdir := getCacheWriteBackSHADir(c.dir, bucket, object)
+		if err = renameAll(pathJoin(wbdir, cacheDataFile), pathJoin(cachedPath, cacheDataFile)); err != nil {
+			return err
+		}
+		removeAll(wbdir) // cleanup writeback/shadir
+	}
+	return nil
 }
 
 // saves object metadata to disk cache
@@ -596,10 +619,10 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
 	metaPath := pathJoin(cachedPath, cacheMetaJSONFile)
 	// Create cache directory if needed
-	if err := os.MkdirAll(cachedPath, 0777); err != nil {
+	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
 		return err
 	}
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0666)
+	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
 	if err != nil {
 		return err
 	}
@@ -659,10 +682,10 @@ func (c *diskCache) updateMetadata(ctx context.Context, bucket, object, etag str
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
 	metaPath := pathJoin(cachedPath, cacheMetaJSONFile)
 	// Create cache directory if needed
-	if err := os.MkdirAll(cachedPath, 0777); err != nil {
+	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
 		return err
 	}
-	f, err := os.OpenFile(metaPath, os.O_RDWR, 0666)
+	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)
 	if err != nil {
 		return err
 	}
@@ -702,9 +725,14 @@ func getCacheSHADir(dir, bucket, object string) string {
 	return pathJoin(dir, getSHA256Hash([]byte(pathJoin(bucket, object))))
 }
 
+// returns temporary writeback cache location.
+func getCacheWriteBackSHADir(dir, bucket, object string) string {
+	return pathJoin(dir, minioMetaBucket, "writeback", getSHA256Hash([]byte(pathJoin(bucket, object))))
+}
+
 // Cache data to disk with bitrot checksum added for each block of 1MB
 func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, string, error) {
-	if err := os.MkdirAll(cachePath, 0777); err != nil {
+	if err := os.MkdirAll(cachePath, 0o777); err != nil {
 		return 0, "", err
 	}
 	filePath := pathJoin(cachePath, fileName)
@@ -779,6 +807,7 @@ func newCacheEncryptReader(content io.Reader, bucket, object string, metadata ma
 	}
 	return reader, nil
 }
+
 func newCacheEncryptMetadata(bucket, object string, metadata map[string]string) ([]byte, error) {
 	var sealedKey crypto.SealedKey
 	if globalCacheKMS == nil {
@@ -799,6 +828,7 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 	metadata[SSECacheEncrypted] = ""
 	return objectKey[:], nil
 }
+
 func (c *diskCache) GetLockContext(ctx context.Context, bucket, object string) (RWLocker, LockContext, error) {
 	cachePath := getCacheSHADir(c.dir, bucket, object)
 	cLock := c.NewNSLockFn(cachePath)
@@ -808,10 +838,6 @@ func (c *diskCache) GetLockContext(ctx context.Context, bucket, object string) (
 
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly, writeback bool) (oi ObjectInfo, err error) {
-	if !c.diskSpaceAvailable(size) {
-		io.Copy(ioutil.Discard, data)
-		return oi, errDiskFull
-	}
 	cLock, lkctx, err := c.GetLockContext(ctx, bucket, object)
 	if err != nil {
 		return oi, err
@@ -850,12 +876,17 @@ func (c *diskCache) put(ctx context.Context, bucket, object string, data io.Read
 	if !c.diskSpaceAvailable(size) {
 		return oi, errDiskFull
 	}
-	if err := os.MkdirAll(cachePath, 0777); err != nil {
+
+	if writeback {
+		cachePath = getCacheWriteBackSHADir(c.dir, bucket, object)
+	}
+
+	if err := os.MkdirAll(cachePath, 0o777); err != nil {
 		return oi, err
 	}
-	var metadata = cloneMSS(opts.UserDefined)
-	var reader = data
-	var actualSize = uint64(size)
+	metadata := cloneMSS(opts.UserDefined)
+	reader := data
+	actualSize := uint64(size)
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
@@ -904,14 +935,14 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 		return errDiskFull
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
-	if err := os.MkdirAll(cachePath, 0777); err != nil {
+	if err := os.MkdirAll(cachePath, 0o777); err != nil {
 		return err
 	}
-	var metadata = cloneMSS(opts.UserDefined)
-	var reader = data
-	var actualSize = uint64(rlen)
+	metadata := cloneMSS(opts.UserDefined)
+	reader := data
+	actualSize := uint64(rlen)
 	// objSize is the actual size of object (with encryption overhead if any)
-	var objSize = uint64(size)
+	objSize := uint64(size)
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
@@ -1135,6 +1166,9 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		}()
 	} else {
 		go func() {
+			if writebackInProgress(objInfo.UserDefined) {
+				cacheObjPath = getCacheWriteBackSHADir(c.dir, bucket, object)
+			}
 			filePath := pathJoin(cacheObjPath, cacheFile)
 			err := c.bitrotReadFromCache(ctx, filePath, startOffset, length, pw)
 			if err != nil {
@@ -1203,7 +1237,7 @@ func (c *diskCache) scanCacheWritebackFailures(ctx context.Context) {
 			return nil
 		}
 
-		objInfo := meta.ToObjectInfo("", "")
+		objInfo := meta.ToObjectInfo()
 		status, ok := objInfo.UserDefined[writeBackStatusHeader]
 		if !ok || status == CommitComplete.String() {
 			return nil
@@ -1237,12 +1271,12 @@ func (c *diskCache) NewMultipartUpload(ctx context.Context, bucket, object, uID
 
 	cachePath := getMultipartCacheSHADir(c.dir, bucket, object)
 	uploadIDDir := path.Join(cachePath, uploadID)
-	if err := os.MkdirAll(uploadIDDir, 0777); err != nil {
+	if err := os.MkdirAll(uploadIDDir, 0o777); err != nil {
 		return uploadID, err
 	}
 	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
 
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0666)
+	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
 	if err != nil {
 		return uploadID, err
 	}
@@ -1299,7 +1333,7 @@ func (c *diskCache) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		return oi, errDiskFull
 	}
 	reader := data
-	var actualSize = uint64(size)
+	actualSize := uint64(size)
 	if globalCacheKMS != nil {
 		reader, err = newCachePartEncryptReader(ctx, bucket, object, partID, data, size, meta.Meta)
 		if err != nil {
@@ -1348,7 +1382,7 @@ func (c *diskCache) SavePartMetadata(ctx context.Context, bucket, object, upload
 	defer uploadLock.Unlock(ulkctx.Cancel)
 
 	metaPath := pathJoin(uploadDir, cacheMetaJSONFile)
-	f, err := os.OpenFile(metaPath, os.O_RDWR, 0666)
+	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)
 	if err != nil {
 		return err
 	}
@@ -1501,6 +1535,8 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	}
 	uploadMeta.Stat.Size = objectSize
 	uploadMeta.Stat.ModTime = roi.ModTime
+	uploadMeta.Bucket = bucket
+	uploadMeta.Object = object
 	// if encrypted - make sure ETag updated
 
 	uploadMeta.Meta["etag"] = roi.ETag
@@ -1524,7 +1560,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	uploadMeta.Hits++
 	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
 
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0666)
+	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
 	if err != nil {
 		return oi, err
 	}
@@ -1536,7 +1572,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	}
 	renameAll(pathJoin(uploadIDDir, cacheMetaJSONFile), pathJoin(cachePath, cacheMetaJSONFile))
 	removeAll(uploadIDDir) // clean up any unused parts in the uploadIDDir
-	return uploadMeta.ToObjectInfo(bucket, object), nil
+	return uploadMeta.ToObjectInfo(), nil
 }
 
 func (c *diskCache) AbortUpload(bucket, object, uploadID string) (err error) {
@@ -1583,9 +1619,6 @@ func getMultipartCacheSHADir(dir, bucket, object string) string {
 
 // clean up stale cache multipart uploads according to cleanup interval.
 func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
-	if !c.commitWritethrough {
-		return
-	}
 	timer := time.NewTimer(cacheStaleUploadCleanupInterval)
 	defer timer.Stop()
 	for {
@@ -1609,6 +1642,22 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 					return nil
 				})
 			})
+			// clean up of writeback folder where cache.json no longer exists in the main c.dir/<sha256(bucket,object> path
+			// and if past upload expiry window.
+			readDirFn(pathJoin(c.dir, minioMetaBucket, cacheWritebackDir), func(shaDir string, typ os.FileMode) error {
+				wbdir := pathJoin(c.dir, minioMetaBucket, cacheWritebackDir, shaDir)
+				cachedir := pathJoin(c.dir, shaDir)
+				if _, err := os.Stat(cachedir); os.IsNotExist(err) {
+					fi, err := os.Stat(wbdir)
+					if err != nil {
+						return nil
+					}
+					if now.Sub(fi.ModTime()) > cacheWBStaleUploadExpiry {
+						return removeAll(wbdir)
+					}
+				}
+				return nil
+			})
 		}
 	}
 }

cmd/disk-cache-utils.go

@@ -113,7 +113,6 @@ func cacheControlOpts(o ObjectInfo) *cacheControl {
 		if strings.EqualFold(k, "cache-control") {
 			headerVal = v
 		}
-
 	}
 	if headerVal == "" {
 		return nil
@@ -419,6 +418,9 @@ func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
 // Returns true if there still is a need to delete files (n+saveBytes >0),
 // false if no more bytes needs to be saved.
 func (f *fileScorer) adjustSaveBytes(n int64) bool {
+	if f == nil {
+		return false
+	}
 	if int64(f.saveBytes)+n <= 0 {
 		f.saveBytes = 0
 		f.trimQueue()
@@ -578,6 +580,18 @@ func (t *multiWriter) Write(p []byte) (n int, err error) {
 	}
 	return len(p), nil
 }
+
 func cacheMultiWriter(w1 io.Writer, w2 *io.PipeWriter) io.Writer {
 	return &multiWriter{backendWriter: w1, cacheWriter: w2}
 }
+
+// writebackInProgress returns true if writeback commit is not complete
+func writebackInProgress(m map[string]string) bool {
+	if v, ok := m[writeBackStatusHeader]; ok {
+		switch cacheCommitStatus(v) {
+		case CommitPending, CommitFailed:
+			return true
+		}
+	}
+	return false
+}

cmd/disk-cache-utils_test.go

@@ -38,7 +38,7 @@ func TestGetCacheControlOpts(t *testing.T) {
 		{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
 		{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
 		{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
+		{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 0o7, 28, 0o0, 0o0, time.UTC)}, false},
 		{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
 	}
 
@@ -61,7 +61,6 @@ func TestGetCacheControlOpts(t *testing.T) {
 }
 
 func TestIsMetadataSame(t *testing.T) {
-
 	testCases := []struct {
 		m1       map[string]string
 		m2       map[string]string
@@ -148,6 +147,7 @@ func TestNewFileScorer(t *testing.T) {
 		t.Fatal("unexpected file list", fs.queueString())
 	}
 }
+
 func TestBytesToClear(t *testing.T) {
 	testCases := []struct {
 		total         int64

cmd/disk-cache.go

@@ -132,12 +132,11 @@ type cacheObjects struct {
 
 func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string, rs *HTTPRangeSpec) error {
 	metadata := map[string]string{"etag": eTag}
-	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true)
+	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true, false)
 }
 
 // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
 func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *diskCache, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo, rs *HTTPRangeSpec) error {
-
 	bkMeta := make(map[string]string, len(bkObjectInfo.UserDefined))
 	cacheMeta := make(map[string]string, len(cacheObjInfo.UserDefined))
 	for k, v := range bkObjectInfo.UserDefined {
@@ -159,7 +158,7 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
 		bkObjectInfo.ETag != cacheObjInfo.ETag ||
 		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
 		!bkObjectInfo.Expires.Equal(cacheObjInfo.Expires) {
-		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false)
+		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false, false)
 	}
 	return c.incHitsToMeta(ctx, dcache, bucket, object, cacheObjInfo.Size, cacheObjInfo.ETag, rs)
 }
@@ -276,6 +275,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			bReader.ObjInfo.CacheStatus = CacheMiss
 			return bReader, err
 		}
+		// serve cached content without ETag verification if writeback commit is not yet complete
+		if writebackInProgress(cacheReader.ObjInfo.UserDefined) {
+			return cacheReader, nil
+		}
 	}
 
 	objInfo, err := c.InnerGetObjectInfoFn(ctx, bucket, object, opts)
@@ -422,6 +425,10 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 			c.cacheStats.incHit()
 			return cachedObjInfo, nil
 		}
+		// serve cache metadata without ETag verification if writeback commit is not yet complete
+		if writebackInProgress(cachedObjInfo.UserDefined) {
+			return cachedObjInfo, nil
+		}
 	}
 
 	objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
@@ -698,7 +705,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 				oi, _, err := dcache.Stat(GlobalContext, bucket, object)
 				// avoid cache overwrite if another background routine filled cache
 				if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, true)
+					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, false)
 				}
 			}()
 		}
@@ -786,6 +793,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
 	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader), opts)
 	wbCommitStatus := CommitComplete
+	size := objInfo.Size
 	if err != nil {
 		wbCommitStatus = CommitFailed
 	}
@@ -796,12 +804,13 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 		retryCnt, _ = strconv.Atoi(meta[writeBackRetryHeader])
 		retryCnt++
 		meta[writeBackRetryHeader] = strconv.Itoa(retryCnt)
+		size = cReader.ObjInfo.Size
 	} else {
 		delete(meta, writeBackRetryHeader)
 	}
 	meta[writeBackStatusHeader] = wbCommitStatus.String()
 	meta["etag"] = oi.ETag
-	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, objInfo.Size, nil, "", false)
+	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, size, nil, "", false, wbCommitStatus == CommitComplete)
 	if retryCnt > 0 {
 		// slow down retries
 		time.Sleep(time.Second * time.Duration(retryCnt%10+1))
@@ -811,6 +820,8 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 
 func (c *cacheObjects) queueWritebackRetry(oi ObjectInfo) {
 	select {
+	case <-GlobalContext.Done():
+		return
 	case c.wbRetryCh <- oi:
 		c.uploadObject(GlobalContext, oi)
 	default:

cmd/disk-cache_test.go

@@ -25,7 +25,7 @@ import (
 // Tests ToObjectInfo function.
 func TestCacheMetadataObjInfo(t *testing.T) {
 	m := cacheMeta{Meta: nil}
-	objInfo := m.ToObjectInfo("testbucket", "testobject")
+	objInfo := m.ToObjectInfo()
 	if objInfo.Size != 0 {
 		t.Fatal("Unexpected object info value for Size", objInfo.Size)
 	}

cmd/dynamic-timeouts_test.go

@@ -26,7 +26,6 @@ import (
 )
 
 func TestDynamicTimeoutSingleIncrease(t *testing.T) {
-
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
 	initial := timeout.Timeout()
@@ -43,7 +42,6 @@ func TestDynamicTimeoutSingleIncrease(t *testing.T) {
 }
 
 func TestDynamicTimeoutDualIncrease(t *testing.T) {
-
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
 	initial := timeout.Timeout()
@@ -66,7 +64,6 @@ func TestDynamicTimeoutDualIncrease(t *testing.T) {
 }
 
 func TestDynamicTimeoutSingleDecrease(t *testing.T) {
-
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
 	initial := timeout.Timeout()
@@ -83,7 +80,6 @@ func TestDynamicTimeoutSingleDecrease(t *testing.T) {
 }
 
 func TestDynamicTimeoutDualDecrease(t *testing.T) {
-
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
 	initial := timeout.Timeout()
@@ -106,7 +102,6 @@ func TestDynamicTimeoutDualDecrease(t *testing.T) {
 }
 
 func TestDynamicTimeoutManyDecreases(t *testing.T) {
-
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
 	initial := timeout.Timeout()
@@ -116,7 +111,6 @@ func TestDynamicTimeoutManyDecreases(t *testing.T) {
 		for i := 0; i < dynamicTimeoutLogSize; i++ {
 			timeout.LogSuccess(successTimeout)
 		}
-
 	}
 
 	adjusted := timeout.Timeout()
@@ -151,7 +145,6 @@ func TestDynamicTimeoutConcurrent(t *testing.T) {
 }
 
 func TestDynamicTimeoutHitMinimum(t *testing.T) {
-
 	const minimum = 30 * time.Second
 	timeout := newDynamicTimeout(time.Minute, minimum)
 
@@ -172,7 +165,6 @@ func TestDynamicTimeoutHitMinimum(t *testing.T) {
 }
 
 func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() float64) {
-
 	const successTimeout = 20 * time.Second
 
 	for i := 0; i < dynamicTimeoutLogSize; i++ {
@@ -192,7 +184,6 @@ func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() fl
 }
 
 func TestDynamicTimeoutAdjustExponential(t *testing.T) {
-
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
 	rand.Seed(0)
@@ -200,9 +191,7 @@ func TestDynamicTimeoutAdjustExponential(t *testing.T) {
 	initial := timeout.Timeout()
 
 	for try := 0; try < 10; try++ {
-
 		testDynamicTimeoutAdjust(t, timeout, rand.ExpFloat64)
-
 	}
 
 	adjusted := timeout.Timeout()
@@ -212,7 +201,6 @@ func TestDynamicTimeoutAdjustExponential(t *testing.T) {
 }
 
 func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
-
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
 	rand.Seed(0)
@@ -220,11 +208,9 @@ func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
 	initial := timeout.Timeout()
 
 	for try := 0; try < 10; try++ {
-
 		testDynamicTimeoutAdjust(t, timeout, func() float64 {
 			return 1.0 + rand.NormFloat64()
 		})
-
 	}
 
 	adjusted := timeout.Timeout()

cmd/encryption-v1.go

@@ -68,6 +68,21 @@ const (
 
 )
 
+// KMSKeyID returns in AWS compatible KMS KeyID() format.
+func (o ObjectInfo) KMSKeyID() string {
+	if len(o.UserDefined) == 0 {
+		return ""
+	}
+	kmsID, ok := o.UserDefined[crypto.MetaKeyID]
+	if !ok {
+		return ""
+	}
+	if strings.HasPrefix(kmsID, "arn:aws:kms:") {
+		return kmsID
+	}
+	return "arn:aws:kms:" + kmsID
+}
+
 // isMultipart returns true if the current object is
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
@@ -76,15 +91,18 @@ func (o *ObjectInfo) isMultipart() bool {
 		return false
 	}
 	_, encrypted := crypto.IsEncrypted(o.UserDefined)
-	if encrypted && !crypto.IsMultiPart(o.UserDefined) {
-		return false
-	}
-	for _, part := range o.Parts {
-		_, err := sio.DecryptedSize(uint64(part.Size))
-		if err != nil {
+	if encrypted {
+		if !crypto.IsMultiPart(o.UserDefined) {
 			return false
 		}
+		for _, part := range o.Parts {
+			_, err := sio.DecryptedSize(uint64(part.Size))
+			if err != nil {
+				return false
+			}
+		}
 	}
+
 	// Further check if this object is uploaded using multipart mechanism
 	// by the user and it is not about Erasure internally splitting the
 	// object into parts in PutObject()
@@ -168,7 +186,7 @@ func rotateKey(oldKey []byte, newKeyID string, newKey []byte, bucket, object str
 		// client provided it. Therefore, we create a copy
 		// of the client provided context and add the bucket
 		// key, if not present.
-		var kmsCtx = kms.Context{}
+		kmsCtx := kms.Context{}
 		for k, v := range ctx {
 			kmsCtx[k] = v
 		}
@@ -235,7 +253,7 @@ func newEncryptMetadata(kind crypto.Type, keyID string, key []byte, bucket, obje
 		// client provided it. Therefore, we create a copy
 		// of the client provided context and add the bucket
 		// key, if not present.
-		var kmsCtx = kms.Context{}
+		kmsCtx := kms.Context{}
 		for k, v := range ctx {
 			kmsCtx[k] = v
 		}
@@ -425,7 +443,6 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
 // DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
 // reader
 func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) {
-
 	bucket, object := oi.Bucket, oi.Name
 	// Single part case
 	if !oi.isMultipart() {

cmd/encryption-v1_test.go

@@ -64,7 +64,6 @@ func TestEncryptRequest(t *testing.T) {
 			req.Header.Set(k, v)
 		}
 		_, _, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
-
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
@@ -285,14 +284,13 @@ func TestGetDecryptedRange(t *testing.T) {
 	)
 
 	// Single part object tests
-	var (
-		mkSPObj = func(s int64) ObjectInfo {
-			return ObjectInfo{
-				Size:        getEncSize(s),
-				UserDefined: udMap(false),
-			}
+
+	mkSPObj := func(s int64) ObjectInfo {
+		return ObjectInfo{
+			Size:        getEncSize(s),
+			UserDefined: udMap(false),
 		}
-	)
+	}
 
 	testSP := []struct {
 		decSz int64
@@ -325,7 +323,7 @@ func TestGetDecryptedRange(t *testing.T) {
 			if err != nil {
 				t.Errorf("Case %d: unexpected err: %v", i, err)
 			}
-			var rLen = pkgSz + 32
+			rLen := pkgSz + 32
 			if test.decSz < pkgSz {
 				rLen = test.decSz + 32
 			}
@@ -341,7 +339,7 @@ func TestGetDecryptedRange(t *testing.T) {
 			if err != nil {
 				t.Errorf("Case %d: unexpected err: %v", i, err)
 			}
-			var rLen = (pkgSz + 32) * 2
+			rLen := (pkgSz + 32) * 2
 			if test.decSz < 2*pkgSz {
 				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32)
 			}
@@ -356,7 +354,7 @@ func TestGetDecryptedRange(t *testing.T) {
 			if err != nil {
 				t.Errorf("Case %d: unexpected err: %v", i, err)
 			}
-			var rLen = (pkgSz + 32) * 2
+			rLen := (pkgSz + 32) * 2
 			if test.decSz-pkgSz < 2*pkgSz {
 				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32*2)
 			}
@@ -551,60 +549,90 @@ var getDefaultOptsTests = []struct {
 	encryptionType encrypt.Type
 	err            error
 }{
-	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
-		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+	{
+		headers: http.Header{
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: encrypt.SSEC,
-		err:            nil}, // 0
-	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
-		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		err:            nil,
+	}, // 0
+	{
+		headers: http.Header{
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
 		copySource:     true,
 		metadata:       nil,
 		encryptionType: "",
-		err:            nil}, // 1
-	{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
-		xhttp.AmzServerSideEncryptionCustomerKey:    []string{"Mz"},
-		xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		err:            nil,
+	}, // 1
+	{
+		headers: http.Header{
+			xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
+			xhttp.AmzServerSideEncryptionCustomerKey:       []string{"Mz"},
+			xhttp.AmzServerSideEncryptionCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: "",
-		err:            crypto.ErrInvalidCustomerKey}, // 2
-	{headers: http.Header{xhttp.AmzServerSideEncryption: []string{"AES256"}},
+		err:            crypto.ErrInvalidCustomerKey,
+	}, // 2
+	{
+		headers:        http.Header{xhttp.AmzServerSideEncryption: []string{"AES256"}},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: encrypt.S3,
-		err:            nil}, // 3
-	{headers: http.Header{},
+		err:            nil,
+	}, // 3
+	{
+		headers:    http.Header{},
 		copySource: false,
-		metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+		metadata: map[string]string{
+			crypto.MetaSealedKeyS3:       base64.StdEncoding.EncodeToString(make([]byte, 64)),
 			crypto.MetaKeyID:             "kms-key",
-			crypto.MetaDataEncryptionKey: "m-key"},
+			crypto.MetaDataEncryptionKey: "m-key",
+		},
 		encryptionType: encrypt.S3,
-		err:            nil}, // 4
-	{headers: http.Header{},
+		err:            nil,
+	}, // 4
+	{
+		headers:    http.Header{},
 		copySource: true,
-		metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
+		metadata: map[string]string{
+			crypto.MetaSealedKeyS3:       base64.StdEncoding.EncodeToString(make([]byte, 64)),
 			crypto.MetaKeyID:             "kms-key",
-			crypto.MetaDataEncryptionKey: "m-key"},
+			crypto.MetaDataEncryptionKey: "m-key",
+		},
 		encryptionType: "",
-		err:            nil}, // 5
-	{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
-		xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		err:            nil,
+	}, // 5
+	{
+		headers: http.Header{
+			xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
 		copySource:     true,
 		metadata:       nil,
 		encryptionType: encrypt.SSEC,
-		err:            nil}, // 6
-	{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
-		xhttp.AmzServerSideEncryptionCopyCustomerKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
-		xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
+		err:            nil,
+	}, // 6
+	{
+		headers: http.Header{
+			xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
+			xhttp.AmzServerSideEncryptionCopyCustomerKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
 		copySource:     false,
 		metadata:       nil,
 		encryptionType: "",
-		err:            nil}, // 7
+		err:            nil,
+	}, // 7
 }
 
 func TestGetDefaultOpts(t *testing.T) {

cmd/endpoint-ellipses.go

@@ -90,7 +90,7 @@ func commonSetDriveCount(divisibleSize uint64, setCounts []uint64) (setSize uint
 // input argument patterns, the symmetry calculation is to ensure that
 // we also use uniform number of drives common across all ellipses patterns.
 func possibleSetCountsWithSymmetry(setCounts []uint64, argPatterns []ellipses.ArgPattern) []uint64 {
-	var newSetCounts = make(map[uint64]struct{})
+	newSetCounts := make(map[uint64]struct{})
 	for _, ss := range setCounts {
 		var symmetry bool
 		for _, argPattern := range argPatterns {
@@ -224,7 +224,7 @@ func (s endpointSet) getEndpoints() (endpoints []string) {
 // this function also intelligently decides on what will
 // be the right set size etc.
 func (s endpointSet) Get() (sets [][]string) {
-	var k = uint64(0)
+	k := uint64(0)
 	endpoints := s.getEndpoints()
 	for i := range s.setIndexes {
 		for j := range s.setIndexes[i] {
@@ -253,7 +253,7 @@ func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 {
 // of endpoints following the ellipses pattern, this is what is used
 // by the object layer for initializing itself.
 func parseEndpointSet(customSetDriveCount uint64, args ...string) (ep endpointSet, err error) {
-	var argPatterns = make([]ellipses.ArgPattern, len(args))
+	argPatterns := make([]ellipses.ArgPattern, len(args))
 	for i, arg := range args {
 		patterns, perr := ellipses.FindEllipsesPatterns(arg)
 		if perr != nil {
@@ -332,15 +332,13 @@ const (
 	EnvErasureSetDriveCount = "MINIO_ERASURE_SET_DRIVE_COUNT"
 )
 
-var (
-	globalCustomErasureDriveCount = false
-)
+var globalCustomErasureDriveCount = false
 
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
 func createServerEndpoints(serverAddr string, args ...string) (
-	endpointServerPools EndpointServerPools, setupType SetupType, err error) {
-
+	endpointServerPools EndpointServerPools, setupType SetupType, err error,
+) {
 	if len(args) == 0 {
 		return nil, -1, errInvalidArgument
 	}

cmd/endpoint-ellipses_test.go

@@ -72,7 +72,8 @@ func TestGetDivisibleSize(t *testing.T) {
 	testCases := []struct {
 		totalSizes []uint64
 		result     uint64
-	}{{[]uint64{24, 32, 16}, 8},
+	}{
+		{[]uint64{24, 32, 16}, 8},
 		{[]uint64{32, 8, 4}, 4},
 		{[]uint64{8, 8, 8}, 8},
 		{[]uint64{24}, 24},
@@ -168,7 +169,7 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			var argPatterns = make([]ellipses.ArgPattern, len(testCase.args))
+			argPatterns := make([]ellipses.ArgPattern, len(testCase.args))
 			for i, arg := range testCase.args {
 				patterns, err := ellipses.FindEllipsesPatterns(arg)
 				if err != nil {
@@ -290,7 +291,7 @@ func TestGetSetIndexes(t *testing.T) {
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			var argPatterns = make([]ellipses.ArgPattern, len(testCase.args))
+			argPatterns := make([]ellipses.ArgPattern, len(testCase.args))
 			for i, arg := range testCase.args {
 				patterns, err := ellipses.FindEllipsesPatterns(arg)
 				if err != nil {
@@ -552,8 +553,10 @@ func TestParseEndpointSet(t *testing.T) {
 					},
 				},
 				nil,
-				[][]uint64{{16, 16, 16, 16, 16, 16, 16, 16,
-					16, 16, 16, 16, 16, 16, 16, 16}},
+				[][]uint64{{
+					16, 16, 16, 16, 16, 16, 16, 16,
+					16, 16, 16, 16, 16, 16, 16, 16,
+				}},
 			},
 			true,
 		},

cmd/endpoint_test.go

@@ -238,13 +238,18 @@ func TestCreateEndpoints(t *testing.T) {
 		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 
 		// Erasure Setup with PathEndpointType
-		{":1234", [][]string{{"/d1", "/d2", "/d3", "/d4"}}, ":1234",
+		{
+			":1234",
+			[][]string{{"/d1", "/d2", "/d3", "/d4"}},
+			":1234",
 			Endpoints{
 				Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true},
 				Endpoint{URL: &url.URL{Path: mustAbs("/d2")}, IsLocal: true},
 				Endpoint{URL: &url.URL{Path: mustAbs("/d3")}, IsLocal: true},
 				Endpoint{URL: &url.URL{Path: mustAbs("/d4")}, IsLocal: true},
-			}, ErasureSetupType, nil},
+			},
+			ErasureSetupType, nil,
+		},
 		// DistErasure Setup with URLEndpointType
 		{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", Endpoints{
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d1"}, IsLocal: true},
@@ -350,12 +355,18 @@ func TestGetLocalPeer(t *testing.T) {
 		expectedResult string
 	}{
 		{[]string{"/d1", "/d2", "d3", "d4"}, "127.0.0.1:9000"},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"},
-			"localhost:9000"},
-		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"},
-			"localhost:9000"},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"},
-			"localhost:9000"},
+		{
+			[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"},
+			"localhost:9000",
+		},
+		{
+			[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"},
+			"localhost:9000",
+		},
+		{
+			[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"},
+			"localhost:9000",
+		},
 	}
 
 	for i, testCase := range testCases {

cmd/erasure-bucket.go

@@ -64,8 +64,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 		}, index)
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, writeQuorum)
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
 	return toObjectErr(err, bucket)
 }
 
@@ -92,7 +91,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
-	var bucketsInfo = make([]BucketInfo, len(storageDisks))
+	bucketsInfo := make([]BucketInfo, len(storageDisks))
 	// Undo previous make bucket entry on all underlying storage disks.
 	for index := range storageDisks {
 		index := index
@@ -121,8 +120,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 	// reduce to one error based on read quorum.
 	// `nil` is deliberately passed for ignoredErrs
 	// because these errors were already ignored.
-	readQuorum := getReadQuorum(len(storageDisks))
-	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, readQuorum)
+	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, er.defaultRQuorum())
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
@@ -167,8 +165,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts D
 		return nil
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum)
+	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, er.defaultWQuorum())
 	if err == errErasureWriteQuorum && !opts.NoRecreate {
 		undoDeleteBucket(storageDisks, bucket)
 	}

cmd/erasure-coding.go

@@ -94,7 +94,7 @@ func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error)
 // It only decodes the data blocks but does not verify them.
 // It returns an error if the decoding failed.
 func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
-	var isZero = 0
+	isZero := 0
 	for _, b := range data {
 		if len(b) == 0 {
 			isZero++

cmd/erasure-common.go

@@ -50,7 +50,7 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 
 	var wg sync.WaitGroup
 	var mu sync.Mutex
-	var newDisks = map[uint64][]StorageAPI{}
+	newDisks := map[uint64][]StorageAPI{}
 	// Based on the random shuffling return back randomized disks.
 	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
 		i := i

cmd/erasure-decode.go

@@ -282,3 +282,51 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 
 	return bytesWritten, derr
 }
+
+// Heal reads from readers, reconstruct shards and writes the data to the writers.
+func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.ReaderAt, totalLength int64) (derr error) {
+	if len(writers) != e.parityBlocks+e.dataBlocks {
+		return errInvalidArgument
+	}
+
+	reader := newParallelReader(readers, e, 0, totalLength)
+
+	startBlock := int64(0)
+	endBlock := totalLength / e.blockSize
+	if totalLength%e.blockSize != 0 {
+		endBlock++
+	}
+
+	var bufs [][]byte
+	for block := startBlock; block < endBlock; block++ {
+		var err error
+		bufs, err = reader.Read(bufs)
+		if len(bufs) > 0 {
+			if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) {
+				if derr == nil {
+					derr = err
+				}
+			}
+		} else if err != nil {
+			return err
+		}
+
+		if err = e.DecodeDataAndParityBlocks(ctx, bufs); err != nil {
+			logger.LogIf(ctx, err)
+			return err
+		}
+
+		w := parallelWriter{
+			writers:     writers,
+			writeQuorum: 1,
+			errs:        make([]error, len(writers)),
+		}
+
+		err = w.Write(ctx, bufs)
+		if err != nil {
+			return err
+		}
+	}
+
+	return derr
+}

cmd/erasure-heal_test.go

@@ -24,9 +24,6 @@ import (
 	"io"
 	"os"
 	"testing"
-
-	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio/internal/bpool"
 )
 
 var erasureHealTests = []struct {
@@ -137,15 +134,8 @@ func TestErasureHeal(t *testing.T) {
 			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 
-		// Number of buffers, max 2GB
-		n := (2 * humanize.GiByte) / (int(test.blocksize) * 2)
-
-		// Initialize byte pool once for all sets, bpool size is set to
-		// setCount * setDriveCount with each memory upto blockSizeV2.
-		bp := bpool.NewBytePoolCap(n, int(test.blocksize), int(test.blocksize)*2)
-
 		// test case setup is complete - now call Heal()
-		err = erasure.Heal(context.Background(), readers, staleWriters, test.size, bp)
+		err = erasure.Heal(context.Background(), staleWriters, readers, test.size)
 		closeBitrotReaders(readers)
 		closeBitrotWriters(staleWriters)
 		if err != nil && !test.shouldFail {

cmd/erasure-healing-common.go

@@ -26,29 +26,59 @@ import (
 )
 
 // commonTime returns a maximally occurring time from a list of time.
-func commonTime(modTimes []time.Time) (modTime time.Time) {
-	timeOccurenceMap := make(map[int64]int, len(modTimes))
+func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime time.Time, maxima int) {
+	timeOccurenceMap := make(map[int64]int, len(times))
+	groupNano := group.Nanoseconds()
 	// Ignore the uuid sentinel and count the rest.
-	for _, t := range modTimes {
+	for _, t := range times {
 		if t.Equal(timeSentinel) {
 			continue
 		}
-		timeOccurenceMap[t.UnixNano()]++
+		nano := t.UnixNano()
+		if group > 0 {
+			for k := range timeOccurenceMap {
+				if k == nano {
+					// We add to ourself later
+					continue
+				}
+				diff := k - nano
+				if diff < 0 {
+					diff = -diff
+				}
+				// We are within the limit
+				if diff < groupNano {
+					timeOccurenceMap[k]++
+				}
+			}
+		}
+		// Add ourself...
+		timeOccurenceMap[nano]++
 	}
 
-	var maxima int // Counter for remembering max occurrence of elements.
+	maxima = 0 // Counter for remembering max occurrence of elements.
+	latest := int64(0)
 
 	// Find the common cardinality from previously collected
 	// occurrences of elements.
 	for nano, count := range timeOccurenceMap {
-		t := time.Unix(0, nano).UTC()
-		if count > maxima || (count == maxima && t.After(modTime)) {
+		if count < maxima {
+			continue
+		}
+
+		// We are at or above maxima
+		if count > maxima || nano > latest {
 			maxima = count
-			modTime = t
+			latest = nano
 		}
 	}
 
-	// Return the collected common modTime.
+	// Return the collected common max time, with maxima
+	return time.Unix(0, latest).UTC(), maxima
+}
+
+// commonTime returns a maximally occurring time from a list of time.
+func commonTime(modTimes []time.Time) (modTime time.Time) {
+	modTime, _ = commonTimeAndOccurence(modTimes, 0)
 	return modTime
 }
 
@@ -88,6 +118,19 @@ func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks
 	}
 }
 
+// Extracts list of disk mtimes from FileInfo slice and returns, skips
+// slice elements that have errors.
+func listObjectDiskMtimes(partsMetadata []FileInfo) (diskMTimes []time.Time) {
+	diskMTimes = bootModtimes(len(partsMetadata))
+	for index, metadata := range partsMetadata {
+		if metadata.IsValid() {
+			// Once the file is found, save the disk mtime saved on disk.
+			diskMTimes[index] = metadata.DiskMTime
+		}
+	}
+	return diskMTimes
+}
+
 // Notes:
 // There are 5 possible states a disk could be in,
 // 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
@@ -182,8 +225,15 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 // - slice of errors about the state of data files on disk - can have
 //   a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
-	errs []error, latestMeta FileInfo,
-	bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
+	errs []error, latestMeta FileInfo, bucket, object string,
+	scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time,
+) {
+	var diskMTime time.Time
+	var shardFix bool
+	if !latestMeta.DataShardFixed() {
+		diskMTime = pickValidDiskTimeWithQuorum(partsMetadata,
+			latestMeta.Erasure.DataBlocks)
+	}
 
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
@@ -234,6 +284,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 
 		if erasureDistributionReliable {
 			if !meta.IsValid() {
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
 				continue
 			}
 
@@ -256,6 +308,23 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			}
 		}
 
+		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+			if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
+				// not with in acceptable delta, skip.
+				// If disk mTime mismatches it is considered outdated
+				// https://github.com/minio/minio/pull/13803
+				//
+				// This check only is active if we could find maximally
+				// occurring disk mtimes that are somewhat same across
+				// the quorum. Allowing to skip those shards which we
+				// might think are wrong.
+				shardFix = true
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+		}
+
 		// Always check data, if we got it.
 		if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
 			checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
@@ -298,5 +367,10 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 	}
 
-	return availableDisks, dataErrs
+	if shardFix {
+		// Only when shard is fixed return an appropriate disk mtime value.
+		return availableDisks, dataErrs, diskMTime
+	} // else return timeSentinel for disk mtime
+
+	return availableDisks, dataErrs, timeSentinel
 }

cmd/erasure-healing-common_test.go

@@ -251,7 +251,8 @@ func TestListOnlineDisks(t *testing.T) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
 			}
-			availableDisks, newErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan)
+			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, bucket, object, madmin.HealDeepScan)
 			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
@@ -433,7 +434,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 					test.expectedTime, modTime)
 			}
 
-			availableDisks, newErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan)
+			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, bucket, object, madmin.HealDeepScan)
 			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
@@ -494,7 +496,8 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	erasureDisks, _ = listOnlineDisks(erasureDisks, partsMetadata, errs)
 
-	filteredDisks, errs := disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -515,7 +518,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[0].ModTime = partsMetadata[0].ModTime.Add(-1 * time.Hour)
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -535,7 +539,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[1].DataDir = "foo-random"
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -571,7 +576,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, errs = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, errs, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))

cmd/erasure-healing.go

@@ -24,12 +24,38 @@ import (
 	"fmt"
 	"io"
 	"sync"
+	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 )
 
+const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
+
+// AcceptableDelta returns 'true' if the fi.DiskMTime is under
+// acceptable delta of "delta" duration with maxTime.
+//
+// This code is primarily used for heuristic detection of
+// incorrect shards, as per https://github.com/minio/minio/pull/13803
+//
+// This check only is active if we could find maximally
+// occurring disk mtimes that are somewhat same across
+// the quorum. Allowing to skip those shards which we
+// might think are wrong.
+func (fi FileInfo) AcceptableDelta(maxTime time.Time, delta time.Duration) bool {
+	diff := maxTime.Sub(fi.DiskMTime)
+	if diff < 0 {
+		diff = -diff
+	}
+	return diff < delta
+}
+
+// DataShardFixed - data shard fixed?
+func (fi FileInfo) DataShardFixed() bool {
+	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
+}
+
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
@@ -220,10 +246,15 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
-	case errors.Is(erErr, errCorruptedFormat):
+	case errors.Is(erErr, errFileCorrupt):
 		return true
 	}
 	if erErr == nil {
+		if meta.XLV1 {
+			// Legacy means heal always
+			// always check first.
+			return true
+		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
@@ -234,19 +265,7 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 				return true
 			}
 		}
-		if !latestMeta.MetadataEquals(meta) {
-			return true
-		}
-		if !latestMeta.TransitionInfoEquals(meta) {
-			return true
-		}
-		if !latestMeta.ReplicationInfoEquals(meta) {
-			return true
-		}
-		if !latestMeta.ModTime.Equal(meta.ModTime) {
-			return true
-		}
-		if meta.XLV1 {
+		if !latestMeta.Equals(meta) {
 			return true
 		}
 	}
@@ -267,12 +286,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// Initialize heal result object
 	result = madmin.HealResultItem{
-		Type:         madmin.HealItemObject,
-		Bucket:       bucket,
-		Object:       object,
-		DiskCount:    len(storageDisks),
-		ParityBlocks: er.defaultParityCount,
-		DataBlocks:   len(storageDisks) - er.defaultParityCount,
+		Type:      madmin.HealItemObject,
+		Bucket:    bucket,
+		Object:    object,
+		DiskCount: len(storageDisks),
 	}
 
 	if !opts.NoLock {
@@ -287,18 +304,27 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
-
-	if _, err = getLatestFileInfo(ctx, partsMetadata, errs); err != nil {
-		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
+	if isAllNotFound(errs) {
+		// Nothing to do, file is already gone.
+		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), nil
 	}
 
-	// List of disks having latest version of the object er.meta
+	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
+	if err != nil {
+		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, nil, opts)
+	}
+
+	result.ParityBlocks = result.DiskCount - readQuorum
+	result.DataBlocks = readQuorum
+
+	// List of disks having latest version of the object xl.meta
 	// (by modtime).
 	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, result.DataBlocks)
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, readQuorum)
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, versionID)
 	}
@@ -312,7 +338,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// used here for reconstruction. This is done to ensure that
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
-	availableDisks, dataErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
 	// Loop to find number of disks with valid data, per-drive
@@ -330,10 +356,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			// If data is sane on any one disk, we can
 			// extract the correct object size.
 			result.ObjectSize = partsMetadata[i].Size
-			if partsMetadata[i].Erasure.ParityBlocks > 0 && partsMetadata[i].Erasure.DataBlocks > 0 {
-				result.ParityBlocks = partsMetadata[i].Erasure.ParityBlocks
-				result.DataBlocks = partsMetadata[i].Erasure.DataBlocks
-			}
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
 		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
@@ -373,18 +395,14 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	}
 
 	if isAllNotFound(errs) {
-		err = toObjectErr(errFileNotFound, bucket, object)
-		if versionID != "" {
-			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-		}
 		// File is fully gone, fileInfo is empty.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs,
-			bucket, object, versionID), err
+			bucket, object, versionID), nil
 	}
 
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
-	if numAvailableDisks < result.DataBlocks {
+	if numAvailableDisks < readQuorum {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, dataErrs, opts)
 	}
 
@@ -444,9 +462,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	copyPartsMetadata = shufflePartsMetadata(copyPartsMetadata, latestMeta.Erasure.Distribution)
 
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		result.DataBlocks = latestMeta.Erasure.DataBlocks
-		result.ParityBlocks = latestMeta.Erasure.ParityBlocks
-
 		// Heal each part. erasureHealFile() will write the healed
 		// part to .minio/tmp/uuid/ which needs to be renamed later to
 		// the final location.
@@ -457,10 +472,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 
 		erasureInfo := latestMeta.Erasure
-		bp := er.bp
-		if erasureInfo.BlockSize == blockSizeV1 {
-			bp = er.bpOld
-		}
 		for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ {
 			partSize := latestMeta.Parts[partIndex].Size
 			partActualSize := latestMeta.Parts[partIndex].ActualSize
@@ -491,7 +502,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
-			err = erasure.Heal(ctx, readers, writers, partSize, bp)
+			err = erasure.Heal(ctx, writers, readers, partSize)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
 			if err != nil {
@@ -566,6 +577,20 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 	}
 
+	if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+		// Update metadata to indicate special fix.
+		_, err = er.PutObjectMetadata(ctx, bucket, object, ObjectOptions{
+			NoLock: true,
+			UserDefined: map[string]string{
+				reservedMetadataPrefixLowerDataShardFix: "true",
+				// another reserved metadata to capture original disk-mtime
+				// captured for this version of the object, to be used
+				// possibly in future to heal other versions if possible.
+				ReservedMetadataPrefixLower + "disk-mtime": diskMTime.String(),
+			},
+		})
+	}
+
 	// Set the size of the object in the heal result
 	result.ObjectSize = latestMeta.Size
 
@@ -634,7 +659,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 	}
 	if dryRun || danglingObject || isAllNotFound(errs) {
 		// Nothing to do, file is already gone.
-		return hr, toObjectErr(errFileNotFound, bucket, object)
+		return hr, nil
 	}
 	for i, err := range errs {
 		if err == errVolumeNotFound || err == errFileNotFound {
@@ -739,8 +764,15 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 // A 0 length slice will always return false.
 func isAllNotFound(errs []error) bool {
 	for _, err := range errs {
-		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) || errors.Is(err, errFileVersionNotFound) {
-			continue
+		if err != nil {
+			switch err.Error() {
+			case errFileNotFound.Error():
+				fallthrough
+			case errVolumeNotFound.Error():
+				fallthrough
+			case errFileVersionNotFound.Error():
+				continue
+			}
 		}
 		return false
 	}
@@ -772,29 +804,28 @@ func isObjectDirDangling(errs []error) (ok bool) {
 }
 
 func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object, versionID string,
-	metaArr []FileInfo, errs []error, dataErrs []error, opts madmin.HealOpts) (madmin.HealResultItem, error) {
-
+	metaArr []FileInfo, errs []error, dataErrs []error, opts madmin.HealOpts) (madmin.HealResultItem, error,
+) {
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 	// Check if the object is dangling, if yes and user requested
 	// remove we simply delete it from namespace.
 	m, ok := isObjectDangling(metaArr, errs, dataErrs)
 	if ok {
-		parityBlocks := m.Erasure.ParityBlocks
-		if m.Erasure.ParityBlocks == 0 {
-			parityBlocks = er.defaultParityCount
-		}
-		dataBlocks := m.Erasure.DataBlocks
-		if m.Erasure.DataBlocks == 0 {
-			dataBlocks = len(storageDisks) - parityBlocks
+		parityBlocks := er.defaultParityCount
+		dataBlocks := len(storageDisks) - parityBlocks
+		if m.IsValid() {
+			parityBlocks = m.Erasure.ParityBlocks
+			dataBlocks = m.Erasure.DataBlocks
 		}
+
 		writeQuorum := dataBlocks
 		if dataBlocks == parityBlocks {
 			writeQuorum++
 		}
+
 		var err error
-		var returnNotFound bool
-		if !opts.DryRun && opts.Remove {
+		if opts.Remove {
 			err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{
 				VersionID: versionID,
 			}, false)
@@ -810,19 +841,10 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 				m.Size = 0
 			}
 
-			// Delete successfully purged dangling content, return ObjectNotFound/VersionNotFound instead.
-			if countErrs(errs, nil) == len(errs) {
-				returnNotFound = true
-			}
-		}
-		if returnNotFound {
-			err = toObjectErr(errFileNotFound, bucket, object)
-			if versionID != "" {
-				err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-			}
-			return er.defaultHealResult(m, storageDisks, storageEndpoints,
-				errs, bucket, object, versionID), err
+			return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+				errs, bucket, object, versionID), nil
 		}
+
 		return er.defaultHealResult(m, storageDisks, storageEndpoints,
 			errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID)
 	}
@@ -839,54 +861,64 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 // files is lesser than number of data blocks.
 func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (validMeta FileInfo, ok bool) {
 	// We can consider an object data not reliable
-	// when er.meta is not found in read quorum disks.
-	// or when er.meta is not readable in read quorum disks.
-	var notFoundErasureMeta, corruptedErasureMeta int
-	for _, readErr := range errs {
-		if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
-			notFoundErasureMeta++
-		} else if errors.Is(readErr, errCorruptedFormat) {
-			corruptedErasureMeta++
-		}
-	}
-	var notFoundParts int
-	for i := range dataErrs {
-		// Only count part errors, if the error is not
-		// same as er.meta error. This is to avoid
-		// double counting when both parts and er.meta
-		// are not available.
-		if errs[i] != dataErrs[i] {
-			if IsErr(dataErrs[i], []error{
-				errFileNotFound,
-				errFileVersionNotFound,
-			}...) {
-				notFoundParts++
+	// when xl.meta is not found in read quorum disks.
+	// or when xl.meta is not readable in read quorum disks.
+	danglingErrsCount := func(cerrs []error) (int, int) {
+		var (
+			notFoundCount  int
+			corruptedCount int
+		)
+		for _, readErr := range cerrs {
+			if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
+				notFoundCount++
+			} else if errors.Is(readErr, errFileCorrupt) {
+				corruptedCount++
 			}
 		}
+		return notFoundCount, corruptedCount
 	}
 
+	ndataErrs := make([]error, len(dataErrs))
+	for i := range dataErrs {
+		if errs[i] != dataErrs[i] {
+			// Only count part errors, if the error is not
+			// same as xl.meta error. This is to avoid
+			// double counting when both parts and xl.meta
+			// are not available.
+			ndataErrs[i] = dataErrs[i]
+		}
+	}
+
+	notFoundMetaErrs, corruptedMetaErrs := danglingErrsCount(errs)
+	notFoundPartsErrs, corruptedPartsErrs := danglingErrsCount(ndataErrs)
+
 	for _, m := range metaArr {
-		if !m.IsValid() {
-			continue
+		if m.IsValid() {
+			validMeta = m
+			break
 		}
-		validMeta = m
-		break
 	}
 
-	if validMeta.Deleted || validMeta.IsRemote() {
-		// notFoundParts is ignored since a
+	if !validMeta.IsValid() {
+		// We have no idea what this file is, leave it as is.
+		return validMeta, false
+	}
+
+	if validMeta.Deleted {
+		// notFoundPartsErrs is ignored since
 		// - delete marker does not have any parts
-		// - transition status of complete has no parts
-		return validMeta, corruptedErasureMeta+notFoundErasureMeta > len(errs)/2
+		return validMeta, corruptedMetaErrs+notFoundMetaErrs > len(errs)/2
 	}
 
-	// We couldn't find any valid meta we are indeed corrupted, return true right away.
-	if validMeta.Erasure.DataBlocks == 0 {
-		return validMeta, true
+	totalErrs := notFoundMetaErrs + corruptedMetaErrs + notFoundPartsErrs + corruptedPartsErrs
+	if validMeta.IsRemote() {
+		// notFoundPartsErrs is ignored since
+		// - transition status of complete has no parts
+		totalErrs = notFoundMetaErrs + corruptedMetaErrs
 	}
 
 	// We have valid meta, now verify if we have enough files with parity blocks.
-	return validMeta, corruptedErasureMeta+notFoundErasureMeta+notFoundParts > validMeta.Erasure.ParityBlocks
+	return validMeta, totalErrs > validMeta.Erasure.ParityBlocks
 }
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.
@@ -925,13 +957,9 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	// This allows to quickly check if all is ok or all are missing.
 	_, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
 	if isAllNotFound(errs) {
-		err = toObjectErr(errFileNotFound, bucket, object)
-		if versionID != "" {
-			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-		}
 		// Nothing to do, file is already gone.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
-			errs, bucket, object, versionID), err
+			errs, bucket, object, versionID), nil
 	}
 
 	// Heal the object.

cmd/erasure-healing_test.go

@@ -21,10 +21,12 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
+	"errors"
+	"io"
 	"os"
 	"path"
 	"reflect"
-	"runtime"
 	"testing"
 	"time"
 
@@ -32,6 +34,192 @@ import (
 	"github.com/minio/madmin-go"
 )
 
+// Tests isObjectDangling function
+func TestIsObjectDangling(t *testing.T) {
+	fi := newFileInfo("test-object", 2, 2)
+	fi.Erasure.Index = 1
+
+	testCases := []struct {
+		name             string
+		metaArr          []FileInfo
+		errs             []error
+		dataErrs         []error
+		expectedMeta     FileInfo
+		expectedDangling bool
+	}{
+		{
+			name: "FileInfoExists-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				fi,
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				nil,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoExists-case2",
+			metaArr: []FileInfo{
+				{},
+				{},
+				fi,
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoUndecided-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				errDiskNotFound,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name:    "FileInfoUndecided-case2",
+			metaArr: []FileInfo{},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				errDiskNotFound,
+				errFileNotFound,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{},
+			expectedDangling: false,
+		},
+		{
+			name:    "FileInfoUndecided-case3(file deleted)",
+			metaArr: []FileInfo{},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{},
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoDecided-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2-delete-marker",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				{Deleted: true},
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{Deleted: true},
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2-(duplicate data errors)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				{Deleted: true},
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				nil,
+				nil,
+			},
+			expectedMeta:     FileInfo{Deleted: true},
+			expectedDangling: true,
+		},
+		// Add new cases as seen
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.name, func(t *testing.T) {
+			gotMeta, dangling := isObjectDangling(testCase.metaArr, testCase.errs, testCase.dataErrs)
+			if !gotMeta.Equals(testCase.expectedMeta) {
+				t.Errorf("Expected %#v, got %#v", testCase.expectedMeta, gotMeta)
+			}
+			if dangling != testCase.expectedDangling {
+				t.Errorf("Expected dangling %t, got %t", testCase.expectedDangling, dangling)
+			}
+		})
+	}
+}
+
 // Tests both object and bucket healing.
 func TestHealing(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
@@ -92,16 +280,16 @@ func TestHealing(t *testing.T) {
 	}
 
 	// After heal the meta file should be as expected.
-	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
+	if !fileInfoPreHeal.Equals(fileInfoPostHeal) {
 		t.Fatal("HealObject failed")
 	}
 
-	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "er.meta"))
+	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "xl.meta"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Write er.meta with different modtime to simulate the case where a disk had
+	// Write xl.meta with different modtime to simulate the case where a disk had
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal
 	fileInfoOutDated.ModTime = time.Now()
@@ -121,7 +309,7 @@ func TestHealing(t *testing.T) {
 	}
 
 	// After heal the meta file should be as expected.
-	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
+	if !fileInfoPreHeal.Equals(fileInfoPostHeal) {
 		t.Fatal("HealObject failed")
 	}
 
@@ -147,10 +335,6 @@ func TestHealing(t *testing.T) {
 }
 
 func TestHealingDanglingObject(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -180,6 +364,9 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
+	disks = objLayer.(*erasureServerPools).serverPools[0].erasureDisks[0]
+	orgDisks := append([]StorageAPI{}, disks...)
+
 	// Enable versioning.
 	globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`))
 
@@ -190,11 +377,13 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0400); err != nil {
-			t.Fatal(err)
-		}
+	setDisks := func(newDisks ...StorageAPI) {
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock()
+		copy(disks, newDisks)
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
 	}
+	// Remove 4 disks.
+	setDisks(nil, nil, nil, nil)
 
 	// Create delete marker under quorum.
 	objInfo, err := objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: true})
@@ -202,11 +391,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0755); err != nil {
-			t.Fatal(err)
-		}
-	}
+	// Restore...
+	setDisks(orgDisks[:4]...)
 
 	fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
@@ -243,11 +429,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		}
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0400); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(nil, nil, nil, nil)
 
 	rd := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "")
 	_, err = objLayer.PutObject(ctx, bucket, object, rd, ObjectOptions{
@@ -257,11 +439,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0755); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(orgDisks[:4]...)
 
 	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
@@ -297,11 +475,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0400); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(nil, nil, nil, nil)
 
 	// Create delete marker under quorum.
 	_, err = objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{
@@ -312,11 +486,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0755); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(orgDisks[:4]...)
 
 	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
@@ -345,6 +515,298 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 }
 
+func TestHealCorrectQuorum(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	cfgFile := pathJoin(bucketMetaPrefix, bucket, ".test.bin")
+	if err = saveConfig(ctx, objLayer, cfgFile, data); err != nil {
+		t.Fatal(err)
+	}
+
+	hopts := madmin.HealOpts{
+		DryRun:   false,
+		Remove:   true,
+		ScanMode: madmin.HealNormalScan,
+	}
+
+	// Test 1: Remove the object backend files from the first disk.
+	z := objLayer.(*erasureServerPools)
+	for _, set := range z.serverPools {
+		er := set.sets[0]
+		erasureDisks := er.getDisks()
+
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, it should heal the content properly.
+		_, err = objLayer.HealObject(ctx, bucket, object, "", hopts)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		if countErrs(errs, nil) != len(fileInfos) {
+			t.Fatal("Expected all xl.meta healed, but partial heal detected")
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false)
+		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, it should heal the content properly.
+		_, err = objLayer.HealObject(ctx, minioMetaBucket, cfgFile, "", hopts)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false)
+		if countErrs(errs, nil) != len(fileInfos) {
+			t.Fatal("Expected all xl.meta healed, but partial heal detected")
+		}
+	}
+}
+
+func TestHealObjectCorruptedPools(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	// Test 1: Remove the object backend files from the first disk.
+	z := objLayer.(*erasureServerPools)
+	for _, set := range z.serverPools {
+		er := set.sets[0]
+		erasureDisks := er.getDisks()
+		firstDisk := erasureDisks[0]
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		if err != nil {
+			t.Fatalf("Failed to delete a file - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+		if err != nil {
+			t.Fatalf("Failed to heal object - %v", err)
+		}
+
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+			t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+		}
+
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+		if err != nil {
+			t.Errorf("Failure during deleting part.1 - %v", err)
+		}
+
+		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+		if err != nil {
+			t.Errorf("Failure during creating part.1 - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			t.Errorf("Expected nil but received %v", err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if !reflect.DeepEqual(fi, nfi) {
+			t.Fatalf("FileInfo not equal after healing")
+		}
+
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+		if err != nil {
+			t.Errorf("Failure during deleting part.1 - %v", err)
+		}
+
+		bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
+		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+		if err != nil {
+			t.Errorf("Failure during creating part.1 - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			t.Errorf("Expected nil but received %v", err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if !reflect.DeepEqual(fi, nfi) {
+			t.Fatalf("FileInfo not equal after healing")
+		}
+
+		// Test 4: checks if HealObject returns an error when xl.meta is not found
+		// in more than read quorum number of disks, to create a corrupted situation.
+		for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, expect to receive errFileNotFound.
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			if _, ok := err.(ObjectNotFound); !ok {
+				t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+			}
+		}
+
+		// since majority of xl.meta's are not available, object should be successfully deleted.
+		_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+		if _, ok := err.(ObjectNotFound); !ok {
+			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+		}
+
+		for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
+			_, err = erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+			if err == nil {
+				t.Errorf("Expected xl.meta file to be not present, but succeeeded")
+			}
+		}
+	}
+}
+
 func TestHealObjectCorrupted(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -422,7 +884,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
 	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
@@ -566,7 +1028,7 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
 	erasureDisks := er.getDisks()
@@ -582,7 +1044,7 @@ func TestHealObjectErasure(t *testing.T) {
 
 	// Try healing now, expect to receive errDiskNotFound.
 	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
-	// since majority of er.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
+	// since majority of xl.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
 	if _, ok := err.(InsufficientReadQuorum); !ok {
 		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)
 	}
@@ -668,3 +1130,134 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 		}
 	}
 }
+
+func TestHealLastDataShard(t *testing.T) {
+	tests := []struct {
+		name     string
+		dataSize int64
+	}{
+		{"4KiB", 4 * humanize.KiByte},
+		{"64KiB", 64 * humanize.KiByte},
+		{"128KiB", 128 * humanize.KiByte},
+		{"1MiB", 1 * humanize.MiByte},
+		{"5MiB", 5 * humanize.MiByte},
+		{"10MiB", 10 * humanize.MiByte},
+		{"5MiB-1KiB", 5*humanize.MiByte - 1*humanize.KiByte},
+		{"10MiB-1Kib", 10*humanize.MiByte - 1*humanize.KiByte},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			nDisks := 16
+			fsDirs, err := getRandomDisks(nDisks)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			defer removeRoots(fsDirs)
+
+			obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+			if err != nil {
+				t.Fatal(err)
+			}
+			bucket := "bucket"
+			object := "object"
+
+			data := make([]byte, test.dataSize)
+			_, err = rand.Read(data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var opts ObjectOptions
+
+			err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			if err != nil {
+				t.Fatalf("Failed to make a bucket - %v", err)
+			}
+
+			_, err = obj.PutObject(ctx, bucket, object,
+				mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			actualH := sha256.New()
+			_, err = io.Copy(actualH, bytes.NewReader(data))
+			if err != nil {
+				return
+			}
+			actualSha256 := actualH.Sum(nil)
+
+			z := obj.(*erasureServerPools)
+			er := z.serverPools[0].getHashedSet(object)
+
+			disks := er.getDisks()
+			distribution := hashOrder(pathJoin(bucket, object), nDisks)
+			shuffledDisks := shuffleDisks(disks, distribution)
+
+			// remove last data shard
+			err = removeAll(pathJoin(shuffledDisks[11].String(), bucket, object))
+			if err != nil {
+				t.Fatalf("Failed to delete a file - %v", err)
+			}
+			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{
+				ScanMode: madmin.HealNormalScan,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
+			defer firstGr.Close()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			firstHealedH := sha256.New()
+			_, err = io.Copy(firstHealedH, firstGr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			firstHealedDataSha256 := firstHealedH.Sum(nil)
+
+			if !bytes.Equal(actualSha256, firstHealedDataSha256) {
+				t.Fatalf("object healed wrong, expected %x, got %x",
+					actualSha256, firstHealedDataSha256)
+			}
+
+			// remove another data shard
+			if err = removeAll(pathJoin(shuffledDisks[1].String(), bucket, object)); err != nil {
+				t.Fatalf("Failed to delete a file - %v", err)
+			}
+
+			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{
+				ScanMode: madmin.HealNormalScan,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
+			defer secondGr.Close()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			secondHealedH := sha256.New()
+			_, err = io.Copy(secondHealedH, secondGr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			secondHealedDataSha256 := secondHealedH.Sum(nil)
+
+			if !bytes.Equal(actualSha256, secondHealedDataSha256) {
+				t.Fatalf("object healed wrong, expected %x, got %x",
+					actualSha256, secondHealedDataSha256)
+			}
+		})
+	}
+}

cmd/erasure-lowlevel-heal.go

@@ -1,59 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"io"
-
-	"github.com/minio/minio/internal/bpool"
-	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
-)
-
-// Heal heals the shard files on non-nil writers. Note that the quorum passed is 1
-// as healing should continue even if it has been successful healing only one shard file.
-func (e Erasure) Heal(ctx context.Context, readers []io.ReaderAt, writers []io.Writer, size int64, bp *bpool.BytePoolCap) error {
-	r, w := xioutil.WaitPipe()
-	go func() {
-		_, err := e.Decode(ctx, w, readers, 0, size, size, nil)
-		w.CloseWithError(err)
-	}()
-
-	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	var buffer []byte
-	switch {
-	case size == 0:
-		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
-	case size >= e.blockSize:
-		buffer = bp.Get()
-		defer bp.Put(buffer)
-	case size < e.blockSize:
-		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
-		buffer = make([]byte, size, 2*size+int64(e.parityBlocks+e.dataBlocks-1))
-	}
-
-	// quorum is 1 because CreateFile should continue writing as long as we are writing to even 1 disk.
-	n, err := e.Encode(ctx, r, writers, buffer, 1)
-	if err == nil && n != size {
-		logger.LogIf(ctx, errLessData)
-		err = errLessData
-	}
-	r.CloseWithError(err)
-	return err
-}

cmd/erasure-metadata-utils_test.go

@@ -87,9 +87,13 @@ func TestReduceErrs(t *testing.T) {
 			errDiskNotFound,
 		}, []error{errDiskNotFound}, errVolumeNotFound},
 		{[]error{}, []error{}, errErasureReadQuorum},
-		{[]error{errFileNotFound, errFileNotFound, errFileNotFound,
-			errFileNotFound, errFileNotFound, nil, nil, nil, nil, nil},
-			nil, nil},
+		{
+			[]error{
+				errFileNotFound, errFileNotFound, errFileNotFound,
+				errFileNotFound, errFileNotFound, nil, nil, nil, nil, nil,
+			},
+			nil, nil,
+		},
 		// Checks if wrapped context cancelation errors are grouped as one.
 		{canceledErrs, nil, context.Canceled},
 	}

cmd/erasure-metadata.go

@@ -351,6 +351,17 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 	return FileInfo{}, errErasureReadQuorum
 }
 
+func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time {
+	diskMTimes := listObjectDiskMtimes(metaArr)
+
+	diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta)
+	if diskMaxima >= quorum {
+		return diskMTime
+	}
+
+	return timeSentinel
+}
+
 // pickValidFileInfo - picks one valid FileInfo content and returns from a
 // slice of FileInfo.
 func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (FileInfo, error) {

cmd/erasure-multipart.go

@@ -820,7 +820,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadataByIndex(onlineDisks, partsMetadata, fi)
 
 	// Save current erasure metadata for validation.
-	var currentFI = fi
+	currentFI := fi
 
 	// Allocate parts similar to incoming slice.
 	fi.Parts = make([]ObjectPartInfo, len(parts))

cmd/erasure-object.go

@@ -145,7 +145,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	var unlockOnDefer bool
-	var nsUnlocker = func() {}
+	nsUnlocker := func() {}
 	defer func() {
 		if unlockOnDefer {
 			nsUnlocker()
@@ -179,6 +179,30 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return nil, toObjectErr(err, bucket, object)
 	}
 
+	if !fi.DataShardFixed() {
+		diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks)
+		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+			for index := range onlineDisks {
+				if onlineDisks[index] == OfflineDisk {
+					continue
+				}
+				if !metaArr[index].IsValid() {
+					continue
+				}
+				if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
+					// If disk mTime mismatches it is considered outdated
+					// https://github.com/minio/minio/pull/13803
+					//
+					// This check only is active if we could find maximally
+					// occurring disk mtimes that are somewhat same across
+					// the quorum. Allowing to skip those shards which we
+					// might think are wrong.
+					onlineDisks[index] = OfflineDisk
+				}
+			}
+		}
+	}
+
 	objInfo := fi.ToObjectInfo(bucket, object)
 	if objInfo.DeleteMarker {
 		if opts.VersionID == "" {
@@ -451,7 +475,6 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
-
 	}
 	objInfo = fi.ToObjectInfo(bucket, object)
 	if opts.VersionID != "" && !fi.VersionPurgeStatus().Empty() {
@@ -474,7 +497,7 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, wquorum int, err error) {
 	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
-		return objInfo, getWriteQuorum(len(er.getDisks())), toObjectErr(err, bucket, object)
+		return objInfo, er.defaultWQuorum(), toObjectErr(err, bucket, object)
 	}
 
 	wquorum = fi.Erasure.DataBlocks
@@ -1089,7 +1112,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		// class for objects which have reduced quorum
 		// storage class only needs to be honored for
 		// Read() requests alone which we already do.
-		writeQuorums[i] = getWriteQuorum(len(storageDisks))
+		writeQuorums[i] = er.defaultWQuorum()
 	}
 
 	versionsMap := make(map[string]FileInfoVersions, len(objects))
@@ -1153,7 +1176,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	}
 
 	// Initialize list of errors.
-	var delObjErrs = make([][]error, len(storageDisks))
+	delObjErrs := make([][]error, len(storageDisks))
 
 	var wg sync.WaitGroup
 	// Remove versions in bulk for each disk
@@ -1174,6 +1197,12 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 					continue
 				}
 				for _, v := range dedupVersions[i].Versions {
+					if err == errFileNotFound || err == errFileVersionNotFound {
+						if !dobjects[v.Idx].DeleteMarker {
+							// Not delete marker, if not found, ok.
+							continue
+						}
+					}
 					delObjErrs[index][v.Idx] = err
 				}
 			}
@@ -1365,6 +1394,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	fvID := mustGetUUID()
 	if markDelete {
 		if opts.Versioned || opts.VersionSuspended {
+			if !deleteMarker {
+				// versioning suspended means we add `null` version as
+				// delete marker, if its not decided already.
+				deleteMarker = opts.VersionSuspended && opts.VersionID == ""
+			}
 			fi := FileInfo{
 				Name:             object,
 				Deleted:          deleteMarker,
@@ -1381,10 +1415,10 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 					fi.VersionID = opts.VersionID
 				}
 			}
-			// versioning suspended means we add `null`
-			// version as delete marker
-			// Add delete marker, since we don't have any version specified explicitly.
-			// Or if a particular version id needs to be replicated.
+			// versioning suspended means we add `null` version as
+			// delete marker. Add delete marker, since we don't have
+			// any version specified explicitly. Or if a particular
+			// version id needs to be replicated.
 			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
 				return objInfo, toObjectErr(err, bucket, object)
 			}
@@ -1439,14 +1473,16 @@ func (er erasureObjects) addPartial(bucket, object, versionID string, size int64
 }
 
 func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
-	// Lock the object before updating tags.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return ObjectInfo{}, err
+	if !opts.NoLock {
+		// Lock the object before updating metadata.
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx.Cancel)
 	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
 
 	disks := er.getDisks()
 
@@ -1569,7 +1605,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	// Wait for all the routines.
 	mErrs := g.Wait()
 
-	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, getWriteQuorum(len(onlineDisks)))
+	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, er.defaultWQuorum())
 }
 
 // DeleteObjectTags - delete object tags from an existing object
@@ -1789,6 +1825,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 		})
 	}
 	_, err = er.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{
-		MTime: oi.ModTime})
+		MTime: oi.ModTime,
+	})
 	return setRestoreHeaderFn(oi, err)
 }