fix batch snowball to close channel after listing finishes (#19316)

panic seen due to premature closing of slow channel while listing is still sending or
list has already closed on the sender's side:
```
panic: close of closed channel

goroutine 13666 [running]:
github.com/minio/minio/internal/ioutil.SafeClose[...](0x101ff51e4?)
	/Users/kp/code/src/github.com/minio/minio/internal/ioutil/ioutil.go:425 +0x24
github.com/minio/minio/cmd.(*erasureServerPools).Walk.func1()
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:2142 +0x170
created by github.com/minio/minio/cmd.(*erasureServerPools).Walk in goroutine 1189
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:1985 +0x228
```

.github/workflows/multipart/migrate.sh

@@ -25,8 +25,8 @@ if [ ! -f ./mc ]; then
 fi
 
 (
-	cd /tmp
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	cd ./docs/debugging/s3-check-md5
+	go install -v
 )
 
 export RELEASE=RELEASE.2023-08-29T23-07-35Z

.github/workflows/typos.yml

@@ -0,0 +1,15 @@
+---
+name: Test GitHub Action
+on: [pull_request]
+
+jobs:
+  run:
+    name: Spell Check with Typos
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Actions Repository
+      uses: actions/checkout@v4
+
+    - name: Check spelling of repo
+      uses: crate-ci/typos@master
+

.github/workflows/vulncheck.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21.5
+          go-version: 1.21.8
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.typos.toml

@@ -0,0 +1,31 @@
+[files]
+extend-exclude = [
+    ".git/",
+    "docs/",
+]
+ignore-hidden = false
+
+[default]
+extend-ignore-re = [
+    "Patrick Collison",
+    "Copyright 2014 Unknwon",
+    "[0-9A-Za-z/+=]{64}",
+    "ZXJuZXQxDjAMBgNVBA-some-junk-Q4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF",
+    "eyJmb28iOiJiYXIifQ",
+    'http\.Header\{"X-Amz-Server-Side-Encryptio":',
+    'sessionToken',
+]
+
+[default.extend-words]
+"encrypter" = "encrypter"
+"requestor" = "requestor"
+
+[default.extend-identifiers]
+"bui" = "bui"
+"toi" = "toi"
+"ot" = "ot"
+"dm2nd" = "dm2nd"
+"HashiCorp" = "HashiCorp"
+"ParseND" = "ParseND"
+"ParseNDStream" = "ParseNDStream"
+"TestGetPartialObjectMisAligned" = "TestGetPartialObjectMisAligned"

CREDITS

@@ -1125,6 +1125,39 @@ https://cloud.google.com/go/storage
 
 ================================================================
 
+filippo.io/edwards25519
+https://filippo.io/edwards25519
+----------------------------------------------------------------
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/Azure/azure-pipeline-go
 https://github.com/Azure/azure-pipeline-go
 ----------------------------------------------------------------
@@ -3353,213 +3386,6 @@ Redistribution and use in source and binary forms, with or without modification,
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ================================================================
 
-github.com/cncf/xds/go
-https://github.com/cncf/xds/go
-----------------------------------------------------------------
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/containerd/console
 https://github.com/containerd/console
 ----------------------------------------------------------------
@@ -5767,214 +5593,6 @@ https://github.com/elastic/go-elasticsearch/v7
 
 ================================================================
 
-github.com/envoyproxy/protoc-gen-validate
-https://github.com/envoyproxy/protoc-gen-validate
-----------------------------------------------------------------
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/fatih/color
 https://github.com/fatih/color
 ----------------------------------------------------------------
@@ -16432,213 +16050,6 @@ https://github.com/matttproud/golang_protobuf_extensions
 
 ================================================================
 
-github.com/matttproud/golang_protobuf_extensions/v2
-https://github.com/matttproud/golang_protobuf_extensions/v2
-----------------------------------------------------------------
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/miekg/dns
 https://github.com/miekg/dns
 ----------------------------------------------------------------
@@ -18271,8 +17682,8 @@ https://github.com/minio/highwayhash
 
 ================================================================
 
-github.com/minio/kes-go
-https://github.com/minio/kes-go
+github.com/minio/kms-go/kes
+https://github.com/minio/kms-go/kes
 ----------------------------------------------------------------
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
@@ -27652,33 +27063,6 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
       the Mozilla Public License, v. 2.0.
 
 
-================================================================
-
-github.com/sirupsen/logrus
-https://github.com/sirupsen/logrus
-----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) 2014 Simon Eskildsen
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
 ================================================================
 
 github.com/stretchr/testify
@@ -32639,36 +32023,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-gopkg.in/h2non/filetype.v1
-https://gopkg.in/h2non/filetype.v1
-----------------------------------------------------------------
-The MIT License
-
-Copyright (c) Tomas Aparicio
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-================================================================
-
 gopkg.in/ini.v1
 https://gopkg.in/ini.v1
 ----------------------------------------------------------------

Makefile

@@ -6,7 +6,8 @@ GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
 
 VERSION ?= $(shell git describe --tags)
-TAG ?= "quay.io/minio/minio:$(VERSION)"
+REPO ?= quay.io/minio
+TAG ?= $(REPO)/minio:$(VERSION)
 
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
 GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
@@ -23,7 +24,7 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.10-0.20240227114326-6d6f813fff1b
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
@@ -46,7 +47,7 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 check: test
 test: verifiers build build-debugging ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue ./...
 
 test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
@@ -116,7 +117,6 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
-	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 	@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
 
 verify-healing-with-root-disks: ## verify healing root disks
@@ -145,12 +145,11 @@ hotfix-vars:
 	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
 	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
-	$(eval TAG := "minio/minio:$(VERSION)")
 
 hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
-	@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.0/pkger_2.2.0_linux_amd64.deb
-	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.0/linux-systemd/distributed/minio.service
-	@sudo apt install ./pkger_2.2.0_linux_amd64.deb --yes
+	@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.1/pkger_2.2.1_linux_amd64.deb
+	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.1/linux-systemd/distributed/minio.service
+	@sudo apt install ./pkger_2.2.1_linux_amd64.deb --yes
 	@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
 	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
 	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)
@@ -178,9 +177,9 @@ docker: build ## builds minio docker container
 	@docker build -q --no-cache -t $(TAG) . -f Dockerfile
 
 install-race: checks ## builds minio to $(PWD)
-	@echo "Building minio binary to './minio'"
+	@echo "Building minio binary with -race to './minio'"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
-	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
+	@echo "Installing minio binary with -race to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
 
 install: build ## builds minio and installs it to $GOPATH/bin.

README.md

@@ -123,7 +123,7 @@ You can also connect using any S3-compatible tool, such as the MinIO Client `mc`
 
 ## Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.19](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.21](https://golang.org/dl/#stable)
 
 ```sh
 go install github.com/minio/minio@latest

buildscripts/heal-manual.go

@@ -44,7 +44,6 @@ func main() {
 	opts := madmin.HealOpts{
 		Recursive: true,                  // recursively heal all objects at 'prefix'
 		Remove:    true,                  // remove content that has lost quorum and not recoverable
-		Recreate:  true,                  // rewrite all old non-inlined xl.meta to new xl.meta
 		ScanMode:  madmin.HealNormalScan, // by default do not do 'deep' scanning
 	}
 

buildscripts/rewrite-old-new.sh

@@ -87,7 +87,11 @@ function verify_rewrite() {
 		exit 1
 	fi
 
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	(
+		cd ./docs/debugging/s3-check-md5
+		go install -v
+	)
+
 	if ! s3-check-md5 \
 		-debug \
 		-versions \

buildscripts/unaligned-healing.sh

@@ -1,177 +0,0 @@
-#!/bin/bash -e
-#
-
-set -E
-set -o pipefail
-set -x
-
-if [ ! -x "$PWD/minio" ]; then
-	echo "minio executable binary not found in current directory"
-	exit 1
-fi
-
-WORK_DIR="$PWD/.verify-$RANDOM"
-MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO_OLD=("$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server)
-MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
-
-function download_old_release() {
-	if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
-		curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
-		chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
-	fi
-}
-
-function start_minio_16drive() {
-	start_port=$1
-
-	export MINIO_ROOT_USER=minio
-	export MINIO_ROOT_PASSWORD=minio123
-	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-	unset MINIO_KMS_AUTO_ENCRYPTION         # do not auto-encrypt objects
-	export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
-	export MINIO_CI_CD=1
-
-	MC_BUILD_DIR="mc-$RANDOM"
-	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-		echo "failed to download https://github.com/minio/mc"
-		purge "${MC_BUILD_DIR}"
-		exit 1
-	fi
-
-	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
-
-	# remove mc source.
-	purge "${MC_BUILD_DIR}"
-
-	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-	pid=$!
-	disown $pid
-	sleep 30
-
-	if ! ps -p ${pid} 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
-	"${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
-	"${WORK_DIR}/mc" cp \
-		"${WORK_DIR}/unaligned" \
-		minio/healing-shard-bucket/unaligned \
-		--disable-multipart --quiet
-
-	## "unaligned" object name gets consistently distributed
-	## to disks in following distribution order
-	##
-	## NOTE: if you change the name make sure to change the
-	## distribution order present here
-	##
-	## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
-
-	## make sure to remove the "last" data shard
-	rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
-	sleep 10
-	## Heal the shard
-	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-	## then remove any other data shard let's pick first disk
-	## - 1st data shard.
-	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
-	sleep 10
-
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	pkill minio
-	sleep 3
-
-	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-	pid=$!
-	disown $pid
-	sleep 30
-
-	if ! ps -p ${pid} 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		mkdir -p inspects
-		(
-			cd inspects
-			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
-		)
-
-		"${WORK_DIR}/mc" mb play/inspects
-		"${WORK_DIR}/mc" mirror inspects play/inspects
-
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		mkdir -p inspects
-		(
-			cd inspects
-			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
-		)
-
-		"${WORK_DIR}/mc" mb play/inspects
-		"${WORK_DIR}/mc" mirror inspects play/inspects
-
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	pkill minio
-	sleep 3
-}
-
-function main() {
-	download_old_release
-
-	start_port=$(shuf -i 10000-65000 -n 1)
-
-	start_minio_16drive ${start_port}
-}
-
-function purge() {
-	rm -rf "$1"
-}
-
-(main "$@")
-rv=$?
-purge "$WORK_DIR"
-exit "$rv"

buildscripts/verify-healing.sh

@@ -83,7 +83,7 @@ function start_minio_3_node() {
 }
 
 function check_online() {
-	if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
+	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
 		echo "1"
 	fi
 }
@@ -109,6 +109,7 @@ function perform_test() {
 
 	rm -rf ${WORK_DIR}/${1}/*/
 
+	set -x
 	start_minio_3_node 120 $2
 
 	rv=$(check_online)

cmd/admin-bucket-handlers.go

@@ -31,7 +31,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -680,24 +680,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			if _, ok := bucketMap[bucket]; !ok {
 				opts := MakeBucketOptions{
 					LockEnabled: config.Enabled(),
+					ForceCreate: true, // ignore if it already exists
 				}
 				err = objectAPI.MakeBucket(ctx, bucket, opts)
 				if err != nil {
-					if _, ok := err.(BucketExists); !ok {
-						rpt.SetStatus(bucket, fileName, err)
-						continue
-					}
+					rpt.SetStatus(bucket, fileName, err)
+					continue
 				}
-				v := newBucketMetadata(bucket)
+				v, _ := globalBucketMetadataSys.Get(bucket)
 				bucketMap[bucket] = &v
 			}
 
-			// Deny object locking configuration settings on existing buckets without object lock enabled.
-			if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
-
 			bucketMap[bucket].ObjectLockConfigXML = configData
 			bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
@@ -724,13 +717,13 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
-					if _, ok := err.(BucketExists); !ok {
-						rpt.SetStatus(bucket, fileName, err)
-						continue
-					}
+				if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
+					ForceCreate: true, // ignore if it already exists
+				}); err != nil {
+					rpt.SetStatus(bucket, fileName, err)
+					continue
 				}
-				v := newBucketMetadata(bucket)
+				v, _ := globalBucketMetadataSys.Get(bucket)
 				bucketMap[bucket] = &v
 			}
 
@@ -776,14 +769,14 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
-			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
+			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
+				ForceCreate: true, // ignore if it already exists
+			})
 			if err != nil {
-				if _, ok := err.(BucketExists); !ok {
-					rpt.SetStatus(bucket, "", err)
-					continue
-				}
+				rpt.SetStatus(bucket, "", err)
+				continue
 			}
-			v := newBucketMetadata(bucket)
+			v, _ := globalBucketMetadataSys.Get(bucket)
 			bucketMap[bucket] = &v
 		}
 		if _, ok := bucketMap[bucket]; !ok {

cmd/admin-handler-utils.go

@@ -23,7 +23,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"

cmd/admin-handlers-config-kv.go

@@ -58,7 +58,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -162,7 +162,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -443,7 +443,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-config.go

@@ -60,7 +60,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-ldap.go

@@ -132,7 +132,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-users.go

@@ -27,11 +27,13 @@ import (
 	"net/http"
 	"os"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
@@ -620,6 +622,11 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	if createReq.AccessKey == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
 	var (
 		targetGroups []string
 		err          error
@@ -774,25 +781,6 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(policy.Args{
-		AccountName:     cred.AccessKey,
-		Groups:          cred.Groups,
-		Action:          policy.UpdateServiceAccountAdminAction,
-		ConditionValues: getConditionValues(r, "", cred),
-		IsOwner:         owner,
-		Claims:          cred.Claims,
-	}) {
-		requestUser := cred.AccessKey
-		if cred.ParentUser != "" {
-			requestUser = cred.ParentUser
-		}
-
-		if requestUser != svcAccount.ParentUser {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-	}
-
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
@@ -813,6 +801,31 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	condValues := getConditionValues(r, "", cred)
+	addExpirationToCondValues(updateReq.NewExpiration, condValues)
+
+	// Permission checks:
+	//
+	// 1. Any type of account (i.e. access keys (previously/still called service
+	// accounts), STS accounts, internal IDP accounts, etc) with the
+	// policy.UpdateServiceAccountAdminAction permission can update any service
+	// account.
+	//
+	// 2. We would like to let a user update their own access keys, however it
+	// is currently blocked pending a re-design. Users are still able to delete
+	// and re-create them.
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.UpdateServiceAccountAdminAction,
+		ConditionValues: condValues,
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
 	var sp *policy.Policy
 	if len(updateReq.NewPolicy) > 0 {
 		sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
@@ -1060,6 +1073,10 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	if serviceAccount == siteReplicatorSvcAcc && globalSiteReplicationSys.isEnabled() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
+		return
+	}
 	// We do not care if service account is readable or not at this point,
 	// since this is a delete call we shall allow it to be deleted if possible.
 	svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
@@ -1181,26 +1198,17 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	bucketStorageCache.Once.Do(func() {
-		// Set this to 10 secs since its enough, as scanner
-		// does not update the bucket usage values frequently.
-		bucketStorageCache.TTL = 10 * time.Second
-
-		// Rely on older value if usage loading fails from disk.
-		bucketStorageCache.Relax = true
-		bucketStorageCache.Update = func() (interface{}, error) {
+	bucketStorageCache.InitOnce(10*time.Second,
+		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
+		func() (DataUsageInfo, error) {
 			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
 			defer done()
 
 			return loadDataUsageFromBackend(ctx, objectAPI)
-		}
-	})
+		},
+	)
 
-	var dataUsageInfo DataUsageInfo
-	v, _ := bucketStorageCache.Get()
-	if v != nil {
-		dataUsageInfo, _ = v.(DataUsageInfo)
-	}
+	dataUsageInfo, _ := bucketStorageCache.Get()
 
 	// If etcd, dns federation configured list buckets from etcd.
 	var err error
@@ -2414,6 +2422,13 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) {
+	if exp == nil {
+		return
+	}
+	condValues["DurationSeconds"] = []string{strconv.FormatInt(int64(exp.Sub(time.Now()).Seconds()), 10)}
+}
+
 func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
 	ctx := r.Context()
 
@@ -2469,13 +2484,16 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
 		claims:      make(map[string]interface{}),
 	}
 
+	condValues := getConditionValues(r, "", cred)
+	addExpirationToCondValues(createReq.Expiration, condValues)
+
 	// Check if action is allowed if creating access key for another user
 	// Check if action is explicitly denied if for self
 	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.CreateServiceAccountAdminAction,
-		ConditionValues: getConditionValues(r, "", cred),
+		ConditionValues: condValues,
 		IsOwner:         owner,
 		Claims:          cred.Claims,
 		DenyOnly:        (targetUser == cred.AccessKey || targetUser == cred.ParentUser),

cmd/admin-handlers-users_test.go

@@ -206,7 +206,9 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
 				suite.TestCannedPolicies(c)
 				suite.TestGroupAddRemove(c)
 				suite.TestServiceAccountOpsByAdmin(c)
+				suite.TestServiceAccountPrivilegeEscalationBug(c)
 				suite.TestServiceAccountOpsByUser(c)
+				suite.TestServiceAccountDurationSecondsCondition(c)
 				suite.TestAddServiceAccountPerms(c)
 				suite.TearDownSuite(c)
 			},
@@ -896,14 +898,6 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 	// 3. Check S3 access
 	c.assertSvcAccS3Access(ctx, s, cr, bucket)
 
-	// 4. Check that svc account can restrict the policy, and that the
-	// session policy can be updated.
-	c.assertSvcAccSessionPolicyUpdate(ctx, s, userAdmClient, accessKey, bucket)
-
-	// 4. Check that service account's secret key and account status can be
-	// updated.
-	c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, accessKey, bucket)
-
 	// 5. Check that service account can be deleted.
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
 
@@ -911,6 +905,93 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 	c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
 }
 
+func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Create policy, user and associate policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Deny",
+   "Action": [
+     "admin:CreateServiceAccount",
+     "admin:UpdateServiceAccount"
+   ],
+   "Condition": {"NumericGreaterThan": {"svc:DurationSeconds": "3600"}}
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	distantExpiration := time.Now().Add(30 * time.Minute)
+	cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: accessKey,
+		AccessKey:  "svc-accesskey",
+		SecretKey:  "svc-secretkey",
+		Expiration: &distantExpiration,
+	})
+	if err != nil {
+		c.Fatalf("Unable to create svc acc: %v", err)
+	}
+
+	c.assertSvcAccS3Access(ctx, s, cr, bucket)
+
+	closeExpiration := time.Now().Add(2 * time.Hour)
+	_, err = userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: accessKey,
+		AccessKey:  "svc-accesskey",
+		SecretKey:  "svc-secretkey",
+		Expiration: &closeExpiration,
+	})
+	if err == nil {
+		c.Fatalf("Creating a svc acc with distant expiration should fail")
+	}
+}
+
 func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
@@ -979,6 +1060,95 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
 }
 
+func (s *TestSuiteIAM) TestServiceAccountPrivilegeEscalationBug(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	err := s.client.MakeBucket(ctx, "public", minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	err = s.client.MakeBucket(ctx, "private", minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	pubPolicyBytes := []byte(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:*"
+   ],
+   "Resource": [
+    "arn:aws:s3:::public",
+    "arn:aws:s3:::public/*"
+   ]
+  }
+ ]
+}`)
+
+	fullS3PolicyBytes := []byte(`{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:*"
+      ],
+      "Resource": [
+        "arn:aws:s3:::*"
+      ]
+    }
+  ]
+}
+`)
+
+	// Create a service account for the root user.
+	cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: globalActiveCred.AccessKey,
+		Policy:     pubPolicyBytes,
+	})
+	if err != nil {
+		c.Fatalf("admin should be able to create service account for themselves %s", err)
+	}
+
+	svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+
+	// Check that the service account can access the public bucket.
+	buckets, err := svcClient.ListBuckets(ctx)
+	if err != nil {
+		c.Fatalf("err fetching buckets %s", err)
+	}
+	if len(buckets) != 1 || buckets[0].Name != "public" {
+		c.Fatalf("service account should only have access to public bucket")
+	}
+
+	// Create an madmin client with the service account creds.
+	svcAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  credentials.NewStaticV4(cr.AccessKey, cr.SecretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating svcacct admin client: %v", err)
+	}
+	svcAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Attempt to update the policy on the service account.
+	err = svcAdmClient.UpdateServiceAccount(ctx, cr.AccessKey,
+		madmin.UpdateServiceAccountReq{
+			NewPolicy: fullS3PolicyBytes,
+		})
+
+	if err == nil {
+		c.Fatalf("service account should not be able to update policy on itself")
+	} else if !strings.Contains(err.Error(), "Access Denied") {
+		c.Fatalf("unexpected error: %v", err)
+	}
+}
+
 func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()

cmd/admin-handlers.go

@@ -50,6 +50,7 @@ import (
 	"github.com/minio/madmin-go/v3/estream"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/dsync"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
@@ -79,6 +80,213 @@ const (
 	mgmtForceStop   = "forceStop"
 )
 
+// ServerUpdateV2Handler - POST /minio/admin/v3/update?updateURL={updateURL}&type=2
+// ----------
+// updates all minio servers and restarts them gracefully.
+func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	updateURL := vars["updateURL"]
+	dryRun := r.Form.Get("dry-run") == "true"
+
+	mode := getMinioMode()
+	if updateURL == "" {
+		updateURL = minioReleaseInfoURL
+		if runtime.GOOS == globalWindowsOSName {
+			updateURL = minioReleaseWindowsInfoURL
+		}
+	}
+
+	u, err := url.Parse(updateURL)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	content, err := downloadReleaseURL(u, updateTimeout, mode)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
+	// Download Binary Once
+	binC, bin, err := downloadBinary(u, mode)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	updateStatus := madmin.ServerUpdateStatusV2{DryRun: dryRun}
+	peerResults := make(map[string]madmin.ServerPeerUpdateStatus)
+
+	local := globalLocalNodeName
+	if local == "" {
+		local = "127.0.0.1"
+	}
+
+	failedClients := make(map[int]struct{})
+
+	if globalIsDistErasure {
+		// Push binary to other servers
+		for idx, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
+			if nerr.Err != nil {
+				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
+					Host:           nerr.Host.String(),
+					Err:            nerr.Err.Error(),
+					CurrentVersion: Version,
+				}
+				failedClients[idx] = struct{}{}
+			} else {
+				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
+					Host:           nerr.Host.String(),
+					CurrentVersion: Version,
+					UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
+				}
+			}
+		}
+	}
+
+	if lrTime.Sub(currentReleaseTime) > 0 {
+		if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
+			peerResults[local] = madmin.ServerPeerUpdateStatus{
+				Host:           local,
+				Err:            err.Error(),
+				CurrentVersion: Version,
+			}
+		} else {
+			peerResults[local] = madmin.ServerPeerUpdateStatus{
+				Host:           local,
+				CurrentVersion: Version,
+				UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
+			}
+		}
+	} else {
+		peerResults[local] = madmin.ServerPeerUpdateStatus{
+			Host:           local,
+			Err:            fmt.Sprintf("server is already running the latest version: %s", Version),
+			CurrentVersion: Version,
+		}
+	}
+
+	if !dryRun {
+		if globalIsDistErasure {
+			ng := WithNPeers(len(globalNotificationSys.peerClients))
+			for idx, client := range globalNotificationSys.peerClients {
+				_, ok := failedClients[idx]
+				if ok {
+					continue
+				}
+				client := client
+				ng.Go(ctx, func() error {
+					return client.CommitBinary(ctx)
+				}, idx, *client.host)
+			}
+
+			for _, nerr := range ng.Wait() {
+				if nerr.Err != nil {
+					prs, ok := peerResults[nerr.Host.String()]
+					if ok {
+						prs.Err = nerr.Err.Error()
+						peerResults[nerr.Host.String()] = prs
+					} else {
+						peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
+							Host:           nerr.Host.String(),
+							Err:            nerr.Err.Error(),
+							CurrentVersion: Version,
+							UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
+						}
+					}
+				}
+			}
+		}
+		prs := peerResults[local]
+		if prs.Err == "" {
+			if err = commitBinary(); err != nil {
+				prs.Err = err.Error()
+			}
+			peerResults[local] = prs
+		}
+	}
+
+	prs, ok := peerResults[local]
+	if ok {
+		prs.WaitingDrives = waitingDrivesNode()
+		peerResults[local] = prs
+	}
+
+	if globalIsDistErasure {
+		// Notify all other MinIO peers signal service.
+		ng := WithNPeers(len(globalNotificationSys.peerClients))
+		for idx, client := range globalNotificationSys.peerClients {
+			_, ok := failedClients[idx]
+			if ok {
+				continue
+			}
+			client := client
+			ng.Go(ctx, func() error {
+				prs, ok := peerResults[client.String()]
+				if ok && prs.CurrentVersion != prs.UpdatedVersion && prs.UpdatedVersion != "" {
+					return client.SignalService(serviceRestart, "", dryRun)
+				}
+				return nil
+			}, idx, *client.host)
+		}
+
+		for _, nerr := range ng.Wait() {
+			if nerr.Err != nil {
+				waitingDrives := map[string]madmin.DiskMetrics{}
+				jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives)
+				if jerr == nil {
+					prs, ok := peerResults[nerr.Host.String()]
+					if ok {
+						prs.WaitingDrives = waitingDrives
+						peerResults[nerr.Host.String()] = prs
+					}
+					continue
+				}
+			}
+		}
+	}
+
+	for _, pr := range peerResults {
+		updateStatus.Results = append(updateStatus.Results, pr)
+	}
+
+	// Marshal API response
+	jsonBytes, err := json.Marshal(updateStatus)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, jsonBytes)
+
+	if !dryRun {
+		if lrTime.Sub(currentReleaseTime) > 0 {
+			globalServiceSignalCh <- serviceRestart
+		}
+	}
+}
+
 // ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
@@ -90,7 +298,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if globalInplaceUpdateDisabled {
+	if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
 		// if MINIO_UPDATE=off - inplace update is disabled, mostly in containers.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
@@ -124,14 +332,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
-	crTime, err := GetCurrentReleaseTime()
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	if lrTime.Sub(crTime) <= 0 {
+	if lrTime.Sub(currentReleaseTime) <= 0 {
 		updateStatus := madmin.ServerUpdateStatus{
 			CurrentVersion: Version,
 			UpdatedVersion: Version,
@@ -148,8 +349,10 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
+
 	// Download Binary Once
-	reader, err := downloadBinary(u, mode)
+	binC, bin, err := downloadBinary(u, mode)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -157,7 +360,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Push binary to other servers
-	for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader) {
+	for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
 		if nerr.Err != nil {
 			err := AdminError{
 				Code:       AdminUpdateApplyFailure,
@@ -171,7 +374,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	err = verifyBinary(u, sha256Sum, releaseInfo, mode, reader)
+	err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin))
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -201,7 +404,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	updateStatus := madmin.ServerUpdateStatus{
 		CurrentVersion: Version,
-		UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+		UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 	}
 
 	// Marshal API response
@@ -248,7 +451,7 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	case madmin.ServiceActionUnfreeze:
 		serviceSig = serviceUnFreeze
 	default:
-		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.Application)
+		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 		return
 	}
@@ -287,6 +490,131 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+type servicePeerResult struct {
+	Host          string                        `json:"host"`
+	Err           string                        `json:"err,omitempty"`
+	WaitingDrives map[string]madmin.DiskMetrics `json:"waitingDrives,omitempty"`
+}
+
+type serviceResult struct {
+	Action  madmin.ServiceAction `json:"action"`
+	DryRun  bool                 `json:"dryRun"`
+	Results []servicePeerResult  `json:"results,omitempty"`
+}
+
+// ServiceV2Handler - POST /minio/admin/v3/service?action={action}&type=2
+// ----------
+// Supports following actions:
+// - restart (restarts all the MinIO instances in a setup)
+// - stop (stops all the MinIO instances in a setup)
+// - freeze (freezes all incoming S3 API calls)
+// - unfreeze (unfreezes previously frozen S3 API calls)
+//
+// This newer API now returns back status per remote peer and local regarding
+// if a "restart/stop" was successful or not. Service signal now supports
+// a dry-run that helps skip the nodes that may have hung drives. By default
+// restart/stop will ignore the servers that are hung on drives. You can use
+// 'force' param to force restart even with hung drives if needed.
+func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	action := vars["action"]
+	dryRun := r.Form.Get("dry-run") == "true"
+
+	var serviceSig serviceSignal
+	act := madmin.ServiceAction(action)
+	switch act {
+	case madmin.ServiceActionRestart:
+		serviceSig = serviceRestart
+	case madmin.ServiceActionStop:
+		serviceSig = serviceStop
+	case madmin.ServiceActionFreeze:
+		serviceSig = serviceFreeze
+	case madmin.ServiceActionUnfreeze:
+		serviceSig = serviceUnFreeze
+	default:
+		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
+		return
+	}
+
+	var objectAPI ObjectLayer
+	switch serviceSig {
+	case serviceRestart:
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
+	case serviceStop:
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
+	case serviceFreeze, serviceUnFreeze:
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
+	}
+	if objectAPI == nil {
+		return
+	}
+
+	// Notify all other MinIO peers signal service.
+	srvResult := serviceResult{Action: act, Results: []servicePeerResult{}}
+
+	process := act == madmin.ServiceActionRestart || act == madmin.ServiceActionStop
+	if process {
+		localhost := globalLocalNodeName
+		if globalLocalNodeName == "" {
+			localhost = "127.0.0.1"
+		}
+		waitingDrives := waitingDrivesNode()
+		srvResult.Results = append(srvResult.Results, servicePeerResult{
+			Host:          localhost,
+			WaitingDrives: waitingDrives,
+		})
+	}
+
+	if globalIsDistErasure {
+		for _, nerr := range globalNotificationSys.SignalServiceV2(serviceSig, dryRun) {
+			if nerr.Err != nil && process {
+				waitingDrives := map[string]madmin.DiskMetrics{}
+				jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives)
+				if jerr == nil {
+					srvResult.Results = append(srvResult.Results, servicePeerResult{
+						Host:          nerr.Host.String(),
+						WaitingDrives: waitingDrives,
+					})
+					continue
+				}
+			}
+			errStr := ""
+			if nerr.Err != nil {
+				errStr = nerr.Err.Error()
+			}
+			srvResult.Results = append(srvResult.Results, servicePeerResult{
+				Host: nerr.Host.String(),
+				Err:  errStr,
+			})
+		}
+	}
+
+	srvResult.DryRun = dryRun
+
+	buf, err := json.Marshal(srvResult)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Reply to the client before restarting, stopping MinIO server.
+	writeSuccessResponseJSON(w, buf)
+
+	switch serviceSig {
+	case serviceFreeze:
+		freezeServices()
+	case serviceUnFreeze:
+		unfreezeServices()
+	case serviceRestart, serviceStop:
+		if !dryRun {
+			globalServiceSignalCh <- serviceSig
+		}
+	}
+}
+
 // ServerProperties holds some server information such as, version, region
 // uptime, etc..
 type ServerProperties struct {
@@ -477,7 +805,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// DataUsageInfoHandler - GET /minio/admin/v3/datausage
+// DataUsageInfoHandler - GET /minio/admin/v3/datausage?capacity={true}
 // ----------
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -500,6 +828,16 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	// Get capacity info when asked.
+	if r.Form.Get("capacity") == "true" {
+		sinfo := objectAPI.StorageInfo(ctx, false)
+		dataUsageInfo.TotalCapacity = GetTotalUsableCapacity(sinfo.Disks, sinfo)
+		dataUsageInfo.TotalFreeCapacity = GetTotalUsableCapacityFree(sinfo.Disks, sinfo)
+		if dataUsageInfo.TotalCapacity > dataUsageInfo.TotalFreeCapacity {
+			dataUsageInfo.TotalUsedCapacity = dataUsageInfo.TotalCapacity - dataUsageInfo.TotalFreeCapacity
+		}
+	}
+
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
@@ -854,7 +1192,7 @@ type healInitParams struct {
 }
 
 // extractHealInitParams - Validates params for heal init API.
-func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) {
+func extractHealInitParams(vars map[string]string, qParams url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) {
 	hip.bucket = vars[mgmtBucket]
 	hip.objPrefix = vars[mgmtPrefix]
 
@@ -875,13 +1213,13 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 		return
 	}
 
-	if len(qParms[mgmtClientToken]) > 0 {
-		hip.clientToken = qParms[mgmtClientToken][0]
+	if len(qParams[mgmtClientToken]) > 0 {
+		hip.clientToken = qParams[mgmtClientToken][0]
 	}
-	if _, ok := qParms[mgmtForceStart]; ok {
+	if _, ok := qParams[mgmtForceStart]; ok {
 		hip.forceStart = true
 	}
-	if _, ok := qParms[mgmtForceStop]; ok {
+	if _, ok := qParams[mgmtForceStop]; ok {
 		hip.forceStop = true
 	}
 
@@ -901,7 +1239,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 	if hip.clientToken == "" {
 		jerr := json.NewDecoder(r).Decode(&hip.hs)
 		if jerr != nil {
-			logger.LogIf(GlobalContext, jerr, logger.Application)
+			logger.LogIf(GlobalContext, jerr, logger.ErrorKind)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -1314,7 +1652,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		duration = time.Second * 10
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx, true)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 	if !sufficientCapacity {
@@ -1591,16 +1929,13 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	setEventStreamHeaders(w)
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
-	// Use buffered channel to take care of burst sends or slow w.Write()
-
-	// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
-	traceCh := make(chan madmin.TraceInfo, 100000)
-
+	// Keep 100k buffered channel.
+	// If receiver cannot keep up with that we drop events.
+	traceCh := make(chan []byte, 100000)
 	peers, _ := newPeerRestClients(globalEndpoints)
-
-	err = globalTrace.Subscribe(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
+	err = globalTrace.SubscribeJSON(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
 		return shouldTrace(entry, traceOpts)
-	})
+	}, nil)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1615,19 +1950,19 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		if peer == nil {
 			continue
 		}
-		peer.Trace(traceCh, ctx.Done(), traceOpts)
+		peer.Trace(ctx, traceCh, traceOpts)
 	}
 
 	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
-	enc := json.NewEncoder(w)
 	for {
 		select {
 		case entry := <-traceCh:
-			if err := enc.Encode(entry); err != nil {
+			if _, err := w.Write(entry); err != nil {
 				return
 			}
+			grid.PutByteBuffer(entry)
 			if len(traceCh) == 0 {
 				// Flush if nothing is queued
 				w.(http.Flusher).Flush()
@@ -1649,7 +1984,6 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 // The ConsoleLogHandler handler sends console logs to the connected HTTP client.
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
-
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
@@ -1674,44 +2008,65 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	setEventStreamHeaders(w)
 
-	logCh := make(chan log.Info, 4000)
-
+	logCh := make(chan log.Info, 1000)
 	peers, _ := newPeerRestClients(globalEndpoints)
-
+	encodedCh := make(chan []byte, 1000+len(peers)*1000)
 	err = globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+	// Convert local entries to JSON
+	go func() {
+		var buf bytes.Buffer
+		enc := json.NewEncoder(&buf)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case li := <-logCh:
+				if !li.SendLog(node, logKind) {
+					continue
+				}
+				buf.Reset()
+				if err := enc.Encode(li); err != nil {
+					continue
+				}
+				select {
+				case <-ctx.Done():
+					return
+				case encodedCh <- append(grid.GetByteBuffer()[:0], buf.Bytes()...):
+				}
+			}
+		}
+	}()
 
+	// Collect from matching peers
 	for _, peer := range peers {
 		if peer == nil {
 			continue
 		}
 		if node == "" || strings.EqualFold(peer.host.Name, node) {
-			peer.ConsoleLog(logCh, ctx.Done())
+			peer.ConsoleLog(ctx, logKind, encodedCh)
 		}
 	}
 
-	enc := json.NewEncoder(w)
-
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
-
 	for {
 		select {
-		case log, ok := <-logCh:
+		case log, ok := <-encodedCh:
 			if !ok {
 				return
 			}
-			if log.SendLog(node, logKind) {
-				if err := enc.Encode(log); err != nil {
-					return
-				}
-				if len(logCh) == 0 {
-					// Flush if nothing is queued
-					w.(http.Flusher).Flush()
-				}
+			_, err = w.Write(log)
+			if err != nil {
+				return
+			}
+			grid.PutByteBuffer(log)
+			if len(logCh) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
 			}
 		case <-keepAliveTicker.C:
 			if len(logCh) > 0 {
@@ -1866,7 +2221,10 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 		return nil, errServerNotInitialized
 	}
 
-	z, _ := objectAPI.(*erasureServerPools)
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		return nil, errServerNotInitialized
+	}
 
 	poolsInfo := make(map[int]map[int]madmin.ErasureSetInfo)
 	for _, d := range allDisks {
@@ -1897,7 +2255,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 	return poolsInfo, nil
 }
 
-func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) madmin.InfoMessage {
+func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) madmin.InfoMessage {
 	ldap := madmin.LDAP{}
 	if globalIAMSys.LDAPConfig.Enabled() {
 		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
@@ -1918,12 +2276,10 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
 
-	local := getLocalServerProperty(globalEndpoints, r)
-	servers := globalNotificationSys.ServerInfo()
+	local := getLocalServerProperty(globalEndpoints, r, metrics)
+	servers := globalNotificationSys.ServerInfo(metrics)
 	servers = append(servers, local)
 
-	assignPoolNumbers(servers)
-
 	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
 	var backend madmin.ErasureBackend
 
@@ -1973,7 +2329,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 			DrivesPerSet:     backendInfo.DrivesPerSet,
 		}
 
-		if poolsInfoEnabled {
+		if pools {
 			poolsInfo, _ = getPoolsInfo(ctx, allDisks)
 		}
 	}
@@ -2019,7 +2375,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 	}
 
 	client := &http.Client{
-		Transport: NewHTTPTransport(),
+		Transport: globalHealthChkTransport,
 		Timeout:   10 * time.Second,
 	}
 
@@ -2339,7 +2695,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 	}
 
 	go func() {
-		defer close(healthInfoCh)
+		defer xioutil.SafeClose(healthInfoCh)
 
 		partialWrite(healthInfo) // Write first message with only version and deployment id populated
 		getAndWritePlatformInfo()
@@ -2355,19 +2711,25 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		getAndWriteSysConfig()
 
 		if query.Get("minioinfo") == "true" {
-			infoMessage := getServerInfo(healthCtx, false, nil)
+			infoMessage := getServerInfo(healthCtx, false, true, nil)
 			servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
 				servers = append(servers, madmin.ServerInfo{
-					State:      server.State,
-					Endpoint:   anonEndpoint,
-					Uptime:     server.Uptime,
-					Version:    server.Version,
-					CommitID:   server.CommitID,
-					Network:    anonymizeNetwork(server.Network),
-					Drives:     anonymizeDrives(server.Disks),
-					PoolNumber: server.PoolNumber,
+					State:    server.State,
+					Endpoint: anonEndpoint,
+					Uptime:   server.Uptime,
+					Version:  server.Version,
+					CommitID: server.CommitID,
+					Network:  anonymizeNetwork(server.Network),
+					Drives:   anonymizeDrives(server.Disks),
+					PoolNumber: func() int {
+						if len(server.PoolNumbers) == 1 {
+							return server.PoolNumbers[0]
+						}
+						return math.MaxInt // this indicates that its unset.
+					}(),
+					PoolNumbers: server.PoolNumbers,
 					MemStats: madmin.MemStats{
 						Alloc:      server.MemStats.Alloc,
 						TotalAlloc: server.MemStats.TotalAlloc,
@@ -2543,8 +2905,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	metrics := r.Form.Get(peerRESTMetrics) == "true"
+
 	// Marshal API response
-	jsonBytes, err := json.Marshal(getServerInfo(ctx, true, r))
+	jsonBytes, err := json.Marshal(getServerInfo(ctx, true, metrics, r))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -2555,22 +2919,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-func assignPoolNumbers(servers []madmin.ServerProperties) {
-	for i := range servers {
-		for idx, ge := range globalEndpoints {
-			for _, endpoint := range ge.Endpoints {
-				if servers[i].Endpoint == endpoint.Host {
-					servers[i].PoolNumber = idx + 1
-				} else if host, err := xnet.ParseHost(servers[i].Endpoint); err == nil {
-					if host.Name == endpoint.Hostname() {
-						servers[i].PoolNumber = idx + 1
-					}
-				}
-			}
-		}
-	}
-}
-
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
@@ -2728,14 +3076,14 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 	resultCh := make(chan madmin.ClusterRegistrationInfo)
 
 	go func() {
-		defer close(resultCh)
+		defer xioutil.SafeClose(resultCh)
 
 		ci := madmin.ClusterRegistrationInfo{}
 		ci.Info.NoOfServerPools = len(globalEndpoints)
-		ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
+		ci.Info.NoOfServers = totalNodeCount()
 		ci.Info.MinioVersion = Version
 
-		si := objectAPI.StorageInfo(ctx, true)
+		si := objectAPI.StorageInfo(ctx, false)
 
 		ci.Info.NoOfDrives = len(si.Disks)
 		for _, disk := range si.Disks {
@@ -2972,7 +3320,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	// save the format.json as part of inspect by default
-	if volume != minioMetaBucket && file != formatConfigFile {
+	if !(volume == minioMetaBucket && file == formatConfigFile) {
 		err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
 	}
 	if !errors.Is(err, errFileNotFound) {
@@ -3054,7 +3402,7 @@ func createHostAnonymizerForFSMode() map[string]string {
 // anonymizeHost - Add entries related to given endpoint in the host anonymizer map
 // The health report data can contain the hostname in various forms e.g. host, host:port,
 // host:port/drivepath, full url (http://host:port/drivepath)
-// The anonymizer map will have mappings for all these varients for efficiently replacing
+// The anonymizer map will have mappings for all these variants for efficiently replacing
 // any of these strings to the anonymized versions at the time of health report generation.
 func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum int, srvrNum int) {
 	if len(endpoint.Host) == 0 {
@@ -3102,7 +3450,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum
 	}
 }
 
-// createHostAnonymizer - Creats a map of various strings to corresponding anonymized names
+// createHostAnonymizer - Creates a map of various strings to corresponding anonymized names
 func createHostAnonymizer() map[string]string {
 	if !globalIsDistErasure {
 		return createHostAnonymizerForFSMode()

cmd/admin-handlers_test.go

@@ -168,6 +168,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
 	queryVal := url.Values{}
 	queryVal.Set("action", string(cmd.toServiceAction()))
+	queryVal.Set("type", "2")
 	resource := adminPathPrefix + adminAPIVersionPrefix + "/service?" + queryVal.Encode()
 	req, err := newTestRequest(http.MethodPost, resource, 0, nil)
 	if err != nil {
@@ -220,12 +221,18 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	rec := httptest.NewRecorder()
 	adminTestBed.router.ServeHTTP(rec, req)
 
+	resp, _ := io.ReadAll(rec.Body)
 	if rec.Code != http.StatusOK {
-		resp, _ := io.ReadAll(rec.Body)
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
 			http.StatusOK, rec.Code, string(resp))
 	}
 
+	result := &serviceResult{}
+	if err := json.Unmarshal(resp, result); err != nil {
+		t.Error(err)
+	}
+	_ = result
+
 	// Wait until testServiceSignalReceiver() called in a goroutine quits.
 	wg.Wait()
 }
@@ -341,7 +348,7 @@ func TestExtractHealInitParams(t *testing.T) {
 		}
 		return v
 	}
-	qParmsArr := []url.Values{
+	qParamsArr := []url.Values{
 		// Invalid cases
 		mkParams("", true, true),
 		mkParams("111", true, true),
@@ -366,9 +373,9 @@ func TestExtractHealInitParams(t *testing.T) {
 	body := `{"recursive": false, "dryRun": true, "remove": false, "scanMode": 0}`
 
 	// Test all combinations!
-	for pIdx, parms := range qParmsArr {
+	for pIdx, params := range qParamsArr {
 		for vIdx, vars := range varsArr {
-			_, err := extractHealInitParams(vars, parms, bytes.NewReader([]byte(body)))
+			_, err := extractHealInitParams(vars, params, bytes.NewReader([]byte(body)))
 			isErrCase := false
 			if pIdx < 4 || vIdx < 1 {
 				isErrCase = true

cmd/admin-heal-ops.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -132,7 +133,13 @@ func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
 func (ahs *allHealState) updateHealStatus(tracker *healingTracker) {
 	ahs.Lock()
 	defer ahs.Unlock()
-	ahs.healStatus[tracker.ID] = *tracker
+
+	tracker.mu.RLock()
+	t := *tracker
+	t.QueuedBuckets = append(make([]string, 0, len(tracker.QueuedBuckets)), tracker.QueuedBuckets...)
+	t.HealedBuckets = append(make([]string, 0, len(tracker.HealedBuckets)), tracker.HealedBuckets...)
+	ahs.healStatus[tracker.ID] = t
+	tracker.mu.RUnlock()
 }
 
 // Sort by zone, set and disk index
@@ -706,7 +713,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		select {
 		case globalBackgroundHealRoutine.tasks <- task:
 			if serverDebugLog {
-				logger.Info("Task in the queue: %#v", task)
+				fmt.Printf("Task in the queue: %#v\n", task)
 			}
 		default:
 			// task queue is full, no more workers, we shall move on and heal later.
@@ -723,7 +730,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	select {
 	case globalBackgroundHealRoutine.tasks <- task:
 		if serverDebugLog {
-			logger.Info("Task in the queue: %#v", task)
+			fmt.Printf("Task in the queue: %#v\n", task)
 		}
 	case <-h.ctx.Done():
 		return nil
@@ -800,7 +807,7 @@ func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
 func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
 	bucketsOnly := false // Heals buckets and objects also.
 	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
-	close(h.traverseAndHealDoneCh)
+	xioutil.SafeClose(h.traverseAndHealDoneCh)
 }
 
 // healMinioSysMeta - heals all files under a given meta prefix, returns a function

cmd/admin-router.go

@@ -19,9 +19,6 @@ package cmd
 
 import (
 	"net/http"
-	"reflect"
-	"runtime"
-	"strings"
 
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
@@ -63,20 +60,12 @@ const (
 	noObjLayerFlag
 )
 
-// Has checks if the the given flag is enabled in `h`.
+// Has checks if the given flag is enabled in `h`.
 func (h hFlag) Has(flag hFlag) bool {
 	// Use bitwise-AND and check if the result is non-zero.
 	return h&flag != 0
 }
 
-func getHandlerName(f http.HandlerFunc) string {
-	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
-	name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.adminAPIHandlers.")
-	name = strings.TrimSuffix(name, "Handler-fm")
-	name = strings.TrimSuffix(name, "-fm")
-	return name
-}
-
 // adminMiddleware performs some common admin handler functionality for all
 // handlers:
 //
@@ -86,9 +75,13 @@ func getHandlerName(f http.HandlerFunc) string {
 //
 // - sets up call to send AuditLog
 //
-// Note that, while this is a middleware function (i.e. it takes a handler
-// function and returns one), due to flags being passed based on required
-// conditions, it is done per-"handler function registration" in the router.
+// While this is a middleware function (i.e. it takes a handler function and
+// returns one), due to flags being passed based on required conditions, it is
+// done per-"handler function registration" in the router.
+//
+// The passed in handler function must be a method of `adminAPIHandlers` for the
+// name displayed in logs and trace to be accurate. The name is extracted via
+// reflection.
 //
 // When no flags are passed, gzip compression, http tracing of headers and
 // checking of object layer availability are all enabled. Use flags to modify
@@ -100,10 +93,8 @@ func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc {
 		handlerFlags |= flag
 	}
 
-	// Get name of the handler using reflection. NOTE: The passed in handler
-	// function must be a method of `adminAPIHandlers` for this extraction to
-	// work as expected.
-	handlerName := getHandlerName(f)
+	// Get name of the handler using reflection.
+	handlerName := getHandlerName(f, "adminAPIHandlers")
 
 	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
 		// Update request context with `logger.ReqInfo`.
@@ -154,9 +145,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 	}
 
 	for _, adminVersion := range adminVersions {
-		// Restart and stop MinIO service.
+		// Restart and stop MinIO service type=2
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceV2Handler, traceAllFlag)).Queries("action", "{action:.*}", "type", "2")
+
+		// Deprecated: Restart and stop MinIO service.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceHandler, traceAllFlag)).Queries("action", "{action:.*}")
-		// Update MinIO servers.
+
+		// Update all MinIO servers type=2
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateV2Handler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}", "type", "2")
+
+		// Deprecated: Update MinIO servers.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateHandler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}")
 
 		// Info operations

cmd/admin-server-info.go

@@ -19,10 +19,12 @@ package cmd
 
 import (
 	"context"
+	"math"
 	"net/http"
 	"os"
 	"runtime"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"time"
 
@@ -34,7 +36,7 @@ import (
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request, metrics bool) madmin.ServerProperties {
 	addr := globalLocalNodeName
 	if r != nil {
 		addr = r.Host
@@ -42,9 +44,13 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	if globalIsDistErasure {
 		addr = globalLocalNodeName
 	}
+	poolNumbers := make(map[int]struct{})
 	network := make(map[string]string)
 	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				poolNumbers[endpoint.PoolIdx+1] = struct{}{}
+			}
 			nodeName := endpoint.Host
 			if nodeName == "" {
 				nodeName = addr
@@ -112,6 +118,17 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		MinioEnvVars: make(map[string]string, 10),
 	}
 
+	for poolNumber := range poolNumbers {
+		props.PoolNumbers = append(props.PoolNumbers, poolNumber)
+	}
+	sort.Ints(props.PoolNumbers)
+	props.PoolNumber = func() int {
+		if len(props.PoolNumbers) == 1 {
+			return props.PoolNumbers[0]
+		}
+		return math.MaxInt // this indicates that its unset.
+	}()
+
 	sensitive := map[string]struct{}{
 		config.EnvAccessKey:         {},
 		config.EnvSecretKey:         {},
@@ -141,7 +158,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 
 	objLayer := newObjectLayerFn()
 	if objLayer != nil {
-		storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
+		storageInfo := objLayer.LocalStorageInfo(GlobalContext, metrics)
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {

cmd/api-errors.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 
@@ -297,7 +298,6 @@ const (
 	ErrAdminConfigLDAPValidation
 	ErrAdminConfigIDPCfgNameAlreadyExists
 	ErrAdminConfigIDPCfgNameDoesNotExist
-	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
 
@@ -390,7 +390,7 @@ const (
 	ErrParseExpectedIdentForGroupName
 	ErrParseExpectedIdentForAlias
 	ErrParseUnsupportedCallWithStar
-	ErrParseNonUnaryAgregateFunctionCall
+	ErrParseNonUnaryAggregateFunctionCall
 	ErrParseMalformedJoin
 	ErrParseExpectedIdentForAt
 	ErrParseAsteriskIsNotAloneInSelectList
@@ -433,6 +433,9 @@ const (
 	// New Codes for GetObjectAttributes and GetObjectVersionAttributes
 	ErrInvalidAttributeName
 
+	ErrAdminNoAccessKey
+	ErrAdminNoSecretKey
+
 	apiErrCodeEnd // This is used only for the testing code
 )
 
@@ -959,7 +962,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationBandwidthLimitError: {
 		Code:           "XMinioAdminReplicationBandwidthLimitError",
-		Description:    "Bandwidth limit for remote target must be atleast 100MBps",
+		Description:    "Bandwidth limit for remote target must be at least 100MBps",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationNoExistingObjects: {
@@ -1359,6 +1362,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The secret key is invalid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoAccessKey: {
+		Code:           "XMinioAdminNoAccessKey",
+		Description:    "No access key was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminNoSecretKey: {
+		Code:           "XMinioAdminNoSecretKey",
+		Description:    "No secret key was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigNoQuorum: {
 		Code:           "XMinioAdminConfigNoQuorum",
 		Description:    "Configuration update failed because server quorum was not met",
@@ -1425,11 +1438,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Unable to perform the requested operation because profiling is not enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrAdminCredentialsMismatch: {
-		Code:           "XMinioAdminCredentialsMismatch",
-		Description:    "Credentials in config mismatch with server environment variables",
-		HTTPStatusCode: http.StatusServiceUnavailable,
-	},
 	ErrAdminBucketQuotaExceeded: {
 		Code:           "XMinioAdminBucketQuotaExceeded",
 		Description:    "Bucket quota exceeded",
@@ -1891,8 +1899,8 @@ var errorCodes = errorCodeMap{
 		Description:    "Only COUNT with (*) as a parameter is supported in the SQL expression.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrParseNonUnaryAgregateFunctionCall: {
-		Code:           "ParseNonUnaryAgregateFunctionCall",
+	ErrParseNonUnaryAggregateFunctionCall: {
+		Code:           "ParseNonUnaryAggregateFunctionCall",
 		Description:    "Only one argument is supported for aggregate functions in the SQL expression.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
@@ -2013,7 +2021,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAddUserInvalidArgument: {
 		Code:           "XMinioInvalidIAMCredentials",
-		Description:    "User is not allowed to be same as admin access key",
+		Description:    "Credential is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrAdminResourceInvalidArgument: {
@@ -2082,6 +2090,11 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Errors that are generated by net.Conn and any context errors must be handled here.
+	if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) {
+		return ErrRequestTimedout
+	}
+
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
 	// This way downstream context.Canceled will still report ErrRequestTimedout
 	if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
@@ -2124,6 +2137,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	case auth.ErrNoAccessKeyWithSecretKey:
+		apiErr = ErrAdminNoAccessKey
+	case auth.ErrNoSecretKeyWithAccessKey:
+		apiErr = ErrAdminNoSecretKey
 	case errInvalidStorageClass:
 		apiErr = ErrInvalidStorageClass
 	case errErasureReadQuorum:
@@ -2351,31 +2368,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case dns.ErrBucketConflict:
 		apiErr = ErrBucketAlreadyExists
 	default:
-		if _, ok := err.(tags.Error); ok {
-			// tag errors are not exported, so we check their custom interface to avoid logging.
-			// The correct type is inserted by toAPIError.
-			apiErr = ErrInternalError
-			break
-		}
-		var ie, iw int
-		// This work-around is to handle the issue golang/go#30648
-		//nolint:gocritic
-		if _, ferr := fmt.Fscanf(strings.NewReader(err.Error()),
-			"request declared a Content-Length of %d but only wrote %d bytes",
-			&ie, &iw); ferr != nil {
-			apiErr = ErrInternalError
-			// Make sure to log the errors which we cannot translate
-			// to a meaningful S3 API errors. This is added to aid in
-			// debugging unexpected/unhandled errors.
-			logger.LogIf(ctx, err)
-		} else if ie > iw {
+		if strings.Contains(err.Error(), "request declared a Content-Length") {
 			apiErr = ErrIncompleteBody
 		} else {
 			apiErr = ErrInternalError
-			// Make sure to log the errors which we cannot translate
-			// to a meaningful S3 API errors. This is added to aid in
-			// debugging unexpected/unhandled errors.
-			logger.LogIf(ctx, err)
 		}
 	}
 
@@ -2519,6 +2515,13 @@ func toAPIError(ctx context.Context, err error) APIError {
 		}
 	}
 
+	if apiErr.Code == "InternalError" {
+		// Make sure to log the errors which we cannot translate
+		// to a meaningful S3 API errors. This is added to aid in
+		// debugging unexpected/unhandled errors.
+		logger.LogIf(ctx, err)
+	}
+
 	return apiErr
 }
 

cmd/api-headers.go

@@ -50,7 +50,7 @@ func setEventStreamHeaders(w http.ResponseWriter) {
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	// Set the "Server" http header.
-	w.Header().Set(xhttp.ServerInfo, "MinIO")
+	w.Header().Set(xhttp.ServerInfo, MinioStoreName)
 
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.

cmd/api-response.go

@@ -891,7 +891,7 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	}
 	// Similar check to http.checkWriteHeaderCode
 	if statusCode < 100 || statusCode > 999 {
-		logger.Error(fmt.Sprintf("invalid WriteHeader code %v", statusCode))
+		logger.LogIf(context.Background(), fmt.Errorf("invalid WriteHeader code %v", statusCode))
 		statusCode = http.StatusInternalServerError
 	}
 	setCommonHeaders(w)
@@ -944,13 +944,15 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
-// writeErrorRespone writes error headers
+// writeErrorResponse writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
-	switch err.Code {
-	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
+	if err.HTTPStatusCode == http.StatusServiceUnavailable {
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "120")
+	}
+
+	switch err.Code {
 	case "InvalidRegion":
 		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region)
 	case "AuthorizationHeaderMalformed":
@@ -959,7 +961,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Similar check to http.checkWriteHeaderCode
 	if err.HTTPStatusCode < 100 || err.HTTPStatusCode > 999 {
-		logger.Error(fmt.Sprintf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
+		logger.LogIf(ctx, fmt.Errorf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
 		err.HTTPStatusCode = http.StatusInternalServerError
 	}
 

cmd/api-router.go

@@ -18,14 +18,11 @@
 package cmd
 
 import (
-	"compress/gzip"
 	"net"
 	"net/http"
 
-	"github.com/klauspost/compress/gzhttp"
 	consoleapi "github.com/minio/console/api"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/wildcard"
 	"github.com/rs/cors"
@@ -171,6 +168,87 @@ var rejectedBucketAPIs = []rejectedAPI{
 	},
 }
 
+// Set of s3 handler options as bit flags.
+type s3HFlag uint8
+
+const (
+	// when provided, disables Gzip compression.
+	noGZS3HFlag = 1 << iota
+
+	// when provided, enables only tracing of headers. Otherwise, both headers
+	// and body are traced.
+	traceHdrsS3HFlag
+
+	// when provided, disables throttling via the `maxClients` middleware.
+	noThrottleS3HFlag
+)
+
+func (h s3HFlag) has(flag s3HFlag) bool {
+	// Use bitwise-AND and check if the result is non-zero.
+	return h&flag != 0
+}
+
+// s3APIMiddleware - performs some common handler functionality for S3 API
+// handlers.
+//
+// It is set per-"handler function registration" in the router to allow for
+// behavior modification via flags.
+//
+// This middleware always calls `collectAPIStats` to collect API stats.
+//
+// The passed in handler function must be a method of `objectAPIHandlers` for
+// the name displayed in logs and trace to be accurate. The name is extracted
+// via reflection.
+//
+// When **no** flags are passed, the behavior is to trace both headers and body,
+// gzip the response and throttle the handler via `maxClients`. Each of these
+// can be disabled via the corresponding `s3HFlag`.
+//
+// CAUTION: for requests involving large req/resp bodies ensure to pass the
+// `traceHdrsS3HFlag`, otherwise both headers and body will be traced, causing
+// high memory usage!
+func s3APIMiddleware(f http.HandlerFunc, flags ...s3HFlag) http.HandlerFunc {
+	// Collect all flags with bitwise-OR and assign operator
+	var handlerFlags s3HFlag
+	for _, flag := range flags {
+		handlerFlags |= flag
+	}
+
+	// Get name of the handler using reflection.
+	handlerName := getHandlerName(f, "objectAPIHandlers")
+
+	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		// Wrap the actual handler with the appropriate tracing middleware.
+		var tracedHandler http.HandlerFunc
+		if handlerFlags.has(traceHdrsS3HFlag) {
+			tracedHandler = httpTraceHdrs(f)
+		} else {
+			tracedHandler = httpTraceAll(f)
+		}
+
+		// Skip wrapping with the gzip middleware if specified.
+		var gzippedHandler http.HandlerFunc = tracedHandler
+		if !handlerFlags.has(noGZS3HFlag) {
+			gzippedHandler = gzipHandler(gzippedHandler)
+		}
+
+		// Skip wrapping with throttling middleware if specified.
+		var throttledHandler http.HandlerFunc = gzippedHandler
+		if !handlerFlags.has(noThrottleS3HFlag) {
+			throttledHandler = maxClients(throttledHandler)
+		}
+
+		// Collect API stats using the API name got from reflection in
+		// `getHandlerName`.
+		statsCollectedHandler := collectAPIStats(handlerName, throttledHandler)
+
+		// Call the final handler.
+		statsCollectedHandler(w, r)
+	}
+
+	return handler
+}
+
 // registerAPIRouter - registers S3 compatible APIs.
 func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
@@ -208,12 +286,6 @@ func registerAPIRouter(router *mux.Router) {
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
-	if err != nil {
-		// Static params, so this is very unlikely.
-		logger.Fatal(err, "Unable to initialize server")
-	}
-
 	for _, router := range routers {
 		// Register all rejected object APIs
 		for _, r := range rejectedObjAPIs {
@@ -225,240 +297,307 @@ func registerAPIRouter(router *mux.Router) {
 
 		// Object operations
 		// HeadObject
-		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
-		// GetObjectAttribytes
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "")
+		router.Methods(http.MethodHead).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.HeadObjectHandler))
+
+		// GetObjectAttributes
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectAttributesHandler, traceHdrsS3HFlag)).
+			Queries("attributes", "")
+
 		// CopyObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
+			HandlerFunc(s3APIMiddleware(api.CopyObjectPartHandler)).
 			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectPartHandler, traceHdrsS3HFlag)).
+			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.ListObjectPartsHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("completemultipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.CompleteMultipartUploadHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("newmultipartupload", maxClients(gz(httpTraceAll(api.NewMultipartUploadHandler))))).Queries("uploads", "")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.NewMultipartUploadHandler)).
+			Queries("uploads", "")
 		// AbortMultipartUpload
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("abortmultipartupload", maxClients(gz(httpTraceAll(api.AbortMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.AbortMultipartUploadHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectacl", maxClients(gz(httpTraceHdrs(api.GetObjectACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectACLHandler, traceHdrsS3HFlag)).
+			Queries("acl", "")
 		// PutObjectACL - this is a dummy call.
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectacl", maxClients(gz(httpTraceHdrs(api.PutObjectACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectACLHandler, traceHdrsS3HFlag)).
+			Queries("acl", "")
 		// GetObjectTagging
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjecttagging", maxClients(gz(httpTraceHdrs(api.GetObjectTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// PutObjectTagging
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjecttagging", maxClients(gz(httpTraceHdrs(api.PutObjectTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// DeleteObjectTagging
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("deleteobjecttagging", maxClients(gz(httpTraceHdrs(api.DeleteObjectTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.DeleteObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// SelectObjectContent
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("selectobjectcontent", maxClients(gz(httpTraceHdrs(api.SelectObjectContentHandler))))).Queries("select", "").Queries("select-type", "2")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.SelectObjectContentHandler, traceHdrsS3HFlag)).
+			Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectretention", maxClients(gz(httpTraceAll(api.GetObjectRetentionHandler))))).Queries("retention", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectRetentionHandler)).
+			Queries("retention", "")
 		// GetObjectLegalHold
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectLegalHoldHandler)).
+			Queries("legal-hold", "")
 		// GetObject with lambda ARNs
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectLambdaHandler, traceHdrsS3HFlag)).
+			Queries("lambdaArn", "{lambdaArn:.*}")
 		// GetObject
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectHandler, traceHdrsS3HFlag))
 		// CopyObject
-		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
-			collectAPIStats("copyobject", maxClients(gz(httpTraceAll(api.CopyObjectHandler)))))
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(s3APIMiddleware(api.CopyObjectHandler))
 		// PutObjectRetention
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectretention", maxClients(gz(httpTraceAll(api.PutObjectRetentionHandler))))).Queries("retention", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectRetentionHandler)).
+			Queries("retention", "")
 		// PutObjectLegalHold
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectlegalhold", maxClients(gz(httpTraceAll(api.PutObjectLegalHoldHandler))))).Queries("legal-hold", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectLegalHoldHandler)).
+			Queries("legal-hold", "")
 
 		// PutObject with auto-extract support for zip
-		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc(
-			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler)))))
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzSnowballExtract, "true").
+			HandlerFunc(s3APIMiddleware(api.PutObjectExtractHandler, traceHdrsS3HFlag))
 
 		// PutObject
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler)))))
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectHandler, traceHdrsS3HFlag))
 
 		// DeleteObject
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("deleteobject", maxClients(gz(httpTraceAll(api.DeleteObjectHandler)))))
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.DeleteObjectHandler))
 
 		// PostRestoreObject
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("restoreobject", maxClients(gz(httpTraceAll(api.PostRestoreObjectHandler))))).Queries("restore", "")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PostRestoreObjectHandler)).
+			Queries("restore", "")
 
 		// Bucket operations
 
 		// GetBucketLocation
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlocation", maxClients(gz(httpTraceAll(api.GetBucketLocationHandler))))).Queries("location", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketLocationHandler)).
+			Queries("location", "")
 		// GetBucketPolicy
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketpolicy", maxClients(gz(httpTraceAll(api.GetBucketPolicyHandler))))).Queries("policy", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketPolicyHandler)).
+			Queries("policy", "")
 		// GetBucketLifecycle
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlifecycle", maxClients(gz(httpTraceAll(api.GetBucketLifecycleHandler))))).Queries("lifecycle", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// GetBucketEncryption
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketencryption", maxClients(gz(httpTraceAll(api.GetBucketEncryptionHandler))))).Queries("encryption", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketEncryptionHandler)).
+			Queries("encryption", "")
 		// GetBucketObjectLockConfig
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketobjectlockconfiguration", maxClients(gz(httpTraceAll(api.GetBucketObjectLockConfigHandler))))).Queries("object-lock", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketObjectLockConfigHandler)).
+			Queries("object-lock", "")
 		// GetBucketReplicationConfig
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.GetBucketReplicationConfigHandler))))).Queries("replication", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// GetBucketVersioning
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketversioning", maxClients(gz(httpTraceAll(api.GetBucketVersioningHandler))))).Queries("versioning", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketVersioningHandler)).
+			Queries("versioning", "")
 		// GetBucketNotification
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketNotificationHandler)).
+			Queries("notification", "")
 		// ListenNotification
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)).
+			Queries("events", "{events:.*}")
 		// ResetBucketReplicationStatus - MinIO extension API
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStatusHandler)).
+			Queries("replication-reset-status", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketacl", maxClients(gz(httpTraceAll(api.GetBucketACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketACLHandler)).
+			Queries("acl", "")
 		// PutBucketACL -- this is a dummy call.
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketacl", maxClients(gz(httpTraceAll(api.PutBucketACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
+			Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketcors", maxClients(gz(httpTraceAll(api.GetBucketCorsHandler))))).Queries("cors", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
+			Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketwebsite", maxClients(gz(httpTraceAll(api.GetBucketWebsiteHandler))))).Queries("website", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)).
+			Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketaccelerate", maxClients(gz(httpTraceAll(api.GetBucketAccelerateHandler))))).Queries("accelerate", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketAccelerateHandler)).
+			Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketrequestpayment", maxClients(gz(httpTraceAll(api.GetBucketRequestPaymentHandler))))).Queries("requestPayment", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketRequestPaymentHandler)).
+			Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlogging", maxClients(gz(httpTraceAll(api.GetBucketLoggingHandler))))).Queries("logging", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)).
+			Queries("logging", "")
 		// GetBucketTaggingHandler
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbuckettagging", maxClients(gz(httpTraceAll(api.GetBucketTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)).
+			Queries("tagging", "")
 		// DeleteBucketWebsiteHandler
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketwebsite", maxClients(gz(httpTraceAll(api.DeleteBucketWebsiteHandler))))).Queries("website", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketWebsiteHandler)).
+			Queries("website", "")
 		// DeleteBucketTaggingHandler
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebuckettagging", maxClients(gz(httpTraceAll(api.DeleteBucketTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketTaggingHandler)).
+			Queries("tagging", "")
 
 		// ListMultipartUploads
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listmultipartuploads", maxClients(gz(httpTraceAll(api.ListMultipartUploadsHandler))))).Queries("uploads", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListMultipartUploadsHandler)).
+			Queries("uploads", "")
 		// ListObjectsV2M
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv2M", maxClients(gz(httpTraceAll(api.ListObjectsV2MHandler))))).Queries("list-type", "2", "metadata", "true")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV2MHandler)).
+			Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV2Handler)).
+			Queries("list-type", "2")
 		// ListObjectVersions
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectVersionsMHandler)).
+			Queries("versions", "", "metadata", "true")
 		// ListObjectVersions
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectVersionsHandler)).
+			Queries("versions", "")
 		// GetBucketPolicyStatus
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getpolicystatus", maxClients(gz(httpTraceAll(api.GetBucketPolicyStatusHandler))))).Queries("policyStatus", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketPolicyStatusHandler)).
+			Queries("policyStatus", "")
 		// PutBucketLifecycle
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketlifecycle", maxClients(gz(httpTraceAll(api.PutBucketLifecycleHandler))))).Queries("lifecycle", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// PutBucketReplicationConfig
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.PutBucketReplicationConfigHandler))))).Queries("replication", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// PutBucketEncryption
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketencryption", maxClients(gz(httpTraceAll(api.PutBucketEncryptionHandler))))).Queries("encryption", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketEncryptionHandler)).
+			Queries("encryption", "")
 
 		// PutBucketPolicy
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketpolicy", maxClients(gz(httpTraceAll(api.PutBucketPolicyHandler))))).Queries("policy", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketPolicyHandler)).
+			Queries("policy", "")
 
 		// PutBucketObjectLockConfig
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.PutBucketObjectLockConfigHandler))))).Queries("object-lock", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketObjectLockConfigHandler)).
+			Queries("object-lock", "")
 		// PutBucketTaggingHandler
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbuckettagging", maxClients(gz(httpTraceAll(api.PutBucketTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketTaggingHandler)).
+			Queries("tagging", "")
 		// PutBucketVersioning
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketversioning", maxClients(gz(httpTraceAll(api.PutBucketVersioningHandler))))).Queries("versioning", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketVersioningHandler)).
+			Queries("versioning", "")
 		// PutBucketNotification
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketnotification", maxClients(gz(httpTraceAll(api.PutBucketNotificationHandler))))).Queries("notification", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketNotificationHandler)).
+			Queries("notification", "")
 		// ResetBucketReplicationStart - MinIO extension API
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("resetbucketreplicationstart", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStartHandler))))).Queries("replication-reset", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStartHandler)).
+			Queries("replication-reset", "")
 
 		// PutBucket
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucket", maxClients(gz(httpTraceAll(api.PutBucketHandler)))))
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketHandler))
 		// HeadBucket
-		router.Methods(http.MethodHead).HandlerFunc(
-			collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
+		router.Methods(http.MethodHead).
+			HandlerFunc(s3APIMiddleware(api.HeadBucketHandler))
 		// PostPolicy
-		router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
-			return isRequestPostPolicySignatureV4(r)
-		}).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
+		router.Methods(http.MethodPost).
+			MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
+				return isRequestPostPolicySignatureV4(r)
+			}).
+			HandlerFunc(s3APIMiddleware(api.PostPolicyBucketHandler, traceHdrsS3HFlag))
 		// DeleteMultipleObjects
-		router.Methods(http.MethodPost).HandlerFunc(
-			collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")
+		router.Methods(http.MethodPost).
+			HandlerFunc(s3APIMiddleware(api.DeleteMultipleObjectsHandler)).
+			Queries("delete", "")
 		// DeleteBucketPolicy
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketpolicy", maxClients(gz(httpTraceAll(api.DeleteBucketPolicyHandler))))).Queries("policy", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketPolicyHandler)).
+			Queries("policy", "")
 		// DeleteBucketReplication
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.DeleteBucketReplicationConfigHandler))))).Queries("replication", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// DeleteBucketLifecycle
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketlifecycle", maxClients(gz(httpTraceAll(api.DeleteBucketLifecycleHandler))))).Queries("lifecycle", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// DeleteBucketEncryption
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketencryption", maxClients(gz(httpTraceAll(api.DeleteBucketEncryptionHandler))))).Queries("encryption", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketEncryptionHandler)).
+			Queries("encryption", "")
 		// DeleteBucket
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucket", maxClients(gz(httpTraceAll(api.DeleteBucketHandler)))))
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketHandler))
 
 		// MinIO extension API for replication.
 		//
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsV2Handler)).
+			Queries("replication-metrics", "2")
 		// deprecated handler
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsHandler)).
+			Queries("replication-metrics", "")
 
 		// ValidateBucketReplicationCreds
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ValidateBucketReplicationCredsHandler)).
+			Queries("replication-check", "")
 
 		// Register rejected bucket APIs
 		for _, r := range rejectedBucketAPIs {
@@ -468,24 +607,25 @@ func registerAPIRouter(router *mux.Router) {
 		}
 
 		// S3 ListObjectsV1 (Legacy)
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv1", maxClients(gz(httpTraceAll(api.ListObjectsV1Handler)))))
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV1Handler))
 	}
 
 	// Root operation
 
 	// ListenNotification
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).
+		HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)).
+		Queries("events", "{events:.*}")
 
 	// ListBuckets
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).
+		HandlerFunc(s3APIMiddleware(api.ListBucketsHandler))
 
 	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
 	// than failing with UnknownAPIRequest we simply handle it for now.
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
-		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).
+		HandlerFunc(s3APIMiddleware(api.ListBucketsHandler))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))

cmd/api-utils.go

@@ -18,6 +18,10 @@
 package cmd
 
 import (
+	"fmt"
+	"net/http"
+	"reflect"
+	"runtime"
 	"strings"
 )
 
@@ -100,3 +104,16 @@ func s3EncodeName(name, encodingType string) string {
 	}
 	return name
 }
+
+// getHandlerName returns the name of the handler function. It takes the type
+// name as a string to clean up the name retrieved via reflection. This function
+// only works correctly when the type is present in the cmd package.
+func getHandlerName(f http.HandlerFunc, cmdType string) string {
+	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
+
+	packageName := fmt.Sprintf("github.com/minio/minio/cmd.%s.", cmdType)
+	name = strings.TrimPrefix(name, packageName)
+	name = strings.TrimSuffix(name, "Handler-fm")
+	name = strings.TrimSuffix(name, "-fm")
+	return name
+}

cmd/auth-handler.go

@@ -162,7 +162,8 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
 		getRequestAuthType(r) == authTypeSigned {
-		// We only support admin credentials to access admin APIs.
+
+		// Get credential information from the request.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
 			return cred, owner, s3Err
@@ -256,7 +257,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(GlobalContext, err, logger.Application)
+			logger.LogIf(GlobalContext, err, logger.ErrorKind)
 			return nil, errAuthentication
 		}
 		claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
@@ -293,7 +294,21 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrInvalidToken
 	}
 
+	// Expired credentials must return error right away.
+	if cred.IsTemp() && cred.IsExpired() {
+		return nil, toAPIErrorCode(r.Context(), errInvalidAccessKeyID)
+	}
 	secret := globalActiveCred.SecretKey
+	if globalSiteReplicationSys.isEnabled() && cred.AccessKey != siteReplicatorSvcAcc {
+		nsecret, err := getTokenSigningKey()
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		// sign root's temporary accounts also with site replicator creds
+		if cred.ParentUser != globalActiveCred.AccessKey || cred.IsTemp() {
+			secret = nsecret
+		}
+	}
 	if cred.IsServiceAccount() {
 		token = cred.SessionToken
 		secret = cred.SecretKey
@@ -338,7 +353,7 @@ func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action po
 
 func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	if logger.GetReqInfo(ctx) == nil {
-		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
+		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
 		return ErrAccessDenied
 	}
 
@@ -377,7 +392,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 		// To extract region from XML in request body, get copy of request body.
 		payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
-			logger.LogIf(ctx, err, logger.Application)
+			logger.LogIf(ctx, err, logger.ErrorKind)
 			return ErrMalformedXML
 		}
 

cmd/auth-handler_test.go

@@ -237,7 +237,7 @@ func TestIsRequestPresignedSignatureV2(t *testing.T) {
 	}
 }
 
-// TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature verision v4 detection.
+// TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature version v4 detection.
 func TestIsRequestPresignedSignatureV4(t *testing.T) {
 	testCases := []struct {
 		inputQueryKey   string
@@ -287,7 +287,7 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -298,7 +298,7 @@ func mustNewSignedV2Request(method string, urlStr string, contentLength int64, b
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -309,7 +309,7 @@ func mustNewPresignedV2Request(method string, urlStr string, contentLength int64
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -320,7 +320,7 @@ func mustNewPresignedRequest(method string, urlStr string, contentLength int64,
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	cred := globalActiveCred
 	if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
-		t.Fatalf("Unable to inititalized new signed http request %s", err)
+		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
 	return req
 }
@@ -491,7 +491,7 @@ func TestValidateAdminSignature(t *testing.T) {
 	for i, testCase := range testCases {
 		req := mustNewRequest(http.MethodGet, "http://localhost:9000/", 0, nil, t)
 		if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
-			t.Fatalf("Unable to inititalized new signed http request %s", err)
+			t.Fatalf("Unable to initialized new signed http request %s", err)
 		}
 		_, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
 		if s3Error != testCase.ErrCode {

cmd/background-heal-ops.go

@@ -102,7 +102,6 @@ func waitForLowHTTPReq() {
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
 	for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
 		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI)
 	}

cmd/background-newdisks-heal-ops.go

@@ -87,6 +87,8 @@ type healingTracker struct {
 	// ID of the current healing operation
 	HealID string
 
+	ItemsSkipped uint64
+	BytesSkipped uint64
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -175,14 +177,18 @@ func (h *healingTracker) setObject(object string) {
 	h.Object = object
 }
 
-func (h *healingTracker) updateProgress(success bool, bytes uint64) {
+func (h *healingTracker) updateProgress(success, skipped bool, bytes uint64) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	if success {
+	switch {
+	case success:
 		h.ItemsHealed++
 		h.BytesDone += bytes
-	} else {
+	case skipped:
+		h.ItemsSkipped++
+		h.BytesSkipped += bytes
+	default:
 		h.ItemsFailed++
 		h.BytesFailed += bytes
 	}
@@ -323,8 +329,10 @@ func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 		ObjectsTotalCount: h.ObjectsTotalCount,
 		ObjectsTotalSize:  h.ObjectsTotalSize,
 		ItemsHealed:       h.ItemsHealed,
+		ItemsSkipped:      h.ItemsSkipped,
 		ItemsFailed:       h.ItemsFailed,
 		BytesDone:         h.BytesDone,
+		BytesSkipped:      h.BytesSkipped,
 		BytesFailed:       h.BytesFailed,
 		Bucket:            h.Bucket,
 		Object:            h.Object,
@@ -353,7 +361,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	globalLocalDrivesMu.RLock()
-	localDrives := globalLocalDrives
+	localDrives := cloneDrives(globalLocalDrives)
 	globalLocalDrivesMu.RUnlock()
 	for _, disk := range localDrives {
 		_, err := disk.GetDiskID()
@@ -376,25 +384,10 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
 func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
-	disk, format, err := connectEndpoint(endpoint)
-	if err != nil {
-		return fmt.Errorf("Error: %w, %s", err, endpoint)
-	}
-	defer disk.Close()
-	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
-	if poolIdx < 0 {
-		return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
-	}
-
-	// Calculate the set index where the current endpoint belongs
-	z.serverPools[poolIdx].erasureDisksMu.RLock()
-	setIdx, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
-	z.serverPools[poolIdx].erasureDisksMu.RUnlock()
-	if err != nil {
-		return err
-	}
-	if setIdx < 0 {
-		return fmt.Errorf("unexpected set index (%d) found for  %s", setIdx, disk.Endpoint())
+	poolIdx, setIdx := endpoint.PoolIdx, endpoint.SetIdx
+	disk := getStorageViaEndpoint(endpoint)
+	if disk == nil {
+		return fmt.Errorf("Unexpected error disk must be initialized by now after formatting: %s", endpoint)
 	}
 
 	// Prevent parallel erasure set healing
@@ -420,7 +413,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		tracker = initHealingTracker(disk, mustGetUUID())
 	}
 
-	logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
+	logger.Event(ctx, "Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint)
 
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple pools/sets, make
@@ -440,10 +433,6 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return buckets[i].Created.After(buckets[j].Created)
 	})
 
-	if serverDebugLog {
-		logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
-	}
-
 	// Load bucket totals
 	cache := dataUsageCache{}
 	if err := cache.load(ctx, z.serverPools[poolIdx].sets[setIdx], dataUsageCacheName); err == nil {
@@ -463,11 +452,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 
-	if tracker.ItemsFailed > 0 {
-		logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
-	} else {
-		logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
-	}
+	logger.Event(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 
 	if len(tracker.QueuedBuckets) > 0 {
 		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
@@ -475,7 +460,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 
 	if serverDebugLog {
 		tracker.printTo(os.Stdout)
-		logger.Info("\n")
+		fmt.Printf("\n")
 	}
 
 	if tracker.HealID == "" { // HealID was empty only before Feb 2023
@@ -490,6 +475,10 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	for _, disk := range disks {
+		if disk == nil {
+			continue
+		}
+
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
 			if !errors.Is(err, errFileNotFound) {

cmd/background-newdisks-heal-ops_gen.go

@@ -188,6 +188,18 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "HealID")
 				return
 			}
+		case "ItemsSkipped":
+			z.ItemsSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ItemsSkipped")
+				return
+			}
+		case "BytesSkipped":
+			z.BytesSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "BytesSkipped")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -201,9 +213,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 23
+	// map header, size 25
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x19, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -446,15 +458,35 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "HealID")
 		return
 	}
+	// write "ItemsSkipped"
+	err = en.Append(0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ItemsSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ItemsSkipped")
+		return
+	}
+	// write "BytesSkipped"
+	err = en.Append(0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.BytesSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "BytesSkipped")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 23
+	// map header, size 25
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x19, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -528,6 +560,12 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "HealID"
 	o = append(o, 0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
 	o = msgp.AppendString(o, z.HealID)
+	// string "ItemsSkipped"
+	o = append(o, 0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ItemsSkipped)
+	// string "BytesSkipped"
+	o = append(o, 0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.BytesSkipped)
 	return
 }
 
@@ -713,6 +751,18 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "HealID")
 				return
 			}
+		case "ItemsSkipped":
+			z.ItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ItemsSkipped")
+				return
+			}
+		case "BytesSkipped":
+			z.BytesSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BytesSkipped")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -735,6 +785,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
-	s += 7 + msgp.StringPrefixSize + len(z.HealID)
+	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size
 	return
 }

cmd/batch-expire.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/versioning"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/wildcard"
@@ -44,7 +45,7 @@ import (
 //   bucket: mybucket # Bucket where this batch job will expire matching objects from
 //   prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
 //   rules:
-//     - type: object  # regular objects with zero ore more older versions
+//     - type: object  # regular objects with zero or more older versions
 //       name: NAME # match object names that satisfy the wildcard expression.
 //       olderThan: 70h # match objects older than this value
 //       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
@@ -426,7 +427,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 					default:
 					}
 					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-					_, err := api.DeleteObject(ctx, exp.Bucket, exp.Name, ObjectOptions{
+					_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
 						DeletePrefix: true,
 					})
 					if err != nil {
@@ -558,7 +559,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 			case <-ctx.Done():
-				// persist in-memory state immediately before exiting due to context cancelation.
+				// persist in-memory state immediately before exiting due to context cancellation.
 				logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 				return
 
@@ -571,7 +572,11 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	}()
 
 	expireCh := make(chan []expireObjInfo, workerSize)
-	go batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
+	expireDoneCh := make(chan struct{})
+	go func() {
+		defer close(expireDoneCh)
+		batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
+	}()
 
 	var (
 		prevObj       ObjectInfo
@@ -648,9 +653,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		case expireCh <- toDel:
 		}
 	}
-	close(expireCh)
+	xioutil.SafeClose(expireCh)
 
-	wk.Wait() // waits for all expire goroutines to complete
+	<-expireDoneCh // waits for the expire goroutine to complete
+	wk.Wait()      // waits for all expire workers to retire
 
 	ri.Complete = ri.ObjectsFailed == 0
 	ri.Failed = ri.ObjectsFailed > 0
@@ -658,7 +664,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	// Close the saverQuitCh - this also triggers saving in-memory state
 	// immediately one last time before we exit this method.
-	close(saverQuitCh)
+	xioutil.SafeClose(saverQuitCh)
 
 	// Notify expire jobs final status to the configured endpoint
 	buf, _ := json.Marshal(ri)

cmd/batch-expire_test.go

@@ -30,7 +30,7 @@ expire: # Expire objects that match a condition
   bucket: mybucket # Bucket where this batch job will expire matching objects from
   prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
   rules:
-    - type: object  # regular objects with zero ore more older versions
+    - type: object  # regular objects with zero or more older versions
       name: NAME # match object names that satisfy the wildcard expression.
       olderThan: 70h # match objects older than this value
       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"

cmd/batch-handlers.go

@@ -47,6 +47,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/env"
@@ -62,7 +63,6 @@ type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
 	User      string               `yaml:"-" json:"user"`
 	Started   time.Time            `yaml:"-" json:"started"`
-	Location  string               `yaml:"-" json:"location"`
 	Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
 	KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
 	Expire    *BatchJobExpire      `yaml:"expire" json:"expire"`
@@ -277,7 +277,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	isTags := len(r.Flags.Filter.Tags) != 0
+	hasTags := len(r.Flags.Filter.Tags) != 0
 	isMetadata := len(r.Flags.Filter.Metadata) != 0
 	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
 
@@ -302,7 +302,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return true
 		}
 
-		if isTags {
+		if hasTags {
 			// Only parse object tags if tags filter is specified.
 			tagMap := map[string]string{}
 			tagStr := oi.UserTags
@@ -407,7 +407,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 						continue
 					}
 				}
-				if isTags {
+				if hasTags {
 					tags, err := c.GetObjectTagging(ctx, r.Source.Bucket, obj.Key, minio.GetObjectTaggingOptions{})
 					if err == nil {
 						oi.UserTags = tags.String()
@@ -545,7 +545,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 	}
 
 	go func() {
-		defer close(input)
+		defer xioutil.SafeClose(input)
 
 		for _, entry := range entries {
 			gr, err := objAPI.GetObjectNInfo(ctx, r.Source.Bucket,
@@ -709,38 +709,52 @@ type batchJobInfo struct {
 }
 
 const (
-	batchReplName      = "batch-replicate.bin"
-	batchReplFormat    = 1
-	batchReplVersionV1 = 1
-	batchReplVersion   = batchReplVersionV1
-	batchJobName       = "job.bin"
-	batchJobPrefix     = "batch-jobs"
+	batchReplName         = "batch-replicate.bin"
+	batchReplFormat       = 1
+	batchReplVersionV1    = 1
+	batchReplVersion      = batchReplVersionV1
+	batchJobName          = "job.bin"
+	batchJobPrefix        = "batch-jobs"
+	batchJobReportsPrefix = batchJobPrefix + "/reports"
 
 	batchReplJobAPIVersion        = "v1"
 	batchReplJobDefaultRetries    = 3
 	batchReplJobDefaultRetryDelay = 250 * time.Millisecond
 )
 
-func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+func getJobReportPath(job BatchJobRequest) string {
 	var fileName string
-	var format, version uint16
 	switch {
 	case job.Replicate != nil:
 		fileName = batchReplName
+	case job.KeyRotate != nil:
+		fileName = batchKeyRotationName
+	case job.Expire != nil:
+		fileName = batchExpireName
+	}
+	return pathJoin(batchJobReportsPrefix, job.ID, fileName)
+}
+
+func getJobPath(job BatchJobRequest) string {
+	return pathJoin(batchJobPrefix, job.ID)
+}
+
+func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	var format, version uint16
+	switch {
+	case job.Replicate != nil:
 		version = batchReplVersionV1
 		format = batchReplFormat
 	case job.KeyRotate != nil:
-		fileName = batchKeyRotationName
 		version = batchKeyRotateVersionV1
 		format = batchKeyRotationFormat
 	case job.Expire != nil:
-		fileName = batchExpireName
 		version = batchExpireVersionV1
 		format = batchExpireFormat
 	default:
 		return errors.New("no supported batch job request specified")
 	}
-	data, err := readConfig(ctx, api, pathJoin(job.Location, fileName))
+	data, err := readConfig(ctx, api, getJobReportPath(job))
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
 			ri.Version = int(version)
@@ -851,8 +865,8 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 	now := UTCNow()
 	ri.mu.Lock()
 	var (
-		format, version  uint16
-		jobTyp, fileName string
+		format, version uint16
+		jobTyp          string
 	)
 
 	if now.Sub(ri.LastUpdate) >= duration {
@@ -861,19 +875,16 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 			format = batchReplFormat
 			version = batchReplVersion
 			jobTyp = string(job.Type())
-			fileName = batchReplName
 			ri.Version = batchReplVersionV1
 		case madmin.BatchJobKeyRotate:
 			format = batchKeyRotationFormat
 			version = batchKeyRotateVersion
 			jobTyp = string(job.Type())
-			fileName = batchKeyRotationName
 			ri.Version = batchKeyRotateVersionV1
 		case madmin.BatchJobExpire:
 			format = batchExpireFormat
 			version = batchExpireVersion
 			jobTyp = string(job.Type())
-			fileName = batchExpireName
 			ri.Version = batchExpireVersionV1
 		default:
 			return errInvalidArgument
@@ -894,7 +905,7 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 		if err != nil {
 			return err
 		}
-		return saveConfig(ctx, api, pathJoin(job.Location, fileName), buf)
+		return saveConfig(ctx, api, getJobReportPath(job), buf)
 	}
 	ri.mu.Unlock()
 	return nil
@@ -924,6 +935,21 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
 	ri.countItem(info.Size, info.DeleteMarker, success)
 }
 
+func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInfo) {
+	if ri == nil {
+		return
+	}
+
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+
+	ri.Bucket = bucket
+	for i := range batch {
+		ri.Object = batch[i].Name
+		ri.countItem(batch[i].Size, batch[i].DeleteMarker, true)
+	}
+}
+
 // Start start the batch replication job, resumes if there was a pending job via "job.ID"
 func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
 	ri := &batchJobInfo{
@@ -1038,8 +1064,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
 		go func() {
-			defer close(slowCh)
-
 			// Snowball currently needs the high level minio-go Client, not the Core one
 			cl, err := miniogo.New(u.Host, &miniogo.Options{
 				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
@@ -1055,19 +1079,23 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			// Already validated before arriving here
 			smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
 
-			var (
-				obj   = ObjectInfo{}
-				batch = make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
-				valid = true
-			)
-
-			for valid {
-				obj, valid = <-walkCh
-
-				if !valid {
-					goto write
+			batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
+			writeFn := func(batch []ObjectInfo) {
+				if len(batch) > 0 {
+					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
+						logger.LogIf(ctx, err)
+						for _, b := range batch {
+							slowCh <- b
+						}
+					} else {
+						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
+						globalBatchJobsMetrics.save(job.ID, ri)
+						// persist in-memory state to disk after every 10secs.
+						logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+					}
 				}
-
+			}
+			for obj := range walkCh {
 				if obj.DeleteMarker || !obj.VersionPurgeStatus.Empty() || obj.Size >= int64(smallerThan) {
 					slowCh <- obj
 					continue
@@ -1078,18 +1106,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				if len(batch) < *r.Source.Snowball.Batch {
 					continue
 				}
-
-			write:
-				if len(batch) > 0 {
-					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
-						logger.LogIf(ctx, err)
-						for _, b := range batch {
-							slowCh <- b
-						}
-					}
-					batch = batch[:0]
-				}
+				writeFn(batch)
+				batch = batch[:0]
 			}
+			writeFn(batch)
+			xioutil.SafeClose(slowCh)
 		}()
 	} else {
 		slowCh = walkCh
@@ -1396,15 +1417,8 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 }
 
 func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
-	switch {
-	case j.Replicate != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchReplName))
-	case j.KeyRotate != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
-	case j.Expire != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchExpireName))
-	}
-	deleteConfig(ctx, api, j.Location)
+	deleteConfig(ctx, api, getJobReportPath(j))
+	deleteConfig(ctx, api, getJobPath(j))
 }
 
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
@@ -1416,13 +1430,12 @@ func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
 		return err
 	}
 
-	j.Location = pathJoin(batchJobPrefix, j.ID)
 	job, err := j.MarshalMsg(nil)
 	if err != nil {
 		return err
 	}
 
-	return saveConfig(ctx, api, j.Location, job)
+	return saveConfig(ctx, api, getJobPath(*j), job)
 }
 
 func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string) error {
@@ -1650,8 +1663,7 @@ func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request)
 	}
 
 	j := BatchJobRequest{
-		ID:       jobID,
-		Location: pathJoin(batchJobPrefix, jobID),
+		ID: jobID,
 	}
 
 	j.delete(ctx, objectAPI)
@@ -1809,7 +1821,7 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 	select {
 	case <-j.ctx.Done():
 		j.once.Do(func() {
-			close(j.jobCh)
+			xioutil.SafeClose(j.jobCh)
 		})
 	case j.jobCh <- req:
 	default:

cmd/batch-handlers_gen.go

@@ -42,12 +42,6 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if dc.IsNil() {
 				err = dc.ReadNil()
@@ -115,9 +109,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 7
+	// map header, size 6
 	// write "ID"
-	err = en.Append(0x87, 0xa2, 0x49, 0x44)
+	err = en.Append(0x86, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -146,16 +140,6 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Started")
 		return
 	}
-	// write "Location"
-	err = en.Append(0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Location)
-	if err != nil {
-		err = msgp.WrapError(err, "Location")
-		return
-	}
 	// write "Replicate"
 	err = en.Append(0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if err != nil {
@@ -213,9 +197,9 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 7
+	// map header, size 6
 	// string "ID"
-	o = append(o, 0x87, 0xa2, 0x49, 0x44)
+	o = append(o, 0x86, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -223,9 +207,6 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Started"
 	o = append(o, 0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
 	o = msgp.AppendTime(o, z.Started)
-	// string "Location"
-	o = append(o, 0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	o = msgp.AppendString(o, z.Location)
 	// string "Replicate"
 	o = append(o, 0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if z.Replicate == nil {
@@ -298,12 +279,6 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
@@ -369,7 +344,7 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobRequest) Msgsize() (s int) {
-	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 9 + msgp.StringPrefixSize + len(z.Location) + 10
+	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 10
 	if z.Replicate == nil {
 		s += msgp.NilSize
 	} else {

cmd/batch-job-common-types.go

@@ -213,11 +213,14 @@ func (b BatchJobSnowball) Validate() error {
 		}
 	}
 	_, err := humanize.ParseBytes(*b.SmallerThan)
-	return BatchJobYamlErr{
-		line: b.line,
-		col:  b.col,
-		msg:  err.Error(),
+	if err != nil {
+		return BatchJobYamlErr{
+			line: b.line,
+			col:  b.col,
+			msg:  err.Error(),
+		}
 	}
+	return nil
 }
 
 // BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan

cmd/batch-rotate.go

@@ -274,7 +274,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	skip := func(info FileInfo) (ok bool) {
+	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
@@ -360,7 +360,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	results := make(chan ObjectInfo, 100)
 	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
 		Marker: lastObject,
-		Filter: skip,
+		Filter: selectObj,
 	}); err != nil {
 		cancel()
 		// Do not need to retry if we can't list objects on source.

cmd/bitrot-streaming.go

@@ -20,16 +20,12 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
-	"fmt"
 	"hash"
 	"io"
-	"strings"
 	"sync"
 
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 )
 
 // Calculates bitrot in chunks and writes the hash into the stream.
@@ -90,7 +86,7 @@ func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize
 }
 
 // Returns streaming bitrot writer implementation.
-func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
+func newStreamingBitrotWriter(disk StorageAPI, origvolume, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
 	r, w := io.Pipe()
 	h := algo.New()
 
@@ -110,7 +106,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
 			totalFileSize = bitrotSumsTotalSize + length
 		}
-		r.CloseWithError(disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r))
+		r.CloseWithError(disk.CreateFile(context.TODO(), origvolume, volume, filePath, totalFileSize, r))
 	}()
 	return bw
 }
@@ -134,7 +130,7 @@ func (b *streamingBitrotReader) Close() error {
 		return nil
 	}
 	if closer, ok := b.rc.(io.Closer); ok {
-		// drain the body for connection re-use at network layer.
+		// drain the body for connection reuse at network layer.
 		xhttp.DrainBody(struct {
 			io.Reader
 			io.Closer
@@ -154,29 +150,12 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
 	}
-	ignoredErrs := []error{
-		errDiskNotFound,
-	}
-	if strings.HasPrefix(b.volume, minioMetaBucket) {
-		ignoredErrs = append(ignoredErrs,
-			errFileNotFound,
-			errVolumeNotFound,
-			errFileVersionNotFound,
-		)
-	}
 	if b.rc == nil {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		if len(b.data) == 0 && b.tillOffset != streamOffset {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
-			if err != nil {
-				if !IsErr(err, ignoredErrs...) {
-					logger.LogOnceIf(GlobalContext,
-						fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
-							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream-"+b.volume+"-"+b.filePath)
-				}
-			}
 		} else {
 			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
 		}
@@ -198,10 +177,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		return 0, err
 	}
 	b.h.Write(buf)
-
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s  -> %s/%s - content hash does not match - expected %s, got %s",
-			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
 		return 0, errFileCorrupt
 	}
 	b.currOffset += int64(len(buf))

cmd/bitrot-whole.go

@@ -19,11 +19,8 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"hash"
 	"io"
-
-	"github.com/minio/minio/internal/logger"
 )
 
 // Implementation to calculate bitrot for the whole file.
@@ -38,12 +35,10 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	return len(p), nil
@@ -72,12 +67,10 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot.go

@@ -102,9 +102,9 @@ func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
 	return
 }
 
-func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
+func newBitrotWriter(disk StorageAPI, origvolume, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
 	if algo == HighwayHash256S {
-		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
+		return newStreamingBitrotWriter(disk, origvolume, volume, filePath, length, algo, shardSize)
 	}
 	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }

cmd/bitrot_test.go

@@ -36,7 +36,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 
 	disk.MakeVol(context.Background(), volume)
 
-	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
+	writer := newBitrotWriter(disk, "", volume, filePath, 35, bitrotAlgo, 10)
 
 	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {

cmd/bootstrap-peer-server.go

@@ -19,71 +19,47 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
+	"math/rand"
 	"reflect"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
-	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/rest"
-	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/env"
 )
 
-const (
-	bootstrapRESTVersion       = "v1"
-	bootstrapRESTVersionPrefix = SlashSeparator + bootstrapRESTVersion
-	bootstrapRESTPrefix        = minioReservedBucketPath + "/bootstrap"
-	bootstrapRESTPath          = bootstrapRESTPrefix + bootstrapRESTVersionPrefix
-)
-
-const (
-	bootstrapRESTMethodHealth = "/health"
-	bootstrapRESTMethodVerify = "/verify"
-)
-
 // To abstract a node over network.
 type bootstrapRESTServer struct{}
 
+//go:generate msgp -file=$GOFILE
+
 // ServerSystemConfig - captures information about server configuration.
 type ServerSystemConfig struct {
-	MinioEndpoints EndpointServerPools
-	MinioEnv       map[string]string
+	NEndpoints int
+	CmdLines   []string
+	MinioEnv   map[string]string
 }
 
 // Diff - returns error on first difference found in two configs.
-func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
-	if s1.MinioEndpoints.NEndpoints() != s2.MinioEndpoints.NEndpoints() {
-		return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.NEndpoints(),
-			s2.MinioEndpoints.NEndpoints())
+func (s1 *ServerSystemConfig) Diff(s2 *ServerSystemConfig) error {
+	ns1 := s1.NEndpoints
+	ns2 := s2.NEndpoints
+	if ns1 != ns2 {
+		return fmt.Errorf("Expected number of endpoints %d, seen %d", ns1, ns2)
 	}
 
-	for i, ep := range s1.MinioEndpoints {
-		if ep.CmdLine != s2.MinioEndpoints[i].CmdLine {
-			return fmt.Errorf("Expected command line argument %s, seen %s", ep.CmdLine,
-				s2.MinioEndpoints[i].CmdLine)
-		}
-		if ep.SetCount != s2.MinioEndpoints[i].SetCount {
-			return fmt.Errorf("Expected set count %d, seen %d", ep.SetCount,
-				s2.MinioEndpoints[i].SetCount)
-		}
-		if ep.DrivesPerSet != s2.MinioEndpoints[i].DrivesPerSet {
-			return fmt.Errorf("Expected drives pet set %d, seen %d", ep.DrivesPerSet,
-				s2.MinioEndpoints[i].DrivesPerSet)
-		}
-		if ep.Platform != s2.MinioEndpoints[i].Platform {
-			return fmt.Errorf("Expected platform '%s', found to be on '%s'",
-				ep.Platform, s2.MinioEndpoints[i].Platform)
+	for i, cmdLine := range s1.CmdLines {
+		if cmdLine != s2.CmdLines[i] {
+			return fmt.Errorf("Expected command line argument %s, seen %s", cmdLine,
+				s2.CmdLines[i])
 		}
 	}
+
 	if reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
 		return nil
 	}
@@ -131,7 +107,7 @@ var skipEnvs = map[string]struct{}{
 	"MINIO_SECRET_KEY":    {},
 }
 
-func getServerSystemCfg() ServerSystemConfig {
+func getServerSystemCfg() *ServerSystemConfig {
 	envs := env.List("MINIO_")
 	envValues := make(map[string]string, len(envs))
 	for _, envK := range envs {
@@ -144,131 +120,115 @@ func getServerSystemCfg() ServerSystemConfig {
 		}
 		envValues[envK] = logger.HashString(env.Get(envK, ""))
 	}
-	return ServerSystemConfig{
-		MinioEndpoints: globalEndpoints,
-		MinioEnv:       envValues,
+	scfg := &ServerSystemConfig{NEndpoints: globalEndpoints.NEndpoints(), MinioEnv: envValues}
+	var cmdLines []string
+	for _, ep := range globalEndpoints {
+		cmdLines = append(cmdLines, ep.CmdLine)
 	}
+	scfg.CmdLines = cmdLines
+	return scfg
 }
 
-func (b *bootstrapRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
-	w.WriteHeader(http.StatusForbidden)
-	w.Write([]byte(err.Error()))
+func (s *bootstrapRESTServer) VerifyHandler(params *grid.MSS) (*ServerSystemConfig, *grid.RemoteErr) {
+	return getServerSystemCfg(), nil
 }
 
-// HealthHandler returns success if request is valid
-func (b *bootstrapRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {}
-
-func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "VerifyHandler")
-
-	if err := storageServerRequestValidate(r); err != nil {
-		b.writeErrorResponse(w, err)
-		return
-	}
-
-	cfg := getServerSystemCfg()
-	logger.LogIf(ctx, json.NewEncoder(w).Encode(&cfg))
-}
+var serverVerifyHandler = grid.NewSingleHandler[*grid.MSS, *ServerSystemConfig](grid.HandlerServerVerify, grid.NewMSS, func() *ServerSystemConfig { return &ServerSystemConfig{} })
 
 // registerBootstrapRESTHandlers - register bootstrap rest router.
-func registerBootstrapRESTHandlers(router *mux.Router) {
-	h := func(f http.HandlerFunc) http.HandlerFunc {
-		return collectInternodeStats(httpTraceHdrs(f))
-	}
-
+func registerBootstrapRESTHandlers(gm *grid.Manager) {
 	server := &bootstrapRESTServer{}
-	subrouter := router.PathPrefix(bootstrapRESTPrefix).Subrouter()
-
-	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodHealth).HandlerFunc(
-		h(server.HealthHandler))
-
-	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodVerify).HandlerFunc(
-		h(server.VerifyHandler))
+	logger.FatalIf(serverVerifyHandler.Register(gm, server.VerifyHandler), "unable to register handler")
 }
 
 // client to talk to bootstrap NEndpoints.
 type bootstrapRESTClient struct {
-	endpoint   Endpoint
-	restClient *rest.Client
+	gridConn *grid.Connection
 }
 
-// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
-// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
-// after verifying format.json
-func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if values == nil {
-		values = make(url.Values)
+// Verify function verifies the server config.
+func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg *ServerSystemConfig) (err error) {
+	if newObjectLayerFn() != nil {
+		return nil
 	}
 
-	respBody, err = client.restClient.Call(ctx, method, values, body, length)
-	if err == nil {
-		return respBody, nil
+	recvCfg, err := serverVerifyHandler.Call(ctx, client.gridConn, grid.NewMSS())
+	if err != nil {
+		return err
 	}
+	// We do not need the response after returning.
+	defer serverVerifyHandler.PutResponse(recvCfg)
 
-	return nil, err
+	return srcCfg.Diff(recvCfg)
 }
 
 // Stringer provides a canonicalized representation of node.
 func (client *bootstrapRESTClient) String() string {
-	return client.endpoint.String()
+	return client.gridConn.String()
 }
 
-// Verify - fetches system server config.
-func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSystemConfig) (err error) {
-	if newObjectLayerFn() != nil {
-		return nil
-	}
-	respBody, err := client.callWithContext(ctx, bootstrapRESTMethodVerify, nil, nil, -1)
-	if err != nil {
-		return
-	}
-	defer xhttp.DrainBody(respBody)
-	recvCfg := ServerSystemConfig{}
-	if err = json.NewDecoder(respBody).Decode(&recvCfg); err != nil {
-		return err
-	}
-	return srcCfg.Diff(recvCfg)
-}
-
-func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools) error {
+func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools, gm *grid.Manager) error {
 	srcCfg := getServerSystemCfg()
-	clnts := newBootstrapRESTClients(endpointServerPools)
+	clnts := newBootstrapRESTClients(endpointServerPools, gm)
 	var onlineServers int
 	var offlineEndpoints []error
 	var incorrectConfigs []error
 	var retries int
+	var mu sync.Mutex
 	for onlineServers < len(clnts)/2 {
+		var wg sync.WaitGroup
+		wg.Add(len(clnts))
+		onlineServers = 0
 		for _, clnt := range clnts {
-			if err := clnt.Verify(ctx, srcCfg); err != nil {
-				bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
-				if !isNetworkError(err) {
-					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
-					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
-				} else {
-					offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt.String(), err))
+			go func(clnt *bootstrapRESTClient) {
+				defer wg.Done()
+
+				if clnt.gridConn.State() != grid.StateConnected {
+					mu.Lock()
+					offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, grid.ErrDisconnected))
+					mu.Unlock()
+					return
 				}
-				continue
-			}
-			onlineServers++
+
+				ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+				defer cancel()
+
+				err := clnt.Verify(ctx, srcCfg)
+				mu.Lock()
+				if err != nil {
+					bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %s", err, clnt))
+					if !isNetworkError(err) {
+						logger.LogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String())
+						incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt, err))
+					} else {
+						offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, err))
+					}
+				} else {
+					onlineServers++
+				}
+				mu.Unlock()
+			}(clnt)
 		}
+		wg.Wait()
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		default:
-			// Sleep for a while - so that we don't go into
-			// 100% CPU when half the endpoints are offline.
-			time.Sleep(100 * time.Millisecond)
+			// Sleep and stagger to avoid blocked CPU and thundering
+			// herd upon start up sequence.
+			time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond))))
 			retries++
 			// after 20 retries start logging that servers are not reachable yet
 			if retries >= 20 {
-				logger.Info(fmt.Sprintf("Waiting for atleast %d remote servers with valid configuration to be online", len(clnts)/2))
+				logger.Info(fmt.Sprintf("Waiting for at least %d remote servers with valid configuration to be online", len(clnts)/2))
 				if len(offlineEndpoints) > 0 {
 					logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
 				}
 				if len(incorrectConfigs) > 0 {
 					logger.Info(fmt.Sprintf("Following servers have mismatching configuration %s", incorrectConfigs))
 				}
-				retries = 0 // reset to log again after 5 retries.
+				retries = 0 // reset to log again after 20 retries.
 			}
 			offlineEndpoints = nil
 			incorrectConfigs = nil
@@ -277,39 +237,20 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	return nil
 }
 
-func newBootstrapRESTClients(endpointServerPools EndpointServerPools) []*bootstrapRESTClient {
-	seenHosts := set.NewStringSet()
+func newBootstrapRESTClients(endpointServerPools EndpointServerPools, gm *grid.Manager) []*bootstrapRESTClient {
+	seenClient := set.NewStringSet()
 	var clnts []*bootstrapRESTClient
 	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
-			if seenHosts.Contains(endpoint.Host) {
+			if endpoint.IsLocal {
 				continue
 			}
-			seenHosts.Add(endpoint.Host)
-
-			// Only proceed for remote endpoints.
-			if !endpoint.IsLocal {
-				cl := newBootstrapRESTClient(endpoint)
-				if serverDebugLog {
-					cl.restClient.TraceOutput = os.Stdout
-				}
-				clnts = append(clnts, cl)
+			if seenClient.Contains(endpoint.Host) {
+				continue
 			}
+			seenClient.Add(endpoint.Host)
+			clnts = append(clnts, &bootstrapRESTClient{gm.Connection(endpoint.GridHost())})
 		}
 	}
 	return clnts
 }
-
-// Returns a new bootstrap client.
-func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
-	serverURL := &url.URL{
-		Scheme: endpoint.Scheme,
-		Host:   endpoint.Host,
-		Path:   bootstrapRESTPath,
-	}
-
-	restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
-	restClient.HealthCheckFn = nil
-
-	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}
-}

cmd/bootstrap-peer-server_gen.go

@@ -0,0 +1,270 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *ServerSystemConfig) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "NEndpoints":
+			z.NEndpoints, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "NEndpoints")
+				return
+			}
+		case "CmdLines":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "CmdLines")
+				return
+			}
+			if cap(z.CmdLines) >= int(zb0002) {
+				z.CmdLines = (z.CmdLines)[:zb0002]
+			} else {
+				z.CmdLines = make([]string, zb0002)
+			}
+			for za0001 := range z.CmdLines {
+				z.CmdLines[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "CmdLines", za0001)
+					return
+				}
+			}
+		case "MinioEnv":
+			var zb0003 uint32
+			zb0003, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "MinioEnv")
+				return
+			}
+			if z.MinioEnv == nil {
+				z.MinioEnv = make(map[string]string, zb0003)
+			} else if len(z.MinioEnv) > 0 {
+				for key := range z.MinioEnv {
+					delete(z.MinioEnv, key)
+				}
+			}
+			for zb0003 > 0 {
+				zb0003--
+				var za0002 string
+				var za0003 string
+				za0002, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv")
+					return
+				}
+				za0003, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv", za0002)
+					return
+				}
+				z.MinioEnv[za0002] = za0003
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *ServerSystemConfig) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "NEndpoints"
+	err = en.Append(0x83, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.NEndpoints)
+	if err != nil {
+		err = msgp.WrapError(err, "NEndpoints")
+		return
+	}
+	// write "CmdLines"
+	err = en.Append(0xa8, 0x43, 0x6d, 0x64, 0x4c, 0x69, 0x6e, 0x65, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.CmdLines)))
+	if err != nil {
+		err = msgp.WrapError(err, "CmdLines")
+		return
+	}
+	for za0001 := range z.CmdLines {
+		err = en.WriteString(z.CmdLines[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "CmdLines", za0001)
+			return
+		}
+	}
+	// write "MinioEnv"
+	err = en.Append(0xa8, 0x4d, 0x69, 0x6e, 0x69, 0x6f, 0x45, 0x6e, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.MinioEnv)))
+	if err != nil {
+		err = msgp.WrapError(err, "MinioEnv")
+		return
+	}
+	for za0002, za0003 := range z.MinioEnv {
+		err = en.WriteString(za0002)
+		if err != nil {
+			err = msgp.WrapError(err, "MinioEnv")
+			return
+		}
+		err = en.WriteString(za0003)
+		if err != nil {
+			err = msgp.WrapError(err, "MinioEnv", za0002)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *ServerSystemConfig) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "NEndpoints"
+	o = append(o, 0x83, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
+	o = msgp.AppendInt(o, z.NEndpoints)
+	// string "CmdLines"
+	o = append(o, 0xa8, 0x43, 0x6d, 0x64, 0x4c, 0x69, 0x6e, 0x65, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.CmdLines)))
+	for za0001 := range z.CmdLines {
+		o = msgp.AppendString(o, z.CmdLines[za0001])
+	}
+	// string "MinioEnv"
+	o = append(o, 0xa8, 0x4d, 0x69, 0x6e, 0x69, 0x6f, 0x45, 0x6e, 0x76)
+	o = msgp.AppendMapHeader(o, uint32(len(z.MinioEnv)))
+	for za0002, za0003 := range z.MinioEnv {
+		o = msgp.AppendString(o, za0002)
+		o = msgp.AppendString(o, za0003)
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *ServerSystemConfig) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "NEndpoints":
+			z.NEndpoints, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NEndpoints")
+				return
+			}
+		case "CmdLines":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "CmdLines")
+				return
+			}
+			if cap(z.CmdLines) >= int(zb0002) {
+				z.CmdLines = (z.CmdLines)[:zb0002]
+			} else {
+				z.CmdLines = make([]string, zb0002)
+			}
+			for za0001 := range z.CmdLines {
+				z.CmdLines[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "CmdLines", za0001)
+					return
+				}
+			}
+		case "MinioEnv":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "MinioEnv")
+				return
+			}
+			if z.MinioEnv == nil {
+				z.MinioEnv = make(map[string]string, zb0003)
+			} else if len(z.MinioEnv) > 0 {
+				for key := range z.MinioEnv {
+					delete(z.MinioEnv, key)
+				}
+			}
+			for zb0003 > 0 {
+				var za0002 string
+				var za0003 string
+				zb0003--
+				za0002, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv")
+					return
+				}
+				za0003, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv", za0002)
+					return
+				}
+				z.MinioEnv[za0002] = za0003
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *ServerSystemConfig) Msgsize() (s int) {
+	s = 1 + 11 + msgp.IntSize + 9 + msgp.ArrayHeaderSize
+	for za0001 := range z.CmdLines {
+		s += msgp.StringPrefixSize + len(z.CmdLines[za0001])
+	}
+	s += 9 + msgp.MapHeaderSize
+	if z.MinioEnv != nil {
+		for za0002, za0003 := range z.MinioEnv {
+			_ = za0003
+			s += msgp.StringPrefixSize + len(za0002) + msgp.StringPrefixSize + len(za0003)
+		}
+	}
+	return
+}

cmd/bootstrap-peer-server_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalServerSystemConfig(t *testing.T) {
+	v := ServerSystemConfig{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeServerSystemConfig(t *testing.T) {
+	v := ServerSystemConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeServerSystemConfig Msgsize() is inaccurate")
+	}
+
+	vn := ServerSystemConfig{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-encryption-handlers.go

@@ -25,7 +25,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"

cmd/bucket-handlers.go

@@ -424,7 +424,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Content-Md5 is requied should be set
+	// Content-Md5 is required should be set
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
@@ -467,13 +467,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
 
-	// Before proceeding validate if bucket exists.
-	_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
 	deleteObjectsFn := objectAPI.DeleteObjects
 
 	// Return Malformed XML as S3 spec if the number of objects is empty
@@ -611,6 +604,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		VersionSuspended: vc.Suspended(),
 	})
 
+	// Are all objects saying bucket not found?
+	if isAllBucketsNotFound(errs) {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errs[0]), r.URL)
+		return
+	}
+
 	for i := range errs {
 		// DeleteMarkerVersionID is not used specifically to avoid
 		// lookup errors, since DeleteMarkerVersionID is only
@@ -618,7 +617,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// specify a versionID.
 		objToDel := ObjectToDelete{
 			ObjectV: ObjectV{
-				ObjectName: dObjects[i].ObjectName,
+				ObjectName: decodeDirObject(dObjects[i].ObjectName),
 				VersionID:  dObjects[i].VersionID,
 			},
 			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus(),
@@ -1395,7 +1394,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			// Notify object created events.
 			sendEvent(eventArgsList[i])
 
-			if eventArgsList[i].Object.NumVersions > dataScannerExcessiveVersionsThreshold {
+			if eventArgsList[i].Object.NumVersions > int(scannerExcessObjectVersions.Load()) {
 				// Send events for excessive versions.
 				sendEvent(eventArgs{
 					EventName:    event.ObjectManyVersions,
@@ -1406,6 +1405,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					UserAgent:    r.UserAgent() + " " + "MinIO-Fan-Out",
 					Host:         handlers.GetSourceIP(r),
 				})
+
+				auditLogInternal(context.Background(), AuditLogOptions{
+					Event:     "scanner:manyversions",
+					APIName:   "PostPolicyBucket",
+					Bucket:    eventArgsList[i].Object.Bucket,
+					Object:    eventArgsList[i].Object.Name,
+					VersionID: eventArgsList[i].Object.VersionID,
+					Status:    http.StatusText(http.StatusOK),
+				})
 			}
 		}
 
@@ -1461,7 +1469,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		Host:         handlers.GetSourceIP(r),
 	})
 
-	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
+	if objInfo.NumVersions > int(scannerExcessObjectVersions.Load()) {
 		defer sendEvent(eventArgs{
 			EventName:    event.ObjectManyVersions,
 			BucketName:   objInfo.Bucket,
@@ -1471,6 +1479,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			UserAgent:    r.UserAgent(),
 			Host:         handlers.GetSourceIP(r),
 		})
+
+		auditLogInternal(context.Background(), AuditLogOptions{
+			Event:     "scanner:manyversions",
+			APIName:   "PostPolicyBucket",
+			Bucket:    objInfo.Bucket,
+			Object:    objInfo.Name,
+			VersionID: objInfo.VersionID,
+			Status:    http.StatusText(http.StatusOK),
+		})
 	}
 
 	if redirectURL != nil { // success_action_redirect is valid and set.

cmd/bucket-handlers_test.go

@@ -252,7 +252,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		},
 		// Test case - 3.
 		// Testing for signature mismatch error.
-		// setting invalid acess and secret key.
+		// setting invalid access and secret key.
 		{
 			bucketName:         bucketName,
 			accessKey:          "abcd",
@@ -415,7 +415,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			shouldPass:         false,
 		},
 		// Test case - 6.
-		// Setting a negative value to max-uploads paramater, should result in http.StatusBadRequest.
+		// Setting a negative value to max-uploads parameter, should result in http.StatusBadRequest.
 		{
 			bucket:             bucketName,
 			prefix:             "",
@@ -580,7 +580,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 			expectedRespStatus: http.StatusOK,
 		},
 		// Test case - 2.
-		// Test case with invalid accessKey to produce and validate Signature MisMatch error.
+		// Test case with invalid accessKey to produce and validate Signature Mismatch error.
 		{
 			bucketName:         bucketName,
 			accessKey:          "abcd",
@@ -880,6 +880,15 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 			expectedContent:    encodedAnonResponseWithPartialPublicAccess,
 			expectedRespStatus: http.StatusOK,
 		},
+		// Test case - 7.
+		// Bucket does not exist.
+		7: {
+			bucket:             "unknown-bucket-name",
+			objects:            successRequest0,
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
 	}
 
 	for i, testCase := range testCases {
@@ -888,13 +897,12 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Generate a signed or anonymous request based on the testCase
 		if testCase.accessKey != "" {
-			req, err = newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", bucketName),
+			req, err = newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", testCase.bucket),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil)
 		} else {
-			req, err = newTestRequest(http.MethodPost, getDeleteMultipleObjectsURL("", bucketName),
+			req, err = newTestRequest(http.MethodPost, getDeleteMultipleObjectsURL("", testCase.bucket),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
 		}
-
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for DeleteMultipleObjects: <ERROR> %v", err)
 		}

cmd/bucket-lifecycle.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -41,9 +40,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
-	"github.com/minio/pkg/v2/env"
 	xnet "github.com/minio/pkg/v2/net"
-	"github.com/minio/pkg/v2/workers"
 	"github.com/zeebo/xxh3"
 )
 
@@ -104,96 +101,284 @@ type expiryTask struct {
 	src     lcEventSrc
 }
 
+// expiryStats records metrics related to ILM expiry activities
+type expiryStats struct {
+	missedExpiryTasks      atomic.Int64
+	missedFreeVersTasks    atomic.Int64
+	missedTierJournalTasks atomic.Int64
+	workers                atomic.Int32
+}
+
+// MissedTasks returns the number of ILM expiry tasks that were missed since
+// there were no available workers.
+func (e *expiryStats) MissedTasks() int64 {
+	return e.missedExpiryTasks.Load()
+}
+
+// MissedFreeVersTasks returns the number of free version collection tasks that
+// were missed since there were no available workers.
+func (e *expiryStats) MissedFreeVersTasks() int64 {
+	return e.missedFreeVersTasks.Load()
+}
+
+// MissedTierJournalTasks returns the number of tasks to remove tiered objects
+// that were missed since there were no available workers.
+func (e *expiryStats) MissedTierJournalTasks() int64 {
+	return e.missedTierJournalTasks.Load()
+}
+
+// NumWorkers returns the number of active workers executing one of ILM expiry
+// tasks or free version collection tasks.
+func (e *expiryStats) NumWorkers() int32 {
+	return e.workers.Load()
+}
+
+type expiryOp interface {
+	OpHash() uint64
+}
+
+type freeVersionTask struct {
+	ObjectInfo
+}
+
+func (f freeVersionTask) OpHash() uint64 {
+	return xxh3.HashString(f.TransitionedObject.Tier + f.TransitionedObject.Name)
+}
+
+func (n newerNoncurrentTask) OpHash() uint64 {
+	return xxh3.HashString(n.bucket + n.versions[0].ObjectV.ObjectName)
+}
+
+func (j jentry) OpHash() uint64 {
+	return xxh3.HashString(j.TierName + j.ObjName)
+}
+
+func (e expiryTask) OpHash() uint64 {
+	return xxh3.HashString(e.objInfo.Bucket + e.objInfo.Name)
+}
+
+// expiryState manages all ILM related expiration activities.
 type expiryState struct {
-	once                sync.Once
-	byDaysCh            chan expiryTask
-	byNewerNoncurrentCh chan newerNoncurrentTask
+	mu      sync.RWMutex
+	workers atomic.Pointer[[]chan expiryOp]
+
+	ctx    context.Context
+	objAPI ObjectLayer
+
+	stats expiryStats
 }
 
 // PendingTasks returns the number of pending ILM expiry tasks.
 func (es *expiryState) PendingTasks() int {
-	return len(es.byDaysCh) + len(es.byNewerNoncurrentCh)
+	w := es.workers.Load()
+	if w == nil || len(*w) == 0 {
+		return 0
+	}
+	var tasks int
+	for _, wrkr := range *w {
+		tasks += len(wrkr)
+	}
+	return tasks
 }
 
-// close closes work channels exactly once.
-func (es *expiryState) close() {
-	es.once.Do(func() {
-		close(es.byDaysCh)
-		close(es.byNewerNoncurrentCh)
-	})
+// enqueueTierJournalEntry enqueues a tier journal entry referring to a remote
+// object corresponding to a 'replaced' object versions. This applies only to
+// non-versioned or version suspended buckets.
+func (es *expiryState) enqueueTierJournalEntry(je jentry) {
+	wrkr := es.getWorkerCh(je.OpHash())
+	if wrkr == nil {
+		es.stats.missedTierJournalTasks.Add(1)
+		return
+	}
+	select {
+	case <-GlobalContext.Done():
+	case wrkr <- je:
+	default:
+		es.stats.missedTierJournalTasks.Add(1)
+	}
+}
+
+// enqueueFreeVersion enqueues a free version to be deleted
+func (es *expiryState) enqueueFreeVersion(oi ObjectInfo) {
+	task := freeVersionTask{ObjectInfo: oi}
+	wrkr := es.getWorkerCh(task.OpHash())
+	if wrkr == nil {
+		es.stats.missedFreeVersTasks.Add(1)
+		return
+	}
+	select {
+	case <-GlobalContext.Done():
+	case wrkr <- task:
+	default:
+		es.stats.missedFreeVersTasks.Add(1)
+	}
 }
 
 // enqueueByDays enqueues object versions expired by days for expiry.
 func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
+	task := expiryTask{objInfo: oi, event: event, src: src}
+	wrkr := es.getWorkerCh(task.OpHash())
+	if wrkr == nil {
+		es.stats.missedExpiryTasks.Add(1)
+		return
+	}
 	select {
 	case <-GlobalContext.Done():
-		es.close()
-	case es.byDaysCh <- expiryTask{objInfo: oi, event: event, src: src}:
+	case wrkr <- task:
 	default:
+		es.stats.missedExpiryTasks.Add(1)
 	}
 }
 
 // enqueueByNewerNoncurrent enqueues object versions expired by
 // NewerNoncurrentVersions limit for expiry.
 func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete, lcEvent lifecycle.Event) {
+	if len(versions) == 0 {
+		return
+	}
+
+	task := newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}
+	wrkr := es.getWorkerCh(task.OpHash())
+	if wrkr == nil {
+		es.stats.missedExpiryTasks.Add(1)
+		return
+	}
 	select {
 	case <-GlobalContext.Done():
-		es.close()
-	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}:
+	case wrkr <- task:
 	default:
+		es.stats.missedExpiryTasks.Add(1)
 	}
 }
 
+// globalExpiryState is the per-node instance which manages all ILM expiry tasks.
 var globalExpiryState *expiryState
 
-func newExpiryState() *expiryState {
-	return &expiryState{
-		byDaysCh:            make(chan expiryTask, 100000),
-		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 100000),
+// newExpiryState creates an expiryState with buffered channels allocated for
+// each ILM expiry task type.
+func newExpiryState(ctx context.Context, objAPI ObjectLayer, n int) *expiryState {
+	es := &expiryState{
+		ctx:    ctx,
+		objAPI: objAPI,
+	}
+	workers := make([]chan expiryOp, 0, n)
+	es.workers.Store(&workers)
+	es.ResizeWorkers(n)
+	return es
+}
+
+func (es *expiryState) getWorkerCh(h uint64) chan<- expiryOp {
+	w := es.workers.Load()
+	if w == nil || len(*w) == 0 {
+		return nil
+	}
+	workers := *w
+	return workers[h%uint64(len(workers))]
+}
+
+func (es *expiryState) ResizeWorkers(n int) {
+	// Lock to avoid multiple resizes to happen at the same time.
+	es.mu.Lock()
+	defer es.mu.Unlock()
+	var workers []chan expiryOp
+	if v := es.workers.Load(); v != nil {
+		// Copy to new array.
+		workers = append(workers, *v...)
+	}
+
+	if n == len(workers) || n < 1 {
+		return
+	}
+
+	for len(workers) < n {
+		input := make(chan expiryOp, 10000)
+		workers = append(workers, input)
+		go es.Worker(input)
+		es.stats.workers.Add(1)
+	}
+
+	for len(workers) > n {
+		worker := workers[len(workers)-1]
+		workers = workers[:len(workers)-1]
+		worker <- expiryOp(nil)
+		es.stats.workers.Add(-1)
+	}
+	// Atomically replace workers.
+	es.workers.Store(&workers)
+}
+
+// Worker handles 4 types of expiration tasks.
+// 1. Expiry of objects, includes regular and transitioned objects
+// 2. Expiry of noncurrent versions due to NewerNoncurrentVersions
+// 3. Expiry of free-versions, for remote objects of transitioned object which have been expired since.
+// 4. Expiry of remote objects corresponding to objects in a
+// non-versioned/version suspended buckets
+func (es *expiryState) Worker(input <-chan expiryOp) {
+	for {
+		select {
+		case <-es.ctx.Done():
+			return
+		case v, ok := <-input:
+			if !ok {
+				return
+			}
+			if v == nil {
+				// ResizeWorkers signaling worker to quit
+				return
+			}
+			switch v := v.(type) {
+			case expiryTask:
+				if v.objInfo.TransitionedObject.Status != "" {
+					applyExpiryOnTransitionedObject(es.ctx, es.objAPI, v.objInfo, v.event, v.src)
+				} else {
+					applyExpiryOnNonTransitionedObjects(es.ctx, es.objAPI, v.objInfo, v.event, v.src)
+				}
+			case newerNoncurrentTask:
+				deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.event)
+			case jentry:
+				logger.LogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+			case freeVersionTask:
+				oi := v.ObjectInfo
+				traceFn := globalLifecycleSys.trace(oi)
+				if !oi.TransitionedObject.FreeVersion {
+					// nothing to be done
+					return
+				}
+
+				ignoreNotFoundErr := func(err error) error {
+					switch {
+					case isErrVersionNotFound(err), isErrObjectNotFound(err):
+						return nil
+					}
+					return err
+				}
+				// Remove the remote object
+				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
+				if ignoreNotFoundErr(err) != nil {
+					logger.LogIf(es.ctx, err)
+					return
+				}
+
+				// Remove this free version
+				_, err = es.objAPI.DeleteObject(es.ctx, oi.Bucket, oi.Name, ObjectOptions{
+					VersionID:        oi.VersionID,
+					InclFreeVersions: true,
+				})
+				if err == nil {
+					auditLogLifecycle(es.ctx, oi, ILMFreeVersionDelete, nil, traceFn)
+				}
+				if ignoreNotFoundErr(err) != nil {
+					logger.LogIf(es.ctx, err)
+				}
+			default:
+				logger.LogIf(es.ctx, fmt.Errorf("Invalid work type - %v", v))
+			}
+		}
 	}
 }
 
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
-	globalExpiryState = newExpiryState()
-
-	workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
-	if workerSize == 0 {
-		workerSize = 4
-	}
-	ewk, err := workers.New(workerSize)
-	if err != nil {
-		logger.LogIf(ctx, err)
-	}
-
-	nwk, err := workers.New(workerSize)
-	if err != nil {
-		logger.LogIf(ctx, err)
-	}
-
-	go func() {
-		for t := range globalExpiryState.byDaysCh {
-			ewk.Take()
-			go func(t expiryTask) {
-				defer ewk.Give()
-				if t.objInfo.TransitionedObject.Status != "" {
-					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.event, t.src)
-				} else {
-					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.event, t.src)
-				}
-			}(t)
-		}
-		ewk.Wait()
-	}()
-	go func() {
-		for t := range globalExpiryState.byNewerNoncurrentCh {
-			nwk.Take()
-			go func(t newerNoncurrentTask) {
-				defer nwk.Give()
-				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions, t.event)
-			}(t)
-		}
-		nwk.Wait()
-	}()
+	globalExpiryState = newExpiryState(ctx, objectAPI, globalILMConfig.getExpirationWorkers())
 }
 
 // newerNoncurrentTask encapsulates arguments required by worker to expire objects
@@ -257,6 +442,10 @@ func newTransitionState(ctx context.Context) *transitionState {
 // of transition workers.
 func (t *transitionState) Init(objAPI ObjectLayer) {
 	n := globalAPIConfig.getTransitionWorkers()
+	// Prefer ilm.transition_workers over now deprecated api.transition_workers
+	if tw := globalILMConfig.getTransitionWorkers(); tw > 0 {
+		n = tw
+	}
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
@@ -394,12 +583,15 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 //
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
-func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, lcEvent lifecycle.Event, src lcEventSrc) error {
+func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) error {
 	traceFn := globalLifecycleSys.trace(*oi)
-	var opts ObjectOptions
-	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
-	opts.VersionID = lcOpts.VersionID
-	opts.Expiration = ExpirationOptions{Expire: true}
+	opts := ObjectOptions{
+		Versioned:  globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		Expiration: ExpirationOptions{Expire: true},
+	}
+	if lcEvent.Action.DeleteVersioned() {
+		opts.VersionID = oi.VersionID
+	}
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
 	if lcEvent.Action.DeleteRestored() {
 		// delete locally restored copy of object or object version
@@ -414,18 +606,18 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		}
 		return err
 	}
-	// When an object is past expiry or when a transitioned object is being
-	// deleted, 'mark' the data in the remote tier for delete.
-	entry := jentry{
-		ObjName:   oi.TransitionedObject.Name,
-		VersionID: oi.TransitionedObject.VersionID,
-		TierName:  oi.TransitionedObject.Tier,
+
+	// Delete remote object from warm-tier
+	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
+	if err == nil {
+		// Skip adding free version since we successfully deleted the
+		// remote object
+		opts.SkipFreeVersion = true
+	} else {
+		logger.LogIf(ctx, err)
 	}
-	if err := globalTierJournal.AddEntry(entry); err != nil {
-		return err
-	}
-	// Delete metadata on source, now that data in remote tier has been
-	// marked for deletion.
+
+	// Now, delete object from hot-tier namespace
 	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
 		return err
 	}
@@ -434,13 +626,13 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 	defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
 
 	eventName := event.ObjectRemovedDelete
-	if lcOpts.DeleteMarker {
+	if oi.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
 	}
 	objInfo := ObjectInfo{
 		Name:         oi.Name,
-		VersionID:    lcOpts.VersionID,
-		DeleteMarker: lcOpts.DeleteMarker,
+		VersionID:    oi.VersionID,
+		DeleteMarker: oi.DeleteMarker,
 	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
@@ -470,7 +662,14 @@ func genTransitionObjName(bucket string) (string, error) {
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, lae lcAuditEvent) error {
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, lae lcAuditEvent) (err error) {
+	defer func() {
+		if err != nil {
+			return
+		}
+		globalScannerMetrics.timeILM(lae.Action)(1)
+	}()
+
 	opts := ObjectOptions{
 		Transition: TransitionOptions{
 			Status: lifecycle.TransitionPending,

cmd/bucket-listobjects-handlers.go

@@ -111,7 +111,7 @@ func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r
 
 	listObjectVersions := objectAPI.ListObjectVersions
 
-	// Inititate a list object versions operation based on the input params.
+	// Initiate a list object versions operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
 	listObjectVersionsInfo, err := listObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, delimiter, maxkeys)
@@ -201,10 +201,10 @@ func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.Re
 	)
 
 	if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(prefix, archivePattern) {
-		// Inititate a list objects operation inside a zip file based in the input params
+		// Initiate a list objects operation inside a zip file based in the input params
 		listObjectsV2Info, err = listObjectsV2InArchive(ctx, objectAPI, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	} else {
-		// Inititate a list objects operation based on the input params.
+		// Initiate a list objects operation based on the input params.
 		// On success would return back ListObjectsInfo object to be
 		// marshaled into S3 compatible XML header.
 		listObjectsV2Info, err = objectAPI.ListObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
@@ -289,7 +289,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Extract all the litsObjectsV1 query params to their native values.
+	// Extract all the listObjectsV1 query params to their native values.
 	prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.Form)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -304,7 +304,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 
 	listObjects := objectAPI.ListObjects
 
-	// Inititate a list objects operation based on the input params.
+	// Initiate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)

cmd/bucket-metadata-sys.go

@@ -22,6 +22,7 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"math/rand"
 	"sync"
 	"time"
 
@@ -482,11 +483,11 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
-			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
-				// Ensure heal opts for bucket metadata be deep healed all the time.
-				ScanMode: madmin.HealDeepScan,
-				Recreate: true,
-			})
+			// Sleep and stagger to avoid blocked CPU and thundering
+			// herd upon start up sequence.
+			time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond))))
+
+			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{Recreate: true})
 			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index].Name)
 			if err != nil {
 				return err

cmd/bucket-metadata.go

@@ -182,26 +182,34 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
 		b.defaultTimestamps()
 	}
 
-	configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
-	if err != nil {
-		return b, err
+	// If bucket metadata is missing look for legacy files,
+	// since we only ever had b.Created as non-zero when
+	// migration was complete in 2020-May release. So this
+	// a check to avoid migrating for buckets that already
+	// have this field set.
+	if b.Created.IsZero() {
+		configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
+		if err != nil {
+			return b, err
+		}
+
+		if len(configs) > 0 {
+			// Old bucket without bucket metadata. Hence we migrate existing settings.
+			if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
+				return b, err
+			}
+		}
 	}
 
-	if len(configs) == 0 {
-		if parse {
-			// nothing to update, parse and proceed.
-			err = b.parseAllConfigs(ctx, objectAPI)
+	if parse {
+		// nothing to update, parse and proceed.
+		if err = b.parseAllConfigs(ctx, objectAPI); err != nil {
+			return b, err
 		}
-	} else {
-		// Old bucket without bucket metadata. Hence we migrate existing settings.
-		err = b.convertLegacyConfigs(ctx, objectAPI, configs)
-	}
-	if err != nil {
-		return b, err
 	}
 
 	// migrate unencrypted remote targets
-	if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
+	if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
 		return b, err
 	}
 
@@ -331,7 +339,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
 	for _, legacyFile := range legacyConfigs {
 		configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile)
 
-		configData, err := readConfig(ctx, objectAPI, configFile)
+		configData, info, err := readConfigWithMetadata(ctx, objectAPI, configFile, ObjectOptions{})
 		if err != nil {
 			if _, ok := err.(ObjectExistsAsDirectory); ok {
 				// in FS mode it possible that we have actual
@@ -346,6 +354,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
 			return nil, err
 		}
 		configs[legacyFile] = configData
+		b.Created = info.ModTime
 	}
 
 	return configs, nil

cmd/bucket-policy-handlers_test.go

@@ -317,7 +317,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
-		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 		// Call the ServeHTTP to execute the handler.
 		apiRouter.ServeHTTP(recV4, reqV4)
 		if recV4.Code != testCase.expectedRespStatus {
@@ -331,7 +331,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
-		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 		// Call the ServeHTTP to execute the handler.
 		apiRouter.ServeHTTP(recV2, reqV2)
 		if recV2.Code != testCase.expectedRespStatus {
@@ -407,7 +407,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
-		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 		// Call the ServeHTTP to execute the handler.
 		apiRouter.ServeHTTP(recV4, reqV4)
 		if recV4.Code != testPolicy.expectedRespStatus {
@@ -421,7 +421,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
-		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 		// Call the ServeHTTP to execute the handler.
 		apiRouter.ServeHTTP(recV2, reqV2)
 		if recV2.Code != testPolicy.expectedRespStatus {

cmd/bucket-policy.go

@@ -66,6 +66,14 @@ func NewPolicySys() *PolicySys {
 	return &PolicySys{}
 }
 
+func getSTSConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
+	m := make(map[string][]string)
+	if d := r.Form.Get("DurationSeconds"); d != "" {
+		m["DurationSeconds"] = []string{d}
+	}
+	return m
+}
+
 func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
 	currTime := UTCNow()
 

cmd/bucket-quota.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -42,44 +43,40 @@ func NewBucketQuotaSys() *BucketQuotaSys {
 	return &BucketQuotaSys{}
 }
 
-var bucketStorageCache timedValue
+var bucketStorageCache = cachevalue.New[DataUsageInfo]()
 
 // Init initialize bucket quota.
 func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
-	bucketStorageCache.Once.Do(func() {
-		// Set this to 10 secs since its enough, as scanner
-		// does not update the bucket usage values frequently.
-		bucketStorageCache.TTL = 10 * time.Second
-		// Rely on older value if usage loading fails from disk.
-		bucketStorageCache.Relax = true
-		bucketStorageCache.Update = func() (interface{}, error) {
+	bucketStorageCache.InitOnce(10*time.Second,
+		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
+		func() (DataUsageInfo, error) {
 			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
 			defer done()
 
 			return loadDataUsageFromBackend(ctx, objAPI)
-		}
-	})
+		},
+	)
 }
 
 // GetBucketUsageInfo return bucket usage info for a given bucket
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
-	v, err := bucketStorageCache.Get()
+	dui, err := bucketStorageCache.Get()
 	timedout := OperationTimedOut{}
 	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
-		if v != nil {
+		if len(dui.BucketsUsage) > 0 {
 			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
 		} else {
 			logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
 		}
 	}
 
-	var bui BucketUsageInfo
-	dui, ok := v.(DataUsageInfo)
-	if ok {
-		bui = dui.BucketsUsage[bucket]
+	if len(dui.BucketsUsage) > 0 {
+		bui, ok := dui.BucketsUsage[bucket]
+		if ok {
+			return bui, nil
+		}
 	}
-
-	return bui, nil
+	return BucketUsageInfo{}, nil
 }
 
 // parseBucketQuota parses BucketQuota from json

cmd/bucket-replication-metrics.go

@@ -379,3 +379,145 @@ func (s *SMA) simpleMovingAvg() float64 {
 const (
 	defaultWindowSize = 10
 )
+
+type proxyStatsCache struct {
+	srProxyStats ProxyMetric
+	bucketStats  map[string]ProxyMetric
+	sync.RWMutex // mutex for proxy stats
+}
+
+func newProxyStatsCache() proxyStatsCache {
+	return proxyStatsCache{
+		bucketStats: make(map[string]ProxyMetric),
+	}
+}
+
+func (p *proxyStatsCache) inc(bucket string, api replProxyAPI, isErr bool) {
+	p.Lock()
+	defer p.Unlock()
+	v, ok := p.bucketStats[bucket]
+	if !ok {
+		v = ProxyMetric{}
+	}
+	switch api {
+	case putObjectTaggingAPI:
+		if !isErr {
+			atomic.AddUint64(&v.PutTagTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.PutTagTotal, 1)
+		} else {
+			atomic.AddUint64(&v.PutTagFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.PutTagFailedTotal, 1)
+		}
+	case getObjectTaggingAPI:
+		if !isErr {
+			atomic.AddUint64(&v.GetTagTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetTagTotal, 1)
+		} else {
+			atomic.AddUint64(&v.GetTagFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetTagFailedTotal, 1)
+		}
+	case removeObjectTaggingAPI:
+		if !isErr {
+			atomic.AddUint64(&v.RmvTagTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.RmvTagTotal, 1)
+		} else {
+			atomic.AddUint64(&v.RmvTagFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.RmvTagFailedTotal, 1)
+		}
+	case headObjectAPI:
+		if !isErr {
+			atomic.AddUint64(&v.HeadTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.HeadTotal, 1)
+		} else {
+			atomic.AddUint64(&v.HeadFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.HeadFailedTotal, 1)
+		}
+	case getObjectAPI:
+		if !isErr {
+			atomic.AddUint64(&v.GetTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetTotal, 1)
+		} else {
+			atomic.AddUint64(&v.GetFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetFailedTotal, 1)
+		}
+	default:
+		return
+	}
+	p.bucketStats[bucket] = v
+}
+
+func (p *proxyStatsCache) getBucketStats(bucket string) ProxyMetric {
+	p.RLock()
+	defer p.RUnlock()
+	v, ok := p.bucketStats[bucket]
+
+	if !ok {
+		return ProxyMetric{}
+	}
+	return ProxyMetric{
+		PutTagTotal: atomic.LoadUint64(&v.PutTagTotal),
+		GetTagTotal: atomic.LoadUint64(&v.GetTagTotal),
+		RmvTagTotal: atomic.LoadUint64(&v.RmvTagTotal),
+		HeadTotal:   atomic.LoadUint64(&v.HeadTotal),
+		GetTotal:    atomic.LoadUint64(&v.GetTotal),
+
+		PutTagFailedTotal: atomic.LoadUint64(&v.PutTagFailedTotal),
+		GetTagFailedTotal: atomic.LoadUint64(&v.GetTagFailedTotal),
+		RmvTagFailedTotal: atomic.LoadUint64(&v.RmvTagFailedTotal),
+		HeadFailedTotal:   atomic.LoadUint64(&v.HeadFailedTotal),
+		GetFailedTotal:    atomic.LoadUint64(&v.GetFailedTotal),
+	}
+}
+
+func (p *proxyStatsCache) getSiteStats() ProxyMetric {
+	v := p.srProxyStats
+	return ProxyMetric{
+		PutTagTotal:       atomic.LoadUint64(&v.PutTagTotal),
+		GetTagTotal:       atomic.LoadUint64(&v.GetTagTotal),
+		RmvTagTotal:       atomic.LoadUint64(&v.RmvTagTotal),
+		HeadTotal:         atomic.LoadUint64(&v.HeadTotal),
+		GetTotal:          atomic.LoadUint64(&v.GetTotal),
+		PutTagFailedTotal: atomic.LoadUint64(&v.PutTagFailedTotal),
+		GetTagFailedTotal: atomic.LoadUint64(&v.GetTagFailedTotal),
+		RmvTagFailedTotal: atomic.LoadUint64(&v.RmvTagFailedTotal),
+		HeadFailedTotal:   atomic.LoadUint64(&v.HeadFailedTotal),
+		GetFailedTotal:    atomic.LoadUint64(&v.GetFailedTotal),
+	}
+}
+
+type replProxyAPI string
+
+const (
+	putObjectTaggingAPI    replProxyAPI = "PutObjectTagging"
+	getObjectTaggingAPI    replProxyAPI = "GetObjectTagging"
+	removeObjectTaggingAPI replProxyAPI = "RemoveObjectTagging"
+	headObjectAPI          replProxyAPI = "HeadObject"
+	getObjectAPI           replProxyAPI = "GetObject"
+)
+
+// ProxyMetric holds stats for replication proxying
+type ProxyMetric struct {
+	PutTagTotal       uint64 `json:"putTaggingProxyTotal" msg:"ptc"`
+	GetTagTotal       uint64 `json:"getTaggingProxyTotal" msg:"gtc"`
+	RmvTagTotal       uint64 `json:"removeTaggingProxyTotal" msg:"rtc"`
+	GetTotal          uint64 `json:"getProxyTotal" msg:"gc"`
+	HeadTotal         uint64 `json:"headProxyTotal" msg:"hc"`
+	PutTagFailedTotal uint64 `json:"putTaggingProxyFailed" msg:"ptf"`
+	GetTagFailedTotal uint64 `json:"getTaggingProxyFailed" msg:"gtf"`
+	RmvTagFailedTotal uint64 `json:"removeTaggingProxyFailed" msg:"rtf"`
+	GetFailedTotal    uint64 `json:"getProxyFailed" msg:"gf"`
+	HeadFailedTotal   uint64 `json:"headProxyFailed" msg:"hf"`
+}
+
+func (p *ProxyMetric) add(p2 ProxyMetric) {
+	atomic.AddUint64(&p.GetTotal, p2.GetTotal)
+	atomic.AddUint64(&p.HeadTotal, p2.HeadTotal)
+	atomic.AddUint64(&p.GetTagTotal, p2.GetTagTotal)
+	atomic.AddUint64(&p.PutTagTotal, p2.PutTagTotal)
+	atomic.AddUint64(&p.RmvTagTotal, p2.RmvTagTotal)
+	atomic.AddUint64(&p.GetFailedTotal, p2.GetFailedTotal)
+	atomic.AddUint64(&p.HeadFailedTotal, p2.HeadFailedTotal)
+	atomic.AddUint64(&p.GetTagFailedTotal, p2.GetTagFailedTotal)
+	atomic.AddUint64(&p.PutTagFailedTotal, p2.PutTagFailedTotal)
+	atomic.AddUint64(&p.RmvTagFailedTotal, p2.RmvTagFailedTotal)
+}

cmd/bucket-replication-metrics_gen.go

@@ -584,6 +584,7 @@ func (z *InQueueStats) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -595,6 +596,7 @@ func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) {
 func (z InQueueStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }
@@ -635,6 +637,334 @@ func (z InQueueStats) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *ProxyMetric) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ptc":
+			z.PutTagTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagTotal")
+				return
+			}
+		case "gtc":
+			z.GetTagTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagTotal")
+				return
+			}
+		case "rtc":
+			z.RmvTagTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagTotal")
+				return
+			}
+		case "gc":
+			z.GetTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetTotal")
+				return
+			}
+		case "hc":
+			z.HeadTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "HeadTotal")
+				return
+			}
+		case "ptf":
+			z.PutTagFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagFailedTotal")
+				return
+			}
+		case "gtf":
+			z.GetTagFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagFailedTotal")
+				return
+			}
+		case "rtf":
+			z.RmvTagFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagFailedTotal")
+				return
+			}
+		case "gf":
+			z.GetFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetFailedTotal")
+				return
+			}
+		case "hf":
+			z.HeadFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "HeadFailedTotal")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *ProxyMetric) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 10
+	// write "ptc"
+	err = en.Append(0x8a, 0xa3, 0x70, 0x74, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.PutTagTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "PutTagTotal")
+		return
+	}
+	// write "gtc"
+	err = en.Append(0xa3, 0x67, 0x74, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetTagTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetTagTotal")
+		return
+	}
+	// write "rtc"
+	err = en.Append(0xa3, 0x72, 0x74, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.RmvTagTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "RmvTagTotal")
+		return
+	}
+	// write "gc"
+	err = en.Append(0xa2, 0x67, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetTotal")
+		return
+	}
+	// write "hc"
+	err = en.Append(0xa2, 0x68, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.HeadTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "HeadTotal")
+		return
+	}
+	// write "ptf"
+	err = en.Append(0xa3, 0x70, 0x74, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.PutTagFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "PutTagFailedTotal")
+		return
+	}
+	// write "gtf"
+	err = en.Append(0xa3, 0x67, 0x74, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetTagFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetTagFailedTotal")
+		return
+	}
+	// write "rtf"
+	err = en.Append(0xa3, 0x72, 0x74, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.RmvTagFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "RmvTagFailedTotal")
+		return
+	}
+	// write "gf"
+	err = en.Append(0xa2, 0x67, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetFailedTotal")
+		return
+	}
+	// write "hf"
+	err = en.Append(0xa2, 0x68, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.HeadFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "HeadFailedTotal")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *ProxyMetric) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 10
+	// string "ptc"
+	o = append(o, 0x8a, 0xa3, 0x70, 0x74, 0x63)
+	o = msgp.AppendUint64(o, z.PutTagTotal)
+	// string "gtc"
+	o = append(o, 0xa3, 0x67, 0x74, 0x63)
+	o = msgp.AppendUint64(o, z.GetTagTotal)
+	// string "rtc"
+	o = append(o, 0xa3, 0x72, 0x74, 0x63)
+	o = msgp.AppendUint64(o, z.RmvTagTotal)
+	// string "gc"
+	o = append(o, 0xa2, 0x67, 0x63)
+	o = msgp.AppendUint64(o, z.GetTotal)
+	// string "hc"
+	o = append(o, 0xa2, 0x68, 0x63)
+	o = msgp.AppendUint64(o, z.HeadTotal)
+	// string "ptf"
+	o = append(o, 0xa3, 0x70, 0x74, 0x66)
+	o = msgp.AppendUint64(o, z.PutTagFailedTotal)
+	// string "gtf"
+	o = append(o, 0xa3, 0x67, 0x74, 0x66)
+	o = msgp.AppendUint64(o, z.GetTagFailedTotal)
+	// string "rtf"
+	o = append(o, 0xa3, 0x72, 0x74, 0x66)
+	o = msgp.AppendUint64(o, z.RmvTagFailedTotal)
+	// string "gf"
+	o = append(o, 0xa2, 0x67, 0x66)
+	o = msgp.AppendUint64(o, z.GetFailedTotal)
+	// string "hf"
+	o = append(o, 0xa2, 0x68, 0x66)
+	o = msgp.AppendUint64(o, z.HeadFailedTotal)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *ProxyMetric) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ptc":
+			z.PutTagTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagTotal")
+				return
+			}
+		case "gtc":
+			z.GetTagTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagTotal")
+				return
+			}
+		case "rtc":
+			z.RmvTagTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagTotal")
+				return
+			}
+		case "gc":
+			z.GetTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetTotal")
+				return
+			}
+		case "hc":
+			z.HeadTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "HeadTotal")
+				return
+			}
+		case "ptf":
+			z.PutTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagFailedTotal")
+				return
+			}
+		case "gtf":
+			z.GetTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagFailedTotal")
+				return
+			}
+		case "rtf":
+			z.RmvTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagFailedTotal")
+				return
+			}
+		case "gf":
+			z.GetFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetFailedTotal")
+				return
+			}
+		case "hf":
+			z.HeadFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "HeadFailedTotal")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *ProxyMetric) Msgsize() (s int) {
+	s = 1 + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *QStat) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/bucket-replication-metrics_gen_test.go

@@ -348,6 +348,119 @@ func BenchmarkDecodeInQueueStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalProxyMetric(t *testing.T) {
+	v := ProxyMetric{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeProxyMetric(t *testing.T) {
+	v := ProxyMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeProxyMetric Msgsize() is inaccurate")
+	}
+
+	vn := ProxyMetric{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalQStat(t *testing.T) {
 	v := QStat{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-replication-stats.go

@@ -45,6 +45,8 @@ type ReplicationStats struct {
 	workers *ActiveWorkerStat
 	// queue stats cache
 	qCache queueCache
+
+	pCache proxyStatsCache
 	// mrf backlog stats
 	mrfStats ReplicationMRFStats
 	// for bucket replication, continue to use existing cache
@@ -305,6 +307,7 @@ func (r *ReplicationStats) getSRMetricsForNode() SRMetricsSummary {
 		Queued:        r.qCache.getSiteStats(),
 		ActiveWorkers: r.ActiveWorkers(),
 		Metrics:       r.srStats.get(),
+		Proxied:       r.pCache.getSiteStats(),
 		ReplicaSize:   atomic.LoadInt64(&r.srStats.ReplicaSize),
 		ReplicaCount:  atomic.LoadInt64(&r.srStats.ReplicaCount),
 	}
@@ -333,6 +336,7 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 	rs := ReplicationStats{
 		Cache:           make(map[string]*BucketReplicationStats),
 		qCache:          newQueueCache(r),
+		pCache:          newProxyStatsCache(),
 		srStats:         newSRStats(),
 		movingAvgTicker: time.NewTicker(2 * time.Second),
 		wTimer:          time.NewTicker(2 * time.Second),
@@ -371,6 +375,7 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucket
 				Stats: make(map[string]*BucketReplicationStat),
 			},
 			QueueStats: ReplicationQueueStats{},
+			ProxyStats: ProxyMetric{},
 		}
 		return bs
 	}
@@ -430,11 +435,16 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucket
 	for _, bs := range bucketStats {
 		qs.Nodes = append(qs.Nodes, bs.QueueStats.Nodes...)
 	}
-
 	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
+
+	var ps ProxyMetric
+	for _, bs := range bucketStats {
+		ps.add(bs.ProxyStats)
+	}
 	bs = BucketStats{
 		ReplicationStats: s,
 		QueueStats:       qs,
+		ProxyStats:       ps,
 	}
 	r.mostRecentStatsMu.Lock()
 	if len(r.mostRecentStats.Stats) == 0 {
@@ -454,7 +464,7 @@ func (r *ReplicationStats) getLatestReplicationStats(bucket string) (s BucketSta
 	return r.calculateBucketReplicationStats(bucket, bucketStats)
 }
 
-func (r *ReplicationStats) incQ(bucket string, sz int64, isDeleleRepl bool, opType replication.Type) {
+func (r *ReplicationStats) incQ(bucket string, sz int64, isDeleteRepl bool, opType replication.Type) {
 	r.qCache.Lock()
 	defer r.qCache.Unlock()
 	v, ok := r.qCache.bucketStats[bucket]
@@ -482,3 +492,12 @@ func (r *ReplicationStats) decQ(bucket string, sz int64, isDelMarker bool, opTyp
 	atomic.AddInt64(&r.qCache.srQueueStats.nowBytes, -1*sz)
 	atomic.AddInt64(&r.qCache.srQueueStats.nowCount, -1)
 }
+
+// incProxy increments proxy metrics for proxied calls
+func (r *ReplicationStats) incProxy(bucket string, api replProxyAPI, isErr bool) {
+	r.pCache.inc(bucket, api, isErr)
+}
+
+func (r *ReplicationStats) getProxyStats(bucket string) ProxyMetric {
+	return r.pCache.getBucketStats(bucket)
+}

cmd/bucket-replication-utils.go

@@ -219,7 +219,7 @@ type ReplicateDecision struct {
 	targetsMap map[string]replicateTargetDecision
 }
 
-// ReplicateAny returns true if atleast one target qualifies for replication
+// ReplicateAny returns true if at least one target qualifies for replication
 func (d ReplicateDecision) ReplicateAny() bool {
 	for _, t := range d.targetsMap {
 		if t.Replicate {
@@ -229,7 +229,7 @@ func (d ReplicateDecision) ReplicateAny() bool {
 	return false
 }
 
-// Synchronous returns true if atleast one target qualifies for synchronous replication
+// Synchronous returns true if at least one target qualifies for synchronous replication
 func (d ReplicateDecision) Synchronous() bool {
 	for _, t := range d.targetsMap {
 		if t.Synchronous {
@@ -336,7 +336,7 @@ type ReplicationState struct {
 	DeleteMarker              bool                   // represents DeleteMarker replication state
 	ReplicationTimeStamp      time.Time              // timestamp when last replication activity happened
 	ReplicationStatusInternal string                 // stringified representation of all replication activity
-	// VersionPurgeStatusInternal is internally in the format "arn1=PENDING;arn2=COMMPLETED;"
+	// VersionPurgeStatusInternal is internally in the format "arn1=PENDING;arn2=COMPLETED;"
 	VersionPurgeStatusInternal string                            // stringified representation of all version purge statuses
 	ReplicateDecisionStr       string                            // stringified representation of replication decision for each target
 	Targets                    map[string]replication.StatusType // map of ARN->replication status for ongoing replication activity

cmd/bucket-replication-utils_gen.go

@@ -749,6 +749,7 @@ func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -760,6 +761,7 @@ func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) {
 func (z ReplicateDecision) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }
@@ -1388,6 +1390,7 @@ func (z *ResyncDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -1399,6 +1402,7 @@ func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) {
 func (z ResyncDecision) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }

cmd/bucket-replication.go

@@ -47,6 +47,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
@@ -178,7 +179,7 @@ func checkRemoteEndpoint(ctx context.Context, epURL *url.URL) error {
 	}
 
 	client := &http.Client{
-		Transport: NewHTTPTransport(),
+		Transport: globalRemoteTargetTransport,
 		Timeout:   10 * time.Second,
 	}
 
@@ -495,7 +496,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		}
 		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtEntry.Arn)
 		if tgtClnt == nil {
-			// Skip stale targets if any and log them to be missing atleast once.
+			// Skip stale targets if any and log them to be missing at least once.
 			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn)
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
@@ -683,7 +684,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		} else {
 			rinfo.VersionPurgeStatus = Failed
 		}
-		logger.LogIf(ctx, fmt.Errorf("Unable to replicate delete marker to %s/%s(%s): %s", tgt.Bucket, dobj.ObjectName, versionID, rmErr))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
 		if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
 		}
@@ -1204,7 +1205,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty for target %s", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1230,7 +1231,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 
 	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s err:%w", bucket, err))
+		logger.LogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1265,14 +1266,14 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			r, objInfo, putOpts); rinfo.Err != nil {
 			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
 			}
 		}
 	} else {
 		if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
 			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
 			}
 		}
 	}
@@ -1336,7 +1337,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to replicate to target %s for %s/%s(%s): %w", tgt.EndpointURL(), bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
@@ -1367,7 +1368,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1397,7 +1398,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if rAction == replicateNone {
 			if ri.OpType == replication.ExistingObjectReplicationType &&
 				objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target %s", bucket, object, tgt.EndpointURL()))
 				sendEvent(eventArgs{
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
@@ -1487,13 +1488,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		if _, rinfo.Err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); rinfo.Err != nil {
 			rinfo.ReplicationStatus = replication.Failed
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
 		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s err:%w", bucket, err))
+			logger.LogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1527,7 +1528,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				r, objInfo, putOpts); rinfo.Err != nil {
 				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1536,7 +1537,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
 				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1562,6 +1563,9 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		if err == nil {
 			break
 		}
+		if minio.ToErrorResponse(err).Code == "PreconditionFailed" {
+			return err
+		}
 		attempts++
 		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 	}
@@ -1894,7 +1898,7 @@ func (p *ReplicationPool) AddLargeWorkers() {
 	go func() {
 		<-p.ctx.Done()
 		for i := 0; i < LargeWorkerCount; i++ {
-			close(p.lrgworkers[i])
+			xioutil.SafeClose(p.lrgworkers[i])
 		}
 	}()
 }
@@ -1953,7 +1957,7 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 	for len(p.workers) > n {
 		worker := p.workers[len(p.workers)-1]
 		p.workers = p.workers[:len(p.workers)-1]
-		close(worker)
+		xioutil.SafeClose(worker)
 	}
 }
 
@@ -2234,6 +2238,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
 		return nil, oi, proxy
 	}
+	var perr error
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
 		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
@@ -2263,6 +2268,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 
 		objInfo, err := tgt.StatObject(ctx, t.TargetBucket, object, gopts)
 		if err != nil {
+			perr = err
 			if isErrInvalidRange(ErrorRespToObjectError(err, bucket, object)) {
 				return nil, oi, proxyResult{Err: err}
 			}
@@ -2299,6 +2305,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 		}
 		return tgt, oi, proxyResult{Proxy: true}
 	}
+	proxy.Err = perr
 	return nil, oi, proxy
 }
 
@@ -2405,8 +2412,7 @@ func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *t
 			taggedCount++
 			continue
 		}
-		errCode := minio.ToErrorResponse(err).Code
-		if errCode != "NoSuchKey" && errCode != "NoSuchVersion" {
+		if err != nil {
 			terr = err
 		}
 	}
@@ -2466,6 +2472,9 @@ func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts
 			tgs, _ = tags.MapToObjectTags(tagSlc[idx])
 		}
 	}
+	if len(errs) == 1 {
+		proxy.Err = errs[0]
+	}
 	return tgs, proxy
 }
 
@@ -2619,7 +2628,7 @@ func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI Objec
 				}
 				if updt {
 					if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("Could not save resync metadata to drive for %s - %w", bucket, err))
+						logger.LogIf(ctx, fmt.Errorf("could not save resync metadata to drive for %s - %w", bucket, err))
 					} else {
 						lastResyncStatusSave[bucket] = brs.LastUpdate
 					}
@@ -2755,7 +2764,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	}
 	workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
 	resultCh := make(chan TargetReplicationResyncStatus, 1)
-	defer close(resultCh)
+	defer xioutil.SafeClose(resultCh)
 	go func() {
 		for r := range resultCh {
 			s.incStats(r, opts)
@@ -2867,7 +2876,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 		}
 	}
 	for i := 0; i < resyncParallelRoutines; i++ {
-		close(workers[i])
+		xioutil.SafeClose(workers[i])
 	}
 	wg.Wait()
 	resyncStatus = ResyncCompleted
@@ -2997,7 +3006,7 @@ func (p *ReplicationPool) startResyncRoutine(ctx context.Context, buckets []Buck
 		}
 		duration := time.Duration(r.Float64() * float64(time.Minute))
 		if duration < time.Second {
-			// Make sure to sleep atleast a second to avoid high CPU ticks.
+			// Make sure to sleep at least a second to avoid high CPU ticks.
 			duration = time.Second
 		}
 		time.Sleep(duration)
@@ -3123,7 +3132,7 @@ func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string,
 	}
 	diffCh := make(chan madmin.DiffInfo, 4000)
 	go func() {
-		defer close(diffCh)
+		defer xioutil.SafeClose(diffCh)
 		for obj := range objInfoCh {
 			if contextCanceled(ctx) {
 				// Just consume input...
@@ -3212,6 +3221,9 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 		return roi
 	}
 
+	if isVeeamSOSAPIObject(oi.Name) {
+		return roi
+	}
 	if rcfg.Config == nil || rcfg.remotes == nil {
 		return roi
 	}
@@ -3316,7 +3328,7 @@ func (p *ReplicationPool) persistMRF() {
 			mTimer.Reset(mrfSaveInterval)
 		case <-p.ctx.Done():
 			p.mrfStopCh <- struct{}{}
-			close(p.mrfSaveCh)
+			xioutil.SafeClose(p.mrfSaveCh)
 			// We try to save if possible, but we don't care beyond that.
 			saveMRFToDisk()
 			return
@@ -3384,12 +3396,12 @@ func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntr
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := globalLocalDrives
+	localDrives := cloneDrives(globalLocalDrives)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
 		r := newReader()
-		err := localDrive.CreateFile(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), -1, r)
+		err := localDrive.CreateFile(ctx, "", minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), -1, r)
 		r.Close()
 		if err == nil {
 			break
@@ -3451,7 +3463,7 @@ func (p *ReplicationPool) loadMRF() (mrfRec MRFReplicateEntries, err error) {
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := globalLocalDrives
+	localDrives := cloneDrives(globalLocalDrives)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
@@ -3551,7 +3563,7 @@ func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch <-chan
 
 	mrfCh := make(chan madmin.ReplicationMRF, 100)
 	go func() {
-		defer close(mrfCh)
+		defer xioutil.SafeClose(mrfCh)
 		for vID, e := range mrfRec.Entries {
 			if bucket != "" && e.Bucket != bucket {
 				continue

cmd/bucket-stats.go

@@ -154,6 +154,7 @@ type BucketStats struct {
 	Uptime           int64                  `json:"uptime"`
 	ReplicationStats BucketReplicationStats `json:"currStats"`  // current replication stats since cluster startup
 	QueueStats       ReplicationQueueStats  `json:"queueStats"` // replication queue stats
+	ProxyStats       ProxyMetric            `json:"proxyStats"`
 }
 
 // BucketReplicationStats represents inline replication statistics

cmd/bucket-stats_gen.go

@@ -1142,6 +1142,12 @@ func (z *BucketStats) DecodeMsg(dc *msgp.Reader) (err error) {
 					}
 				}
 			}
+		case "ProxyStats":
+			err = z.ProxyStats.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "ProxyStats")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -1155,9 +1161,9 @@ func (z *BucketStats) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
+	// map header, size 4
 	// write "Uptime"
-	err = en.Append(0x83, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
+	err = en.Append(0x84, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -1209,15 +1215,25 @@ func (z *BucketStats) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "QueueStats", "Uptime")
 		return
 	}
+	// write "ProxyStats"
+	err = en.Append(0xaa, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = z.ProxyStats.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "ProxyStats")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
+	// map header, size 4
 	// string "Uptime"
-	o = append(o, 0x83, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
+	o = append(o, 0x84, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
 	o = msgp.AppendInt64(o, z.Uptime)
 	// string "ReplicationStats"
 	o = append(o, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
@@ -1242,6 +1258,13 @@ func (z *BucketStats) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Uptime"
 	o = append(o, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
 	o = msgp.AppendInt64(o, z.QueueStats.Uptime)
+	// string "ProxyStats"
+	o = append(o, 0xaa, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73)
+	o, err = z.ProxyStats.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "ProxyStats")
+		return
+	}
 	return
 }
 
@@ -1323,6 +1346,12 @@ func (z *BucketStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					}
 				}
 			}
+		case "ProxyStats":
+			bts, err = z.ProxyStats.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ProxyStats")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -1341,7 +1370,7 @@ func (z *BucketStats) Msgsize() (s int) {
 	for za0001 := range z.QueueStats.Nodes {
 		s += z.QueueStats.Nodes[za0001].Msgsize()
 	}
-	s += 7 + msgp.Int64Size
+	s += 7 + msgp.Int64Size + 11 + z.ProxyStats.Msgsize()
 	return
 }
 

cmd/build-constants.go

@@ -17,6 +17,8 @@
 
 package cmd
 
+import "runtime"
+
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
@@ -40,4 +42,28 @@ var (
 
 	// CopyrightYear - dynamic value of the copyright end year
 	CopyrightYear = "0000"
+
+	// MinioReleaseTagTimeLayout - release tag time layout.
+	MinioReleaseTagTimeLayout = "2006-01-02T15-04-05Z"
+
+	// MinioOSARCH - OS and ARCH.
+	minioOSARCH = runtime.GOOS + "-" + runtime.GOARCH
+
+	// MinioReleaseBaseURL - release url without os and arch.
+	MinioReleaseBaseURL = "https://dl.min.io/server/minio/release/"
+
+	// MinioReleaseURL - release URL.
+	MinioReleaseURL = MinioReleaseBaseURL + minioOSARCH + SlashSeparator
+
+	// MinioStoreName - MinIO store name.
+	MinioStoreName = "MinIO"
+
+	// MinioUAName - MinIO user agent name.
+	MinioUAName = "MinIO"
+
+	// MinioBannerName - MinIO banner name for startup message.
+	MinioBannerName = "MinIO Object Storage Server"
+
+	// MinioLicense - MinIO server license.
+	MinioLicense = "GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"
 )

cmd/callhome.go

@@ -60,7 +60,7 @@ func initCallhome(ctx context.Context, objAPI ObjectLayer) {
 			// sleep for some time and try again.
 			duration := time.Duration(r.Float64() * float64(globalCallhomeConfig.FrequencyDur()))
 			if duration < time.Second {
-				// Make sure to sleep atleast a second to avoid high CPU ticks.
+				// Make sure to sleep at least a second to avoid high CPU ticks.
 				duration = time.Second
 			}
 			time.Sleep(duration)

cmd/common-main.go

@@ -27,7 +27,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"math/rand"
 	"net"
 	"net/url"
 	"os"
@@ -50,7 +49,7 @@ import (
 	"github.com/minio/console/api/operations"
 	consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2"
 	consoleCerts "github.com/minio/console/pkg/certs"
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -69,7 +68,7 @@ import (
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
 
-var shardDiskTimeDelta time.Duration
+var currentReleaseTime time.Time
 
 func init() {
 	if runtime.GOOS == "windows" {
@@ -83,13 +82,9 @@ func init() {
 		}
 	}
 
-	rand.Seed(time.Now().UTC().UnixNano())
-
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	initGlobalContext()
-
 	globalBatchJobsMetrics = batchJobMetrics{metrics: make(map[string]*batchJobInfo)}
 	go globalBatchJobsMetrics.purgeJobMetrics()
 
@@ -105,18 +100,15 @@ func init() {
 	gob.Register(StorageErr(""))
 	gob.Register(madmin.TimeInfo{})
 	gob.Register(madmin.XFSErrorConfigs{})
+	gob.Register(map[string]string{})
 	gob.Register(map[string]interface{}{})
 
-	var err error
-	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
-	if err != nil {
-		shardDiskTimeDelta = 1 * time.Minute
-	}
-
 	// All minio-go and madmin-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
 	madmin.MaxRetry = 1
+
+	currentReleaseTime, _ = GetCurrentReleaseTime()
 }
 
 const consolePrefix = "CONSOLE_"
@@ -301,9 +293,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	// Its OK to ignore any errors during doUpdate() here.
-	crTime, err := GetCurrentReleaseTime()
-	if err != nil {
+	if currentReleaseTime.IsZero() {
 		return
 	}
 
@@ -314,8 +304,8 @@ func checkUpdate(mode string) {
 
 	var older time.Duration
 	var downloadURL string
-	if lrTime.After(crTime) {
-		older = lrTime.Sub(crTime)
+	if lrTime.After(currentReleaseTime) {
+		older = lrTime.Sub(currentReleaseTime)
 		downloadURL = getDownloadURL(releaseTimeToReleaseTag(lrTime))
 	}
 
@@ -324,7 +314,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
+	logger.Info(prepareUpdateMessage("Run `mc admin update ALIAS`", lrTime.Sub(currentReleaseTime)))
 }
 
 func newConfigDir(dir string, dirSet bool, getDefaultDir func() string) (*ConfigDir, error) {
@@ -371,9 +361,16 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 		ctxt.ConsoleAddr = ctx.String("console-address")
 	}
 
+	if cxml := ctx.String("crossdomain-xml"); cxml != "" {
+		buf, err := os.ReadFile(cxml)
+		if err != nil {
+			return err
+		}
+		ctxt.CrossDomainXML = string(buf)
+	}
+
 	// Check "no-compat" flag from command line argument.
 	ctxt.StrictS3Compat = !(ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat"))
-	ctxt.PreAllocate = ctx.IsSet("pre-allocate") || ctx.GlobalIsSet("pre-allocate")
 
 	switch {
 	case ctx.IsSet("config-dir"):
@@ -400,10 +397,13 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
 	ctxt.ConnReadDeadline = ctx.Duration("conn-read-deadline")
 	ctxt.ConnWriteDeadline = ctx.Duration("conn-write-deadline")
+	ctxt.ConnClientReadDeadline = ctx.Duration("conn-client-read-deadline")
+	ctxt.ConnClientWriteDeadline = ctx.Duration("conn-client-write-deadline")
 
 	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
 	ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
+	ctxt.MaxIdleConnsPerHost = ctx.Int("max-idle-conns-per-host")
 
 	if conf := ctx.String("config"); len(conf) > 0 {
 		err = mergeServerCtxtFromConfigFile(conf, ctxt)
@@ -757,7 +757,12 @@ func serverHandleEnvVars() {
 		for _, endpoint := range minioEndpoints {
 			if net.ParseIP(endpoint) == nil {
 				// Checking if the IP is a DNS entry.
-				addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint)
+				lookupHost := globalDNSCache.LookupHost
+				if IsKubernetes() || IsDocker() {
+					lookupHost = net.DefaultResolver.LookupHost
+				}
+
+				addrs, err := lookupHost(GlobalContext, endpoint)
 				if err != nil {
 					logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
 				}
@@ -847,7 +852,7 @@ func loadRootCredentials() {
 // It depends on KMS env variables and global cli flags.
 func handleKMSConfig() {
 	if env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint) {
-		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
+		logger.Fatal(errors.New("ambiguous KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
 	}
 
 	if env.IsSet(kms.EnvKMSSecretKey) {
@@ -860,10 +865,10 @@ func handleKMSConfig() {
 	if env.IsSet(kms.EnvKESEndpoint) {
 		if env.IsSet(kms.EnvKESAPIKey) {
 			if env.IsSet(kms.EnvKESClientKey) {
-				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientKey))
+				logger.Fatal(errors.New("ambiguous KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientKey))
 			}
 			if env.IsSet(kms.EnvKESClientCert) {
-				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientCert))
+				logger.Fatal(errors.New("ambiguous KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientCert))
 			}
 		}
 		if !env.IsSet(kms.EnvKESKeyName) {
@@ -900,7 +905,6 @@ func handleKMSConfig() {
 			}
 			kmsConf = kms.Config{
 				Endpoints:    endpoints,
-				Enclave:      env.Get(kms.EnvKESEnclave, ""),
 				DefaultKeyID: env.Get(kms.EnvKESKeyName, ""),
 				APIKey:       key,
 				RootCAs:      rootCAs,
@@ -946,7 +950,6 @@ func handleKMSConfig() {
 
 			kmsConf = kms.Config{
 				Endpoints:        endpoints,
-				Enclave:          env.Get(kms.EnvKESEnclave, ""),
 				DefaultKeyID:     env.Get(kms.EnvKESKeyName, ""),
 				Certificate:      certificate,
 				ReloadCertEvents: reloadCertEvents,
@@ -1040,7 +1043,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		}
 		if err = manager.AddCertificate(certFile, keyFile); err != nil {
 			err = fmt.Errorf("Unable to load TLS certificate '%s,%s': %w", certFile, keyFile, err)
-			logger.LogIf(GlobalContext, err, logger.Minio)
+			logger.LogIf(GlobalContext, err, logger.ErrorKind)
 		}
 	}
 	secureConn = true

cmd/config-current.go

@@ -18,13 +18,17 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"strings"
 	"sync"
 
+	"github.com/minio/kms-go/kes"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/browser"
+	"github.com/minio/minio/internal/kms"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
@@ -41,6 +45,7 @@ import (
 	"github.com/minio/minio/internal/config/identity/openid"
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
+	"github.com/minio/minio/internal/config/ilm"
 	"github.com/minio/minio/internal/config/lambda"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
@@ -74,6 +79,7 @@ func initHelp() {
 		config.SubnetSubSys:         subnet.DefaultKVS,
 		config.CallhomeSubSys:       callhome.DefaultKVS,
 		config.DriveSubSys:          drive.DefaultKVS,
+		config.ILMSubSys:            ilm.DefaultKVS,
 		config.CacheSubSys:          cache.DefaultKVS,
 		config.BatchSubSys:          batch.DefaultKVS,
 		config.BrowserSubSys:        browser.DefaultKVS,
@@ -499,7 +505,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 			if globalDNSConfig != nil {
-				// if global DNS is already configured, indicate with a warning, incase
+				// if global DNS is already configured, indicate with a warning, in case
 				// users are confused.
 				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
 			} else {
@@ -569,6 +575,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		}
 
 		globalAPIConfig.init(apiConfig, setDriveCounts)
+		autoGenerateRootCredentials() // Generate the KMS root credentials here since we don't know whether API root access is disabled until now.
 
 		// Initialize remote instance transport once.
 		getRemoteInstanceTransportOnce.Do(func() {
@@ -602,6 +609,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		// update dynamic scanner values.
 		scannerIdleMode.Store(scannerCfg.IdleMode)
 		scannerCycle.Store(scannerCfg.Cycle)
+		scannerExcessObjectVersions.Store(scannerCfg.ExcessVersions)
+		scannerExcessFolders.Store(scannerCfg.ExcessFolders)
 		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.LoggerWebhookSubSys)
@@ -709,6 +718,18 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			return fmt.Errorf("Unable to apply browser config: %w", err)
 		}
 		globalBrowserConfig.Update(browserCfg)
+	case config.ILMSubSys:
+		ilmCfg, err := ilm.LookupConfig(s[config.ILMSubSys][config.Default])
+		if err != nil {
+			return fmt.Errorf("Unable to apply ilm config: %w", err)
+		}
+		if globalTransitionState != nil {
+			globalTransitionState.UpdateWorkers(ilmCfg.TransitionWorkers)
+		}
+		if globalExpiryState != nil {
+			globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
+		}
+		globalILMConfig.update(ilmCfg)
 	}
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()
@@ -718,6 +739,55 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 	return nil
 }
 
+// autoGenerateRootCredentials generates root credentials deterministically if
+// a KMS is configured, no manual credentials have been specified and if root
+// access is disabled.
+func autoGenerateRootCredentials() {
+	if GlobalKMS == nil {
+		return
+	}
+	if globalAPIConfig.permitRootAccess() || !globalActiveCred.Equal(auth.DefaultCredentials) {
+		return
+	}
+
+	if manager, ok := GlobalKMS.(kms.KeyManager); ok {
+		stat, err := GlobalKMS.Stat(GlobalContext)
+		if err != nil {
+			logger.LogIf(GlobalContext, err, "Unable to generate root credentials using KMS")
+			return
+		}
+
+		aKey, err := manager.HMAC(GlobalContext, stat.DefaultKey, []byte("root access key"))
+		if errors.Is(err, kes.ErrNotAllowed) {
+			return // If we don't have permission to compute the HMAC, don't change the cred.
+		}
+		if err != nil {
+			logger.Fatal(err, "Unable to generate root access key using KMS")
+		}
+
+		sKey, err := manager.HMAC(GlobalContext, stat.DefaultKey, []byte("root secret key"))
+		if err != nil {
+			// Here, we must have permission. Otherwise, we would have failed earlier.
+			logger.Fatal(err, "Unable to generate root secret key using KMS")
+		}
+
+		accessKey, err := auth.GenerateAccessKey(20, bytes.NewReader(aKey))
+		if err != nil {
+			logger.Fatal(err, "Unable to generate root access key")
+		}
+		secretKey, err := auth.GenerateSecretKey(32, bytes.NewReader(sKey))
+		if err != nil {
+			logger.Fatal(err, "Unable to generate root secret key")
+		}
+
+		logger.Info("Automatically generated root access key and secret key with the KMS")
+		globalActiveCred = auth.Credentials{
+			AccessKey: accessKey,
+			SecretKey: secretKey,
+		}
+	}
+}
+
 // applyDynamicConfig will apply dynamic config values.
 // Dynamic systems should be in config.SubSystemsDynamic as well.
 func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {

cmd/consolelogger.go

@@ -196,7 +196,7 @@ func (sys *HTTPConsoleLoggerSys) Send(ctx context.Context, entry interface{}) er
 	sys.logBuf.Value = lg
 	sys.logBuf = sys.logBuf.Next()
 	sys.Unlock()
-	err := sys.console.Send(entry, string(logger.All))
+	err := sys.console.Send(entry)
 	if err != nil {
 		atomic.AddInt64(&sys.failedMessages, 1)
 	}

cmd/crossdomain-xml-handler.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,10 +32,14 @@ const crossDomainXMLEntity = "/crossdomain.xml"
 // policy file that grants access to the source domain, allowing the client to continue the transaction.
 func setCrossDomainPolicyMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		cxml := crossDomainXML
+		if globalServerCtxt.CrossDomainXML != "" {
+			cxml = globalServerCtxt.CrossDomainXML
+		}
 		// Look for 'crossdomain.xml' in the incoming request.
 		if r.URL.Path == crossDomainXMLEntity {
 			// Write the standard cross domain policy xml.
-			w.Write([]byte(crossDomainXML))
+			w.Write([]byte(cxml))
 			// Request completed, no need to serve to other handlers.
 			return
 		}

cmd/data-scanner.go

@@ -40,6 +40,7 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/heal"
 	"github.com/minio/minio/internal/event"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	uatomic "go.uber.org/atomic"
@@ -56,18 +57,17 @@ const (
 
 	healDeleteDangling   = true
 	healObjectSelectProb = 1024 // Overall probability of a file being scanned; one in n.
-
-	dataScannerExcessiveVersionsThreshold = 100   // Issue a warning when a single object has more versions than this
-	dataScannerExcessiveFoldersThreshold  = 50000 // Issue a warning when a folder has more subfolders than this in a *set*
 )
 
 var (
 	globalHealConfig heal.Config
 
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper  = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
-	scannerCycle    = uatomic.NewDuration(dataScannerStartDelay)
-	scannerIdleMode = uatomic.NewInt32(0) // default is throttled when idle
+	scannerSleeper              = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
+	scannerCycle                = uatomic.NewDuration(dataScannerStartDelay)
+	scannerIdleMode             = uatomic.NewInt32(0) // default is throttled when idle
+	scannerExcessObjectVersions = uatomic.NewInt64(100)
+	scannerExcessFolders        = uatomic.NewInt64(50000)
 )
 
 // initDataScanner will start the scanner in the background.
@@ -79,7 +79,7 @@ func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			runDataScanner(ctx, objAPI)
 			duration := time.Duration(r.Float64() * float64(scannerCycle.Load()))
 			if duration < time.Second {
-				// Make sure to sleep atleast a second to avoid high CPU ticks.
+				// Make sure to sleep at least a second to avoid high CPU ticks.
 				duration = time.Second
 			}
 			time.Sleep(duration)
@@ -529,18 +529,27 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
 			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
 
-		if len(existingFolders)+len(newFolders) > dataScannerExcessiveFoldersThreshold {
-			// Notify object accessed via a GET request.
+		if totalFolders := len(existingFolders) + len(newFolders); totalFolders > int(scannerExcessFolders.Load()) {
+			prefixName := strings.TrimSuffix(folder.name, "/") + "/"
 			sendEvent(eventArgs{
 				EventName:  event.PrefixManyFolders,
 				BucketName: f.root,
 				Object: ObjectInfo{
-					Name: strings.TrimSuffix(folder.name, "/") + "/",
-					Size: int64(len(existingFolders) + len(newFolders)),
+					Name: prefixName,
+					Size: int64(totalFolders),
 				},
-				UserAgent: "scanner",
+				UserAgent: "Scanner",
 				Host:      globalMinioHost,
 			})
+			auditLogInternal(context.Background(), AuditLogOptions{
+				Event:   "scanner:manyprefixes",
+				APIName: "Scanner",
+				Bucket:  f.root,
+				Object:  prefixName,
+				Tags: map[string]interface{}{
+					"x-minio-prefixes-total": strconv.Itoa(totalFolders),
+				},
+			})
 		}
 		if !into.Compacted && shouldCompact {
 			into.Compacted = true
@@ -710,7 +719,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				partial: func(entries metaCacheEntries, errs []error) {
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
-						// check if we can get one entry atleast
+						// check if we can get one entry at least
 						// proceed to heal nonetheless, since
 						// this object might be dangling.
 						entry, _ = entries.firstFound()
@@ -954,15 +963,6 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
 		}
 	}
-	defer func() {
-		if lcEvt.Action != lifecycle.NoneAction {
-			numVersions := uint64(1)
-			if lcEvt.Action == lifecycle.DeleteAllVersionsAction {
-				numVersions = uint64(oi.NumVersions)
-			}
-			globalScannerMetrics.timeILM(lcEvt.Action)(numVersions)
-		}
-	}()
 
 	switch lcEvt.Action {
 	// This version doesn't contribute towards sizeS only when it is permanently deleted.
@@ -982,45 +982,9 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 	return lcEvt.Action, size
 }
 
-// applyTierObjSweep removes remote object pending deletion and the free-version
-// tracking this information.
-func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
-	traceFn := globalLifecycleSys.trace(oi)
-	if !oi.TransitionedObject.FreeVersion {
-		// nothing to be done
-		return
-	}
-
-	ignoreNotFoundErr := func(err error) error {
-		switch {
-		case isErrVersionNotFound(err), isErrObjectNotFound(err):
-			return nil
-		}
-		return err
-	}
-	// Remove the remote object
-	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
-	if ignoreNotFoundErr(err) != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
-
-	// Remove this free version
-	_, err = o.DeleteObject(ctx, oi.Bucket, oi.Name, ObjectOptions{
-		VersionID:        oi.VersionID,
-		InclFreeVersions: true,
-	})
-	if err == nil {
-		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, nil, traceFn)
-	}
-	if ignoreNotFoundErr(err) != nil {
-		logger.LogIf(ctx, err)
-	}
-}
-
 // applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo, expState *expiryState) ([]ObjectInfo, error) {
 	done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
 	defer done()
 
@@ -1087,20 +1051,22 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		})
 	}
 
-	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
+	if len(toDel) > 0 {
+		expState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
+	}
 	return objectInfos, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
-func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
-	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo, expState *expiryState) ([]ObjectInfo, error) {
+	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs, expState)
 	if err != nil {
 		return nil, err
 	}
 
 	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
-	if len(objInfos) > dataScannerExcessiveVersionsThreshold {
+	if len(objInfos) > int(scannerExcessObjectVersions.Load()) {
 		// Notify object accessed via a GET request.
 		sendEvent(eventArgs{
 			EventName:  event.ObjectManyVersions,
@@ -1108,9 +1074,19 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			Object: ObjectInfo{
 				Name: i.objectPath(),
 			},
-			UserAgent:    "Internal: [Scanner]",
+			UserAgent:    "Scanner",
 			Host:         globalLocalNodeName,
-			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(fivs))},
+			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(objInfos))},
+		})
+
+		auditLogInternal(context.Background(), AuditLogOptions{
+			Event:   "scanner:manyversions",
+			APIName: "Scanner",
+			Bucket:  i.bucket,
+			Object:  i.objectPath(),
+			Tags: map[string]interface{}{
+				"x-minio-versions": strconv.Itoa(len(objInfos)),
+			},
 		})
 	}
 
@@ -1209,7 +1185,17 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 }
 
 func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) bool {
-	if err := expireTransitionedObject(ctx, objLayer, &obj, obj.ToLifecycleOpts(), lcEvent, src); err != nil {
+	var err error
+	defer func() {
+		if err != nil {
+			return
+		}
+		// Note: DeleteAllVersions action is not supported for
+		// transitioned objects
+		globalScannerMetrics.timeILM(lcEvent.Action)(1)
+	}()
+
+	if err = expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
@@ -1236,8 +1222,25 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	if lcEvent.Action.DeleteAll() {
 		opts.DeletePrefix = true
 	}
+	var (
+		dobj ObjectInfo
+		err  error
+	)
+	defer func() {
+		if err != nil {
+			return
+		}
 
-	obj, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
+		if lcEvent.Action != lifecycle.NoneAction {
+			numVersions := uint64(1)
+			if lcEvent.Action == lifecycle.DeleteAllVersionsAction {
+				numVersions = uint64(obj.NumVersions)
+			}
+			globalScannerMetrics.timeILM(lcEvent.Action)(numVersions)
+		}
+	}()
+
+	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
@@ -1246,21 +1249,26 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		logger.LogOnceIf(ctx, err, "non-transition-expiry")
 		return false
 	}
+	if dobj.Name == "" {
+		dobj = obj
+	}
 
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
 	// Send audit for the lifecycle delete operation
-	auditLogLifecycle(ctx, obj, ILMExpiry, tags, traceFn)
+	auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
 
 	eventName := event.ObjectRemovedDelete
 	if obj.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
 	}
-
+	if lcEvent.Action.DeleteAll() {
+		eventName = event.ObjectRemovedDeleteAllVersions
+	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,
-		BucketName: obj.Bucket,
-		Object:     obj,
+		BucketName: dobj.Bucket,
+		Object:     dobj,
 		UserAgent:  "Internal: [ILM-Expiry]",
 		Host:       globalLocalNodeName,
 	})
@@ -1471,7 +1479,7 @@ func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) {
 }
 
 // Update the current settings and cycle all waiting.
-// Parameters are the same as in the contructor.
+// Parameters are the same as in the constructor.
 func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error {
 	d.mu.Lock()
 	defer d.mu.Unlock()
@@ -1479,7 +1487,7 @@ func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error {
 		return nil
 	}
 	// Update values and cycle waiting.
-	close(d.cycle)
+	xioutil.SafeClose(d.cycle)
 	d.factor = factor
 	d.maxSleep = maxWait
 	d.cycle = make(chan struct{})

cmd/data-scanner_test.go

@@ -39,14 +39,20 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 	globalBucketMetadataSys = NewBucketMetadataSys()
 	globalBucketObjectLockSys = &BucketObjectLockSys{}
 	globalBucketVersioningSys = &BucketVersioningSys{}
-	globalExpiryState = newExpiryState()
+	es := newExpiryState(context.Background(), objAPI, 0)
+	workers := []chan expiryOp{make(chan expiryOp)}
+	es.workers.Store(&workers)
+	globalExpiryState = es
 	var wg sync.WaitGroup
 	wg.Add(1)
 	expired := make([]ObjectToDelete, 0, 5)
 	go func() {
 		defer wg.Done()
-		for t := range globalExpiryState.byNewerNoncurrentCh {
-			expired = append(expired, t.versions...)
+		workers := globalExpiryState.workers.Load()
+		for t := range (*workers)[0] {
+			if t, ok := t.(newerNoncurrentTask); ok {
+				expired = append(expired, t.versions...)
+			}
 		}
 	}()
 	lc := lifecycle.Lifecycle{
@@ -116,7 +122,7 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 	for i, fi := range fivs[:2] {
 		wants[i] = fi.ToObjectInfo(bucket, obj, versioned)
 	}
-	gots, err := item.applyNewerNoncurrentVersionLimit(context.TODO(), objAPI, fivs)
+	gots, err := item.applyNewerNoncurrentVersionLimit(context.TODO(), objAPI, fivs, es)
 	if err != nil {
 		t.Fatalf("Failed with err: %v", err)
 	}
@@ -125,7 +131,7 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 	}
 
 	// Close expiry state's channel to inspect object versions enqueued for expiration
-	close(globalExpiryState.byNewerNoncurrentCh)
+	close(workers[0])
 	wg.Wait()
 	for _, obj := range expired {
 		switch obj.ObjectV.VersionID {

cmd/data-usage-cache.go

@@ -980,7 +980,7 @@ type objectIO interface {
 // The loader is optimistic and has no locking, but tries 5 times before giving up.
 // If the object is not found, a nil error with empty data usage cache is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
-	// By defaut, empty data usage cache
+	// By default, empty data usage cache
 	*d = dataUsageCache{}
 
 	load := func(name string, timeout time.Duration) (bool, error) {

cmd/data-usage-cache_gen.go

@@ -1762,7 +1762,7 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(10)
 	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
@@ -1866,7 +1866,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not omitted
 		// write "rs"
 		err = en.Append(0xa2, 0x72, 0x73)
 		if err != nil {
@@ -1885,7 +1885,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x100) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not omitted
 		// write "ats"
 		err = en.Append(0xa3, 0x61, 0x74, 0x73)
 		if err != nil {
@@ -1920,7 +1920,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(10)
 	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
@@ -1968,7 +1968,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0002 := range z.ObjVersions {
 		o = msgp.AppendUint64(o, z.ObjVersions[za0002])
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not omitted
 		// string "rs"
 		o = append(o, 0xa2, 0x72, 0x73)
 		if z.ReplicationStats == nil {
@@ -1981,7 +1981,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x100) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not omitted
 		// string "ats"
 		o = append(o, 0xa3, 0x61, 0x74, 0x73)
 		if z.AllTierStats == nil {
@@ -3243,7 +3243,7 @@ func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(3)
 	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
@@ -3267,7 +3267,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if zb0001Len == 0 {
 		return
 	}
-	if (zb0001Mask & 0x1) == 0 { // if not empty
+	if (zb0001Mask & 0x1) == 0 { // if not omitted
 		// write "t"
 		err = en.Append(0xa1, 0x74)
 		if err != nil {
@@ -3291,7 +3291,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x2) == 0 { // if not empty
+	if (zb0001Mask & 0x2) == 0 { // if not omitted
 		// write "r"
 		err = en.Append(0xa1, 0x72)
 		if err != nil {
@@ -3303,7 +3303,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x4) == 0 { // if not empty
+	if (zb0001Mask & 0x4) == 0 { // if not omitted
 		// write "rc"
 		err = en.Append(0xa2, 0x72, 0x63)
 		if err != nil {
@@ -3321,7 +3321,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(3)
 	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
@@ -3342,7 +3342,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	if zb0001Len == 0 {
 		return
 	}
-	if (zb0001Mask & 0x1) == 0 { // if not empty
+	if (zb0001Mask & 0x1) == 0 { // if not omitted
 		// string "t"
 		o = append(o, 0xa1, 0x74)
 		o = msgp.AppendMapHeader(o, uint32(len(z.Targets)))
@@ -3355,12 +3355,12 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x2) == 0 { // if not empty
+	if (zb0001Mask & 0x2) == 0 { // if not omitted
 		// string "r"
 		o = append(o, 0xa1, 0x72)
 		o = msgp.AppendUint64(o, z.ReplicaSize)
 	}
-	if (zb0001Mask & 0x4) == 0 { // if not empty
+	if (zb0001Mask & 0x4) == 0 { // if not omitted
 		// string "rc"
 		o = append(o, 0xa2, 0x72, 0x63)
 		o = msgp.AppendUint64(o, z.ReplicaCount)
@@ -3478,6 +3478,8 @@ func (z *replicationAllStatsV1) DecodeMsg(dc *msgp.Reader) (err error) {
 			delete(z.Targets, key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0002 > 0 {
 		zb0002--
 		var za0001 string
@@ -3588,6 +3590,8 @@ func (z *replicationAllStatsV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			delete(z.Targets, key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0002 > 0 {
 		var za0001 string
 		var za0002 replicationStats

cmd/data-usage-utils.go

@@ -71,6 +71,10 @@ type BucketUsageInfo struct {
 
 // DataUsageInfo represents data usage stats of the underlying Object API
 type DataUsageInfo struct {
+	TotalCapacity     uint64 `json:"capacity,omitempty"`
+	TotalUsedCapacity uint64 `json:"usedCapacity,omitempty"`
+	TotalFreeCapacity uint64 `json:"freeCapacity,omitempty"`
+
 	// LastUpdate is the timestamp of when the data usage info was last updated.
 	// This does not indicate a full scan.
 	LastUpdate time.Time `json:"lastUpdate"`
@@ -87,6 +91,7 @@ type DataUsageInfo struct {
 	// Objects total size across all buckets
 	ObjectsTotalSize uint64                           `json:"objectsTotalSize"`
 	ReplicationInfo  map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+
 	// Total number of buckets in this cluster
 	BucketsCount uint64 `json:"bucketsCount"`
 
@@ -135,7 +140,7 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 	return infos
 }
 
-func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
+func (dui DataUsageInfo) tierMetrics() (metrics []MetricV2) {
 	if dui.TierStats == nil {
 		return nil
 	}
@@ -143,17 +148,17 @@ func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
 	//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
 	//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
 	for tier, st := range dui.TierStats.Tiers {
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedBytesMD(),
 			Value:          float64(st.TotalSize),
 			VariableLabels: map[string]string{"tier": tier},
 		})
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedObjectsMD(),
 			Value:          float64(st.NumObjects),
 			VariableLabels: map[string]string{"tier": tier},
 		})
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedVersionsMD(),
 			Value:          float64(st.NumVersions),
 			VariableLabels: map[string]string{"tier": tier},

cmd/data-usage.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -41,7 +42,7 @@ const (
 	dataUsageCacheName = ".usage-cache.bin"
 )
 
-// storeDataUsageInBackend will store all objects sent on the gui channel until closed.
+// storeDataUsageInBackend will store all objects sent on the dui channel until closed.
 func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
 	attempts := 1
 	for dataUsageInfo := range dui {
@@ -62,7 +63,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 	}
 }
 
-var prefixUsageCache timedValue
+var prefixUsageCache = cachevalue.New[map[string]uint64]()
 
 // loadPrefixUsageFromBackend returns prefix usages found in passed buckets
 //
@@ -76,12 +77,10 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 
 	cache := dataUsageCache{}
 
-	prefixUsageCache.Once.Do(func() {
-		prefixUsageCache.TTL = 30 * time.Second
-
+	prefixUsageCache.InitOnce(30*time.Second,
 		// No need to fail upon Update() error, fallback to old value.
-		prefixUsageCache.Relax = true
-		prefixUsageCache.Update = func() (interface{}, error) {
+		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
+		func() (map[string]uint64, error) {
 			m := make(map[string]uint64)
 			for _, pool := range z.serverPools {
 				for _, er := range pool.sets {
@@ -106,15 +105,10 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 				}
 			}
 			return m, nil
-		}
-	})
+		},
+	)
 
-	v, _ := prefixUsageCache.Get()
-	if v != nil {
-		return v.(map[string]uint64), nil
-	}
-
-	return map[string]uint64{}, nil
+	return prefixUsageCache.Get()
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
@@ -155,7 +149,9 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 			bui.ReplicationFailedSizeV1 > 0 || bui.ReplicationPendingCountV1 > 0 {
 			cfg, _ := getReplicationConfig(GlobalContext, bucket)
 			if cfg != nil && cfg.RoleArn != "" {
-				dataUsageInfo.ReplicationInfo = make(map[string]BucketTargetUsageInfo)
+				if dataUsageInfo.ReplicationInfo == nil {
+					dataUsageInfo.ReplicationInfo = make(map[string]BucketTargetUsageInfo)
+				}
 				dataUsageInfo.ReplicationInfo[cfg.RoleArn] = BucketTargetUsageInfo{
 					ReplicationFailedSize:   bui.ReplicationFailedSizeV1,
 					ReplicationFailedCount:  bui.ReplicationFailedCountV1,

cmd/encryption-v1.go

@@ -34,7 +34,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
@@ -664,7 +664,7 @@ func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
 	mac.Write(partIDbin[:])
 	partEncryptionKey := mac.Sum(nil)
 
-	// Limit the reader, so the decryptor doesnt receive bytes
+	// Limit the reader, so the decryptor doesn't receive bytes
 	// from the next part (different DARE stream)
 	encLenToRead := d.parts[d.partIndex].Size - d.partEncRelOffset
 	decrypter, err := newDecryptReaderWithObjectKey(io.LimitReader(d.reader, encLenToRead), partEncryptionKey, d.startSeqNum)
@@ -751,7 +751,7 @@ func (o ObjectInfo) DecryptedSize() (int64, error) {
 // However, DecryptETag does not try to decrypt the ETag if
 // it consists of a 128 bit hex value (32 hex chars) and exactly
 // one '-' followed by a 32-bit number.
-// This special case adresses randomly-generated ETags generated
+// This special case addresses randomly-generated ETags generated
 // by the MinIO server when running in non-compat mode. These
 // random ETags are not encrypt.
 //

cmd/endpoint.go

@@ -62,8 +62,9 @@ type ProxyEndpoint struct {
 // Node holds information about a node in this cluster
 type Node struct {
 	*url.URL
-	Pools   []int
-	IsLocal bool
+	Pools    []int
+	IsLocal  bool
+	GridHost string
 }
 
 // Endpoint - any type of endpoint.
@@ -74,6 +75,16 @@ type Endpoint struct {
 	PoolIdx, SetIdx, DiskIdx int
 }
 
+// Equal returns true if endpoint == ep
+func (endpoint Endpoint) Equal(ep Endpoint) bool {
+	if endpoint.IsLocal == ep.IsLocal && endpoint.PoolIdx == ep.PoolIdx && endpoint.SetIdx == ep.SetIdx && endpoint.DiskIdx == ep.DiskIdx {
+		if endpoint.Path == ep.Path && endpoint.Host == ep.Host {
+			return true
+		}
+	}
+	return false
+}
+
 func (endpoint Endpoint) String() string {
 	if endpoint.Host == "" {
 		return endpoint.Path
@@ -254,6 +265,7 @@ func (l EndpointServerPools) GetNodes() (nodes []Node) {
 					Scheme: ep.Scheme,
 					Host:   ep.Host,
 				}
+				node.GridHost = ep.GridHost()
 			}
 			if !slices.Contains(node.Pools, ep.PoolIdx) {
 				node.Pools = append(node.Pools, ep.PoolIdx)
@@ -386,6 +398,95 @@ func (l EndpointServerPools) NEndpoints() (count int) {
 	return count
 }
 
+// GridHosts will return all peers, including local.
+// in websocket grid compatible format, The local peer
+// is returned as a separate string.
+func (l EndpointServerPools) GridHosts() (gridHosts []string, gridLocal string) {
+	seenHosts := set.NewStringSet()
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			u := endpoint.GridHost()
+			if seenHosts.Contains(u) {
+				continue
+			}
+			seenHosts.Add(u)
+
+			// Set local endpoint
+			if endpoint.IsLocal {
+				gridLocal = u
+			}
+
+			gridHosts = append(gridHosts, u)
+		}
+	}
+
+	return gridHosts, gridLocal
+}
+
+// FindGridHostsFromPeerPool will return a matching peerPool from provided peer (as string)
+func (l EndpointServerPools) FindGridHostsFromPeerPool(peer string) []int {
+	if peer == "" {
+		return nil
+	}
+
+	var pools []int
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				continue
+			}
+
+			if !slices.Contains(pools, endpoint.PoolIdx) {
+				pools = append(pools, endpoint.PoolIdx)
+			}
+		}
+	}
+
+	return pools
+}
+
+// FindGridHostsFromPeerStr will return a matching peer from provided peer (as string)
+func (l EndpointServerPools) FindGridHostsFromPeerStr(peer string) (peerGrid string) {
+	if peer == "" {
+		return ""
+	}
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				continue
+			}
+
+			if endpoint.Host == peer {
+				return endpoint.GridHost()
+			}
+		}
+	}
+	return ""
+}
+
+// FindGridHostsFromPeer will return a matching peer from provided peer.
+func (l EndpointServerPools) FindGridHostsFromPeer(peer *xnet.Host) (peerGrid string) {
+	if peer == nil {
+		return ""
+	}
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				continue
+			}
+			host, err := xnet.ParseHost(endpoint.Host)
+			if err != nil {
+				continue
+			}
+
+			if host.String() == peer.String() {
+				return endpoint.GridHost()
+			}
+		}
+	}
+	return ""
+}
+
 // Hostnames - returns list of unique hostnames
 func (l EndpointServerPools) Hostnames() []string {
 	foundSet := set.NewStringSet()
@@ -435,7 +536,9 @@ func (l EndpointServerPools) peers() (peers []string, local string) {
 			peer := endpoint.Host
 			if endpoint.IsLocal {
 				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
-					local = peer
+					if local == "" {
+						local = peer
+					}
 				}
 			}
 
@@ -474,13 +577,6 @@ func (endpoints Endpoints) GetAllStrings() (all []string) {
 func hostResolveToLocalhost(endpoint Endpoint) bool {
 	hostIPs, err := getHostIP(endpoint.Hostname())
 	if err != nil {
-		// Log the message to console about the host resolving
-		reqInfo := (&logger.ReqInfo{}).AppendTags(
-			"host",
-			endpoint.Hostname(),
-		)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
 		return false
 	}
 	var loopback int
@@ -552,7 +648,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
 						logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
-							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.Application)
+							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.ErrorKind)
 					}
 
 					continue
@@ -581,7 +677,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.Application)
+						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.ErrorKind)
 					}
 				} else {
 					resolvedList[i] = true
@@ -746,7 +842,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
 							logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
-								endpoint.Hostname()), endpoint.Hostname(), logger.Application)
+								endpoint.Hostname()), endpoint.Hostname(), logger.ErrorKind)
 						}
 						continue
 					}
@@ -774,7 +870,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 								))
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
-							logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
+							logger.LogOnceIf(ctx, fmt.Errorf("Unable to resolve DNS for %s: %w", endpoint, err), endpoint.Hostname(), logger.ErrorKind)
 						}
 					} else {
 						resolvedList[endpoint] = true

cmd/erasure-coding.go

@@ -80,11 +80,9 @@ func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error)
 	}
 	encoded, err := e.encoder().Split(data)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return nil, err
 	}
 	if err = e.encoder().Encode(encoded); err != nil {
-		logger.LogIf(ctx, err)
 		return nil, err
 	}
 	return encoded, nil
@@ -111,11 +109,7 @@ func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
 // DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
 // It returns an error if the decoding failed.
 func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
-	if err := e.encoder().Reconstruct(data); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	return nil
+	return e.encoder().Reconstruct(data)
 }
 
 // ShardSize - returns actual shared size from erasure blockSize.

cmd/erasure-common.go

@@ -42,7 +42,7 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i].DiskInfo(context.Background(), false)
+			di, err := disks[i].DiskInfo(context.Background(), DiskInfoOptions{})
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.

cmd/erasure-decode.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/minio/minio/internal/logger"
+	xioutil "github.com/minio/minio/internal/ioutil"
 )
 
 // Reads in parallel from readers.
@@ -118,7 +118,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 	}
 
 	readTriggerCh := make(chan bool, len(p.readers))
-	defer close(readTriggerCh) // close the channel upon return
+	defer xioutil.SafeClose(readTriggerCh) // close the channel upon return
 
 	for i := 0; i < p.dataBlocks; i++ {
 		// Setup read triggers for p.dataBlocks number of reads so that it reads in parallel.
@@ -158,7 +158,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 			bufIdx := p.readerToBuf[i]
 			if p.buf[bufIdx] == nil {
 				// Reading first time on this disk, hence the buffer needs to be allocated.
-				// Subsequent reads will re-use this buffer.
+				// Subsequent reads will reuse this buffer.
 				p.buf[bufIdx] = make([]byte, p.shardSize)
 			}
 			// For the last shard, the shardsize might be less than previous shard sizes.
@@ -210,11 +210,9 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 // A set of preferred drives can be supplied. In that case they will be used and the data reconstructed.
 func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64, prefer []bool) (written int64, derr error) {
 	if offset < 0 || length < 0 {
-		logger.LogIf(ctx, errInvalidArgument)
 		return -1, errInvalidArgument
 	}
 	if offset+length > totalLength {
-		logger.LogIf(ctx, errInvalidArgument)
 		return -1, errInvalidArgument
 	}
 
@@ -268,7 +266,6 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 		}
 
 		if err = e.DecodeDataBlocks(bufs); err != nil {
-			logger.LogIf(ctx, err)
 			return -1, err
 		}
 
@@ -281,7 +278,6 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 	}
 
 	if bytesWritten != length {
-		logger.LogIf(ctx, errLessData)
 		return bytesWritten, errLessData
 	}
 
@@ -320,7 +316,6 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 		}
 
 		if err = e.DecodeDataAndParityBlocks(ctx, bufs); err != nil {
-			logger.LogOnceIf(ctx, err, "erasure-heal-decode")
 			return err
 		}
 
@@ -331,7 +326,6 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 		}
 
 		if err = w.Write(ctx, bufs); err != nil {
-			logger.LogOnceIf(ctx, err, "erasure-heal-write")
 			return err
 		}
 	}

cmd/erasure-decode_test.go

@@ -106,7 +106,7 @@ func TestErasureDecode(t *testing.T) {
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -144,7 +144,7 @@ func TestErasureDecode(t *testing.T) {
 		}
 		if err == nil {
 			if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) {
-				t.Errorf("Test %d: read retruns wrong file content.", i)
+				t.Errorf("Test %d: read returns wrong file content.", i)
 			}
 		}
 
@@ -228,7 +228,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	// 10000 iterations with random offsets and lengths.
@@ -297,7 +297,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	content := make([]byte, size)

cmd/erasure-encode.go

@@ -22,9 +22,6 @@ import (
 	"fmt"
 	"io"
 	"sync"
-
-	"github.com/minio/minio/internal/hash"
-	"github.com/minio/minio/internal/logger"
 )
 
 // Writes in parallel to writers
@@ -92,29 +89,26 @@ func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer
 				io.EOF,
 				io.ErrUnexpectedEOF,
 			}...) {
-				if !hash.IsChecksumMismatch(err) {
-					logger.LogIf(ctx, err)
-				}
 				return 0, err
 			}
 		}
+
 		eof := err == io.EOF || err == io.ErrUnexpectedEOF
 		if n == 0 && total != 0 {
 			// Reached EOF, nothing more to be done.
 			break
 		}
+
 		// We take care of the situation where if n == 0 and total == 0 by creating empty data and parity files.
 		blocks, err = e.EncodeData(ctx, buf[:n])
 		if err != nil {
-			logger.LogIf(ctx, err)
-
 			return 0, err
 		}
 
 		if err = writer.Write(ctx, blocks); err != nil {
-			logger.LogIf(ctx, err)
 			return 0, err
 		}
+
 		total += int64(n)
 		if eof {
 			break

cmd/erasure-encode_test.go

@@ -41,7 +41,7 @@ func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset
 	return nil, errFaultyDisk
 }
 
-func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
+func (a badDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error {
 	return errFaultyDisk
 }
 
@@ -103,7 +103,7 @@ func TestErasureEncode(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -127,7 +127,7 @@ func TestErasureEncode(t *testing.T) {
 				if disk == nil {
 					continue
 				}
-				writers[i] = newBitrotWriter(disk, "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
+				writers[i] = newBitrotWriter(disk, "", "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
 				switch w := writers[j].(type) {
@@ -192,7 +192,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 				Recursive: false,
 				Immediate: false,
 			})
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)

cmd/erasure-heal_test.go

@@ -86,7 +86,7 @@ func TestErasureHeal(t *testing.T) {
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -128,7 +128,7 @@ func TestErasureHeal(t *testing.T) {
 				continue
 			}
 			os.Remove(pathJoin(disk.String(), "testbucket", "testobject"))
-			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
+			staleWriters[i] = newBitrotWriter(disk, "", "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 
 		// test case setup is complete - now call Heal()

cmd/erasure-healing-common.go

@@ -58,8 +58,8 @@ func commonETags(etags []string) (etag string, maxima int) {
 }
 
 // commonTime returns a maximally occurring time from a list of time.
-func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime time.Time, maxima int) {
-	timeOccurenceMap := make(map[int64]int, len(times))
+func commonTimeAndOccurrence(times []time.Time, group time.Duration) (maxTime time.Time, maxima int) {
+	timeOccurrenceMap := make(map[int64]int, len(times))
 	groupNano := group.Nanoseconds()
 	// Ignore the uuid sentinel and count the rest.
 	for _, t := range times {
@@ -68,7 +68,7 @@ func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime tim
 		}
 		nano := t.UnixNano()
 		if group > 0 {
-			for k := range timeOccurenceMap {
+			for k := range timeOccurrenceMap {
 				if k == nano {
 					// We add to ourself later
 					continue
@@ -79,12 +79,12 @@ func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime tim
 				}
 				// We are within the limit
 				if diff < groupNano {
-					timeOccurenceMap[k]++
+					timeOccurrenceMap[k]++
 				}
 			}
 		}
 		// Add ourself...
-		timeOccurenceMap[nano]++
+		timeOccurrenceMap[nano]++
 	}
 
 	maxima = 0 // Counter for remembering max occurrence of elements.
@@ -92,7 +92,7 @@ func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime tim
 
 	// Find the common cardinality from previously collected
 	// occurrences of elements.
-	for nano, count := range timeOccurenceMap {
+	for nano, count := range timeOccurrenceMap {
 		if count < maxima {
 			continue
 		}
@@ -111,7 +111,7 @@ func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime tim
 // commonTime returns a maximally occurring time from a list of time if it
 // occurs >= quorum, else return timeSentinel
 func commonTime(modTimes []time.Time, quorum int) time.Time {
-	if modTime, count := commonTimeAndOccurence(modTimes, 0); count >= quorum {
+	if modTime, count := commonTimeAndOccurrence(modTimes, 0); count >= quorum {
 		return modTime
 	}
 
@@ -191,19 +191,6 @@ func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks
 	}
 }
 
-// Extracts list of disk mtimes from FileInfo slice and returns, skips
-// slice elements that have errors.
-func listObjectDiskMtimes(partsMetadata []FileInfo) (diskMTimes []time.Time) {
-	diskMTimes = bootModtimes(len(partsMetadata))
-	for index, metadata := range partsMetadata {
-		if metadata.IsValid() {
-			// Once the file is found, save the disk mtime saved on disk.
-			diskMTimes[index] = metadata.DiskMTime
-		}
-	}
-	return diskMTimes
-}
-
 // Notes:
 // There are 5 possible states a disk could be in,
 // 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
@@ -277,13 +264,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 	errs []error, latestMeta FileInfo, bucket, object string,
 	scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time,
 ) {
-	var diskMTime time.Time
-	var shardFix bool
-	if !latestMeta.DataShardFixed() {
-		diskMTime = pickValidDiskTimeWithQuorum(partsMetadata,
-			latestMeta.Erasure.DataBlocks)
-	}
-
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 
@@ -351,23 +331,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			}
 		}
 
-		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
-			if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
-				// not with in acceptable delta, skip.
-				// If disk mTime mismatches it is considered outdated
-				// https://github.com/minio/minio/pull/13803
-				//
-				// This check only is active if we could find maximally
-				// occurring disk mtimes that are somewhat same across
-				// the quorum. Allowing to skip those shards which we
-				// might think are wrong.
-				shardFix = true
-				partsMetadata[i] = FileInfo{}
-				dataErrs[i] = errFileCorrupt
-				continue
-			}
-		}
-
 		// Always check data, if we got it.
 		if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
 			checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
@@ -410,10 +373,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 	}
 
-	if shardFix {
-		// Only when shard is fixed return an appropriate disk mtime value.
-		return availableDisks, dataErrs, diskMTime
-	} // else return timeSentinel for disk mtime
-
 	return availableDisks, dataErrs, timeSentinel
 }

cmd/erasure-healing-common_test.go

@@ -31,7 +31,7 @@ import (
 
 // Returns the latest updated FileInfo files and error in case of failure.
 func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultParityCount int, errs []error) (FileInfo, error) {
-	// There should be atleast half correct entries, if not return failure
+	// There should be at least half correct entries, if not return failure
 	expectedRQuorum := len(partsMetadata) / 2
 	if defaultParityCount == 0 {
 		// if parity count is '0', we expected all entries to be present.
@@ -57,7 +57,7 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultPar
 		return FileInfo{}, errErasureReadQuorum
 	}
 
-	// Interate through all the modTimes and count the FileInfo(s) with latest time.
+	// Iterate through all the modTimes and count the FileInfo(s) with latest time.
 	for index, t := range modTimes {
 		if partsMetadata[index].IsValid() && t.Equal(modTime) {
 			latestFileInfo = partsMetadata[index]
@@ -246,7 +246,7 @@ func TestListOnlineDisks(t *testing.T) {
 				t.Fatalf("Failed to putObject %v", err)
 			}
 
-			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
@@ -424,7 +424,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				t.Fatalf("Failed to putObject %v", err)
 			}
 
-			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", true, true)
+			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", true, true)
 			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
@@ -437,7 +437,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				partsMetadata[j].ModTime = test.modTimes[j]
 			}
 
-			if erasureDisks, err = writeUniqueFileInfo(ctx, erasureDisks, bucket, object, partsMetadata, diskCount(erasureDisks)); err != nil {
+			if erasureDisks, err = writeUniqueFileInfo(ctx, erasureDisks, "", bucket, object, partsMetadata, diskCount(erasureDisks)); err != nil {
 				t.Fatal(ctx, err)
 			}
 
@@ -534,7 +534,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	_, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	readQuorum := len(erasureDisks) / 2
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 		t.Fatalf("Failed to read xl meta data %v", reducedErr)
@@ -542,7 +542,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	// Test 1: Test that all disks are returned without any failures with
 	// unmodified meta data
-	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	if err != nil {
 		t.Fatalf("Failed to read xl meta data %v", err)
 	}

cmd/erasure-healing.go

@@ -33,8 +33,6 @@ import (
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
-const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
-
 //go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE
 
 type healingMetric uint8
@@ -45,29 +43,6 @@ const (
 	healingMetricCheckAbandonedParts
 )
 
-// AcceptableDelta returns 'true' if the fi.DiskMTime is under
-// acceptable delta of "delta" duration with maxTime.
-//
-// This code is primarily used for heuristic detection of
-// incorrect shards, as per https://github.com/minio/minio/pull/13803
-//
-// This check only is active if we could find maximally
-// occurring disk mtimes that are somewhat same across
-// the quorum. Allowing to skip those shards which we
-// might think are wrong.
-func (fi FileInfo) AcceptableDelta(maxTime time.Time, delta time.Duration) bool {
-	diff := maxTime.Sub(fi.DiskMTime)
-	if diff < 0 {
-		diff = -diff
-	}
-	return diff < delta
-}
-
-// DataShardFixed - data shard fixed?
-func (fi FileInfo) DataShardFixed() bool {
-	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
-}
-
 func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -108,7 +83,7 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.Heal
 		partial: func(entries metaCacheEntries, _ []error) {
 			entry, ok := entries.resolve(&resolver)
 			if !ok {
-				// check if we can get one entry atleast
+				// check if we can get one entry at least
 				// proceed to heal nonetheless.
 				entry, _ = entries.firstFound()
 			}
@@ -169,7 +144,7 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
-func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo, doinline bool) bool {
+func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool {
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
@@ -182,10 +157,6 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 			// always check first.
 			return true
 		}
-		if doinline {
-			// convert small files to 'inline'
-			return true
-		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
@@ -203,7 +174,10 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	return false
 }
 
-const xMinIOHealing = ReservedMetadataPrefix + "healing"
+const (
+	xMinIOHealing = ReservedMetadataPrefix + "healing"
+	xMinIODataMov = ReservedMetadataPrefix + "data-mov"
+)
 
 // SetHealing marks object (version) as being healed.
 // Note: this is to be used only from healObject
@@ -221,6 +195,21 @@ func (fi FileInfo) Healing() bool {
 	return ok
 }
 
+// SetDataMov marks object (version) as being currently
+// in movement, such as decommissioning or rebalance.
+func (fi *FileInfo) SetDataMov() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	fi.Metadata[xMinIODataMov] = "true"
+}
+
+// DataMov returns true if object is being in movement
+func (fi FileInfo) DataMov() bool {
+	_, ok := fi.Metadata[xMinIODataMov]
+	return ok
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
@@ -255,7 +244,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	}
 
 	// Re-read when we have lock...
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true, true)
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, "", bucket, object, versionID, true, true)
 	if isAllNotFound(errs) {
 		err := errFileNotFound
 		if versionID != "" {
@@ -311,11 +300,10 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	// used here for reconstruction. This is done to ensure that
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
-	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+	availableDisks, dataErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
 	var erasure Erasure
-	var recreate bool
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
 		// Initialize erasure coding
 		erasure, err = NewErasure(ctx, latestMeta.Erasure.DataBlocks,
@@ -323,15 +311,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		if err != nil {
 			return result, err
 		}
-
-		// Is only 'true' if the opts.Recreate is true and
-		// the object shardSize < smallFileThreshold do not
-		// set this to 'true' arbitrarily and must be only
-		// 'true' with caller ask.
-		recreate = (opts.Recreate &&
-			!latestMeta.InlineData() &&
-			len(latestMeta.Parts) == 1 &&
-			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
 	}
 
 	result.ObjectSize, err = latestMeta.ToObjectInfo(bucket, object, true).GetActualSize()
@@ -360,7 +339,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta, recreate) {
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
@@ -408,7 +387,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		return result, nil
 	}
 
-	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// Allow for dangling deletes, on versions that have DataDir missing etc.
 		// this would end up restoring the correct readable versions.
 		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrs, ObjectOptions{
@@ -506,7 +485,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 
 	var inlineBuffers []*bytes.Buffer
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		if latestMeta.InlineData() || recreate {
+		if latestMeta.InlineData() {
 			inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 		}
 
@@ -543,7 +522,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(latestMeta.Size)+32))
 					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 				} else {
-					writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, partPath,
+					writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath,
 						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
@@ -610,11 +589,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			return result, err
 		}
 
-		// - Remove any parts from healed disks after its been inlined.
 		// - Remove any remaining parts from outdated disks from before transition.
-		if recreate || partsMetadata[i].IsRemote() {
+		if partsMetadata[i].IsRemote() {
 			rmDataDir := partsMetadata[i].DataDir
-			disk.DeleteVol(ctx, pathJoin(bucket, encodeDirObject(object), rmDataDir), true)
+			disk.Delete(ctx, bucket, pathJoin(encodeDirObject(object), rmDataDir), DeleteOptions{
+				Immediate: true,
+				Recursive: true,
+			})
 		}
 
 		for i, v := range result.Before.Drives {
@@ -624,20 +605,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		}
 	}
 
-	if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
-		// Update metadata to indicate special fix.
-		_, err = er.PutObjectMetadata(ctx, bucket, object, ObjectOptions{
-			NoLock: true,
-			UserDefined: map[string]string{
-				reservedMetadataPrefixLowerDataShardFix: "true",
-				// another reserved metadata to capture original disk-mtime
-				// captured for this version of the object, to be used
-				// possibly in future to heal other versions if possible.
-				ReservedMetadataPrefixLower + "disk-mtime": diskMTime.String(),
-			},
-		})
-	}
-
 	return result, nil
 }
 
@@ -832,7 +799,7 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 		}
 		index := index
 		g.Go(func() error {
-			entries, err := storageDisks[index].ListDir(ctx, bucket, prefix, 1)
+			entries, err := storageDisks[index].ListDir(ctx, "", bucket, prefix, 1)
 			if err != nil {
 				return err
 			}
@@ -846,6 +813,10 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 	return g.Wait()
 }
 
+func isAllVolumeNotFound(errs []error) bool {
+	return countErrs(errs, errVolumeNotFound) == len(errs)
+}
+
 // isAllNotFound will return if any element of the error slice is not
 // errFileNotFound, errFileVersionNotFound or errVolumeNotFound.
 // A 0 length slice will always return false.
@@ -875,8 +846,12 @@ func isAllBucketsNotFound(errs []error) bool {
 	}
 	notFoundCount := 0
 	for _, err := range errs {
-		if err != nil && errors.Is(err, errVolumeNotFound) {
-			notFoundCount++
+		if err != nil {
+			if errors.Is(err, errVolumeNotFound) {
+				notFoundCount++
+			} else if isErrBucketNotFound(err) {
+				notFoundCount++
+			}
 		}
 	}
 	return len(errs) == notFoundCount
@@ -914,38 +889,28 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	// We can consider an object data not reliable
 	// when xl.meta is not found in read quorum disks.
 	// or when xl.meta is not readable in read quorum disks.
-	danglingErrsCount := func(cerrs []error) (int, int, int) {
+	danglingErrsCount := func(cerrs []error) (int, int) {
 		var (
-			notFoundCount     int
-			corruptedCount    int
-			diskNotFoundCount int
+			notFoundCount      int
+			nonActionableCount int
 		)
 		for _, readErr := range cerrs {
+			if readErr == nil {
+				continue
+			}
 			switch {
 			case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
 				notFoundCount++
-			case errors.Is(readErr, errFileCorrupt):
-				corruptedCount++
-			case errors.Is(readErr, errDiskNotFound):
-				diskNotFoundCount++
+			default:
+				// All other errors are non-actionable
+				nonActionableCount++
 			}
 		}
-		return notFoundCount, corruptedCount, diskNotFoundCount
+		return notFoundCount, nonActionableCount
 	}
 
-	ndataErrs := make([]error, len(dataErrs))
-	for i := range dataErrs {
-		if errs[i] != dataErrs[i] {
-			// Only count part errors, if the error is not
-			// same as xl.meta error. This is to avoid
-			// double counting when both parts and xl.meta
-			// are not available.
-			ndataErrs[i] = dataErrs[i]
-		}
-	}
-
-	notFoundMetaErrs, corruptedMetaErrs, driveNotFoundMetaErrs := danglingErrsCount(errs)
-	notFoundPartsErrs, corruptedPartsErrs, driveNotFoundPartsErrs := danglingErrsCount(ndataErrs)
+	notFoundMetaErrs, nonActionableMetaErrs := danglingErrsCount(errs)
+	notFoundPartsErrs, nonActionablePartsErrs := danglingErrsCount(dataErrs)
 
 	for _, m := range metaArr {
 		if m.IsValid() {
@@ -955,10 +920,9 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	}
 
 	if !validMeta.IsValid() {
-		// validMeta is invalid because notFoundPartsErrs is
-		// greater than parity blocks, thus invalidating the FileInfo{}
-		// every dataErrs[i], metaArr[i] is an empty FileInfo{}
-		dataBlocks := (len(ndataErrs) + 1) / 2
+		// validMeta is invalid because all xl.meta is missing apparently
+		// we should figure out if dataDirs are also missing > dataBlocks.
+		dataBlocks := (len(dataErrs) + 1) / 2
 		if notFoundPartsErrs > dataBlocks {
 			// Not using parity to ensure that we do not delete
 			// any valid content, if any is recoverable. But if
@@ -975,25 +939,34 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 		return validMeta, false
 	}
 
-	if driveNotFoundMetaErrs > 0 || driveNotFoundPartsErrs > 0 {
+	if nonActionableMetaErrs > 0 || nonActionablePartsErrs > 0 {
 		return validMeta, false
 	}
 
 	if validMeta.Deleted {
 		// notFoundPartsErrs is ignored since
 		// - delete marker does not have any parts
-		return validMeta, corruptedMetaErrs+notFoundMetaErrs > len(errs)/2
+		dataBlocks := (len(errs) + 1) / 2
+		return validMeta, notFoundMetaErrs > dataBlocks
 	}
 
-	totalErrs := notFoundMetaErrs + corruptedMetaErrs + notFoundPartsErrs + corruptedPartsErrs
-	if validMeta.IsRemote() {
-		// notFoundPartsErrs is ignored since
-		// - transition status of complete has no parts
-		totalErrs = notFoundMetaErrs + corruptedMetaErrs
+	// TODO: It is possible to replay the object via just single
+	// xl.meta file, considering quorum number of data-dirs are still
+	// present on other drives.
+	//
+	// However this requires a bit of a rewrite, leave this up for
+	// future work.
+	if notFoundMetaErrs > 0 && notFoundMetaErrs > validMeta.Erasure.ParityBlocks {
+		// All xl.meta is beyond data blocks missing, this is dangling
+		return validMeta, true
 	}
 
-	// We have valid meta, now verify if we have enough files with parity blocks.
-	return validMeta, totalErrs > validMeta.Erasure.ParityBlocks
+	if !validMeta.IsRemote() && notFoundPartsErrs > 0 && notFoundPartsErrs > validMeta.Erasure.ParityBlocks {
+		// All data-dir is beyond data blocks missing, this is dangling
+		return validMeta, true
+	}
+
+	return validMeta, false
 }
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.
@@ -1025,7 +998,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 
 	// Perform quick read without lock.
 	// This allows to quickly check if all is ok or all are missing.
-	_, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false, false)
+	_, errs := readAllFileInfo(healCtx, storageDisks, "", bucket, object, versionID, false, false)
 	if isAllNotFound(errs) {
 		err := errFileNotFound
 		if versionID != "" {
@@ -1059,10 +1032,9 @@ func healTrace(funcName healingMetric, startTime time.Time, bucket, object strin
 	}
 	if opts != nil {
 		tr.Custom = map[string]string{
-			"dry":      fmt.Sprint(opts.DryRun),
-			"remove":   fmt.Sprint(opts.Remove),
-			"recreate": fmt.Sprint(opts.Recreate),
-			"mode":     fmt.Sprint(opts.ScanMode),
+			"dry":    fmt.Sprint(opts.DryRun),
+			"remove": fmt.Sprint(opts.Remove),
+			"mode":   fmt.Sprint(opts.ScanMode),
 		}
 		if result != nil {
 			tr.Custom["version-id"] = result.VersionID

cmd/erasure-healing_test.go

@@ -41,6 +41,10 @@ func TestIsObjectDangling(t *testing.T) {
 	fi := newFileInfo("test-object", 2, 2)
 	fi.Erasure.Index = 1
 
+	ifi := newFileInfo("test-object", 2, 2)
+	ifi.SetInlineData()
+	ifi.Erasure.Index = 1
+
 	testCases := []struct {
 		name             string
 		metaArr          []FileInfo
@@ -129,13 +133,77 @@ func TestIsObjectDangling(t *testing.T) {
 			expectedMeta:     FileInfo{},
 			expectedDangling: false,
 		},
+		{
+			name: "FileInfoUnDecided-case4",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				ifi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     ifi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoUnDecided-case5-(ignore errFileCorrupt error)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				nil,
+				nil,
+			},
+			dataErrs: []error{
+				errFileCorrupt,
+				errFileNotFound,
+				nil,
+				errFileCorrupt,
+			},
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoUnDecided-case6-(data-dir intact)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+			},
+			dataErrs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				nil,
+				nil,
+			},
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
 		{
 			name: "FileInfoDecided-case1",
 			metaArr: []FileInfo{
 				{},
 				{},
 				{},
-				fi,
+				ifi,
 			},
 			errs: []error{
 				errFileNotFound,
@@ -144,25 +212,7 @@ func TestIsObjectDangling(t *testing.T) {
 				nil,
 			},
 			dataErrs:         nil,
-			expectedMeta:     fi,
-			expectedDangling: true,
-		},
-		{
-			name: "FileInfoDecided-case2",
-			metaArr: []FileInfo{
-				{},
-				{},
-				{},
-				fi,
-			},
-			errs: []error{
-				errFileNotFound,
-				errFileCorrupt,
-				errFileCorrupt,
-				nil,
-			},
-			dataErrs:         nil,
-			expectedMeta:     fi,
+			expectedMeta:     ifi,
 			expectedDangling: true,
 		},
 		{
@@ -175,8 +225,8 @@ func TestIsObjectDangling(t *testing.T) {
 			},
 			errs: []error{
 				errFileNotFound,
-				errFileCorrupt,
-				errFileCorrupt,
+				errFileNotFound,
+				errFileNotFound,
 				nil,
 			},
 			dataErrs:         nil,
@@ -184,26 +234,26 @@ func TestIsObjectDangling(t *testing.T) {
 			expectedDangling: true,
 		},
 		{
-			name: "FileInfoDecided-case2-(duplicate data errors)",
+			name: "FileInfoDecided-case3-(enough data-dir missing)",
 			metaArr: []FileInfo{
 				{},
 				{},
 				{},
-				{Deleted: true},
+				fi,
 			},
 			errs: []error{
 				errFileNotFound,
-				errFileCorrupt,
-				errFileCorrupt,
+				errFileNotFound,
+				nil,
 				nil,
 			},
 			dataErrs: []error{
 				errFileNotFound,
-				errFileCorrupt,
-				nil,
+				errFileNotFound,
 				nil,
+				errFileNotFound,
 			},
-			expectedMeta:     FileInfo{Deleted: true},
+			expectedMeta:     fi,
 			expectedDangling: true,
 		},
 		// Add new cases as seen
@@ -266,7 +316,7 @@ func TestHealing(t *testing.T) {
 	}
 
 	disk := er.getDisks()[0]
-	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -289,7 +339,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -308,7 +358,7 @@ func TestHealing(t *testing.T) {
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal
 	fileInfoOutDated.ModTime = time.Now()
-	err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
+	err = disk.WriteMetadata(context.Background(), "", bucket, object, fileInfoOutDated)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -318,7 +368,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -426,11 +476,11 @@ func TestHealingVersioned(t *testing.T) {
 	}
 
 	disk := er.getDisks()[0]
-	fileInfoPreHeal1, err := disk.ReadVersion(context.Background(), bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal1, err := disk.ReadVersion(context.Background(), "", bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
-	fileInfoPreHeal2, err := disk.ReadVersion(context.Background(), bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal2, err := disk.ReadVersion(context.Background(), "", bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -453,11 +503,11 @@ func TestHealingVersioned(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal1, err := disk.ReadVersion(context.Background(), bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal1, err := disk.ReadVersion(context.Background(), "", bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
-	fileInfoPostHeal2, err := disk.ReadVersion(context.Background(), bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal2, err := disk.ReadVersion(context.Background(), "", bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -479,7 +529,7 @@ func TestHealingVersioned(t *testing.T) {
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal1
 	fileInfoOutDated.ModTime = time.Now()
-	err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
+	err = disk.WriteMetadata(context.Background(), "", bucket, object, fileInfoOutDated)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -489,7 +539,7 @@ func TestHealingVersioned(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal1, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal1, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -499,7 +549,7 @@ func TestHealingVersioned(t *testing.T) {
 		t.Fatal("HealObject failed")
 	}
 
-	fileInfoPostHeal2, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal2, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -564,7 +614,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	resetGlobalHealState()
 	defer resetGlobalHealState()
 
-	// Set globalStoragClass.STANDARD to EC:4 for this test
+	// Set globalStorageClass.STANDARD to EC:4 for this test
 	saveSC := globalStorageClass
 	defer func() {
 		globalStorageClass.Update(saveSC)
@@ -637,7 +687,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	// Restore...
 	setDisks(orgDisks[:4]...)
 
-	fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -654,7 +704,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err := disks[0].ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -684,7 +734,7 @@ func TestHealingDanglingObject(t *testing.T) {
 
 	setDisks(orgDisks[:4]...)
 	disk := getDisk(0)
-	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -702,7 +752,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	disk = getDisk(0)
-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -733,7 +783,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	setDisks(orgDisks[:4]...)
 
 	disk = getDisk(0)
-	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -751,7 +801,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	disk = getDisk(0)
-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -836,7 +886,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 		er := set.sets[0]
 		erasureDisks := er.getDisks()
 
-		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 		nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 		if errors.Is(err, errFileNotFound) {
 			continue
@@ -858,12 +908,12 @@ func TestHealCorrectQuorum(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 		if countErrs(errs, nil) != len(fileInfos) {
 			t.Fatal("Expected all xl.meta healed, but partial heal detected")
 		}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false, true)
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", minioMetaBucket, cfgFile, "", false, true)
 		nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 		if errors.Is(err, errFileNotFound) {
 			continue
@@ -885,7 +935,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false, true)
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", minioMetaBucket, cfgFile, "", false, true)
 		if countErrs(errs, nil) != len(fileInfos) {
 			t.Fatal("Expected all xl.meta healed, but partial heal detected")
 		}
@@ -970,7 +1020,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -998,14 +1048,12 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi) {
 		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 	}
@@ -1029,14 +1077,12 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi) {
 		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 	}
@@ -1067,7 +1113,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
 		stats, _ := erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 		if len(stats) != 0 {
-			t.Errorf("Expected xl.meta file to be not present, but succeeeded")
+			t.Errorf("Expected xl.meta file to be not present, but succeeded")
 		}
 	}
 }
@@ -1133,7 +1179,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	firstDisk := erasureDisks[0]
 
 	// Test 1: Remove the object backend files from the first disk.
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -1156,14 +1202,12 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi1, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi1.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi1) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
@@ -1179,14 +1223,12 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi2, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi2.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi2) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
@@ -1277,7 +1319,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	firstDisk := erasureDisks[0]
 	secondDisk := erasureDisks[1]
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -1603,7 +1645,6 @@ func TestHealLastDataShard(t *testing.T) {
 
 			_, err = obj.PutObject(ctx, bucket, object,
 				mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
-
 			if err != nil {
 				t.Fatal(err)
 			}

cmd/erasure-metadata-utils.go

@@ -20,9 +20,7 @@ package cmd
 import (
 	"context"
 	"errors"
-	"fmt"
 	"hash/crc32"
-	"io"
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
@@ -64,7 +62,7 @@ func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error)
 		if IsErrIgnored(err, ignoredErrs...) {
 			continue
 		}
-		// Errors due to context cancelation may be wrapped - group them by context.Canceled.
+		// Errors due to context cancellation may be wrapped - group them by context.Canceled.
 		if errors.Is(err, context.Canceled) {
 			errorCounts[context.Canceled]++
 			continue
@@ -148,29 +146,9 @@ func hashOrder(key string, cardinality int) []int {
 	return nums
 }
 
-var readFileInfoIgnoredErrs = append(objectOpIgnoredErrs,
-	errFileNotFound,
-	errVolumeNotFound,
-	errFileVersionNotFound,
-	io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
-	io.EOF,              // some times we would read without locks, ignore these errors
-)
-
-func readFileInfo(ctx context.Context, disk StorageAPI, bucket, object, versionID string, opts ReadOptions) (FileInfo, error) {
-	fi, err := disk.ReadVersion(ctx, bucket, object, versionID, opts)
-
-	if err != nil && !IsErr(err, readFileInfoIgnoredErrs...) {
-		logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
-			disk.String(), bucket, object, err),
-			disk.String())
-	}
-
-	return fi, err
-}
-
 // Reads all `xl.meta` metadata as a FileInfo slice.
 // Returns error slice indicating the failed metadata reads.
-func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, readData, healing bool) ([]FileInfo, []error) {
+func readAllFileInfo(ctx context.Context, disks []StorageAPI, origbucket string, bucket, object, versionID string, readData, healing bool) ([]FileInfo, []error) {
 	metadataArray := make([]FileInfo, len(disks))
 
 	opts := ReadOptions{
@@ -186,7 +164,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			metadataArray[index], err = readFileInfo(ctx, disks[index], bucket, object, versionID, opts)
+			metadataArray[index], err = disks[index].ReadVersion(ctx, origbucket, bucket, object, versionID, opts)
 			return err
 		}, index)
 	}
@@ -330,15 +308,12 @@ var (
 // returns error if totalSize is -1, partSize is 0, partIndex is 0.
 func calculatePartSizeFromIdx(ctx context.Context, totalSize int64, partSize int64, partIndex int) (currPartSize int64, err error) {
 	if totalSize < -1 {
-		logger.LogIf(ctx, errInvalidArgument)
 		return 0, errInvalidArgument
 	}
 	if partSize == 0 {
-		logger.LogIf(ctx, errPartSizeZero)
 		return 0, errPartSizeZero
 	}
 	if partIndex < 1 {
-		logger.LogIf(ctx, errPartSizeIndex)
 		return 0, errPartSizeIndex
 	}
 	if totalSize == -1 {

cmd/erasure-metadata-utils_test.go

@@ -94,7 +94,7 @@ func TestReduceErrs(t *testing.T) {
 			},
 			nil, nil,
 		},
-		// Checks if wrapped context cancelation errors are grouped as one.
+		// Checks if wrapped context cancellation errors are grouped as one.
 		{canceledErrs, nil, context.Canceled},
 	}
 	// Validates list of all the testcases for returning valid errors.

cmd/erasure-metadata.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -300,17 +301,23 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 				fmt.Fprintf(h, "%v", meta.Erasure.Distribution)
 			}
 
-			// ILM transition fields
-			fmt.Fprint(h, meta.TransitionStatus)
-			fmt.Fprint(h, meta.TransitionTier)
-			fmt.Fprint(h, meta.TransitionedObjName)
-			fmt.Fprint(h, meta.TransitionVersionID)
+			if meta.IsRemote() {
+				// ILM transition fields
+				fmt.Fprint(h, meta.TransitionStatus)
+				fmt.Fprint(h, meta.TransitionTier)
+				fmt.Fprint(h, meta.TransitionedObjName)
+				fmt.Fprint(h, meta.TransitionVersionID)
+			}
 
-			// Server-side replication fields
-			fmt.Fprintf(h, "%v", meta.MarkDeleted)
-			fmt.Fprint(h, meta.Metadata[string(meta.ReplicationState.ReplicaStatus)])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicationStatusInternal])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.VersionPurgeStatusInternal])
+			// If metadata says encrypted, ask for it in quorum.
+			if etyp, ok := crypto.IsEncrypted(meta.Metadata); ok {
+				fmt.Fprint(h, etyp)
+			}
+
+			// If compressed, look for compressed FileInfo only
+			if meta.IsCompressed() {
+				fmt.Fprint(h, meta.Metadata[ReservedMetadataPrefix+"compression"])
+			}
 
 			metaHashes[i] = hex.EncodeToString(h.Sum(nil))
 			h.Reset()
@@ -374,17 +381,6 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 	return FileInfo{}, errErasureReadQuorum
 }
 
-func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time {
-	diskMTimes := listObjectDiskMtimes(metaArr)
-
-	diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta)
-	if diskMaxima >= quorum {
-		return diskMTime
-	}
-
-	return timeSentinel
-}
-
 // pickValidFileInfo - picks one valid FileInfo content and returns from a
 // slice of FileInfo.
 func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, etag string, quorum int) (FileInfo, error) {
@@ -392,7 +388,7 @@ func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Tim
 }
 
 // writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
-func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
+func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
 	g := errgroup.WithNErrs(len(disks))
 
 	// Start writing `xl.meta` to all disks in parallel.
@@ -406,7 +402,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 			fi := files[index]
 			fi.Erasure.Index = index + 1
 			if fi.IsValid() {
-				return disks[index].WriteMetadata(ctx, bucket, prefix, fi)
+				return disks[index].WriteMetadata(ctx, origbucket, bucket, prefix, fi)
 			}
 			return errCorruptedFormat
 		}, index)
@@ -483,7 +479,7 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 // readQuorum is the min required disks to read data.
 // writeQuorum is the min required disks to write data.
 func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
-	// There should be atleast half correct entries, if not return failure
+	// There should be at least half correct entries, if not return failure
 	expectedRQuorum := len(partsMetaData) / 2
 	if defaultParityCount == 0 {
 		// if parity count is '0', we expected all entries to be present.
@@ -506,14 +502,6 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 		return -1, -1, errErasureReadQuorum
 	}
 
-	if parityBlocks == 0 {
-		// For delete markers do not use 'defaultParityCount' as it is not expected to be the case.
-		// Use maximum allowed read quorum instead, writeQuorum+1 is returned for compatibility sake
-		// but there are no callers that shall be using this.
-		readQuorum := len(partsMetaData) / 2
-		return readQuorum, readQuorum + 1, nil
-	}
-
 	dataBlocks := len(partsMetaData) - parityBlocks
 
 	writeQuorum := dataBlocks
@@ -529,6 +517,7 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 const (
 	tierFVID     = "tier-free-versionID"
 	tierFVMarker = "tier-free-marker"
+	tierSkipFVID = "tier-skip-fvid"
 )
 
 // SetTierFreeVersionID sets free-version's versionID. This method is used by
@@ -555,6 +544,23 @@ func (fi *FileInfo) SetTierFreeVersion() {
 	fi.Metadata[ReservedMetadataPrefixLower+tierFVMarker] = ""
 }
 
+// SetSkipTierFreeVersion indicates to skip adding a tier free version id.
+// Note: Used only when expiring tiered objects and the remote content has
+// already been scheduled for deletion
+func (fi *FileInfo) SetSkipTierFreeVersion() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	fi.Metadata[ReservedMetadataPrefixLower+tierSkipFVID] = ""
+}
+
+// SkipTierFreeVersion returns true if set, false otherwise.
+// See SetSkipTierVersion for its purpose.
+func (fi *FileInfo) SkipTierFreeVersion() bool {
+	_, ok := fi.Metadata[ReservedMetadataPrefixLower+tierSkipFVID]
+	return ok
+}
+
 // TierFreeVersion returns true if version is a free-version.
 func (fi *FileInfo) TierFreeVersion() bool {
 	_, ok := fi.Metadata[ReservedMetadataPrefixLower+tierFVMarker]

cmd/erasure-metadata_test.go

@@ -314,3 +314,11 @@ func TestTransitionInfoEquals(t *testing.T) {
 		t.Fatalf("Expected to be inequal: fi %v ofi %v", fi, ofi)
 	}
 }
+
+func TestSkipTierFreeVersion(t *testing.T) {
+	fi := newFileInfo("object", 8, 8)
+	fi.SetSkipTierFreeVersion()
+	if ok := fi.SkipTierFreeVersion(); !ok {
+		t.Fatal("Expected SkipTierFreeVersion to be set on FileInfo but wasn't")
+	}
+}

cmd/erasure-multipart.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
@@ -61,7 +62,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
 // checkUploadIDExists - verify if a given uploadID exists and is valid.
 func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) {
 	defer func() {
-		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+		if errors.Is(err, errFileNotFound) {
 			err = errUploadIDNotFound
 		}
 	}()
@@ -71,7 +72,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket,
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, minioMetaMultipartBucket,
 		uploadIDPath, "", false, false)
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
@@ -87,15 +88,14 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	// List all online disks.
 	_, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, quorum)
 
+	var reducedErr error
 	if write {
-		reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-		if errors.Is(reducedErr, errErasureWriteQuorum) {
-			return fi, nil, reducedErr
-		}
+		reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	} else {
-		if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
-			return fi, nil, reducedErr
-		}
+		reducedErr = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
+	}
+	if reducedErr != nil {
+		return fi, nil, reducedErr
 	}
 
 	// Pick one from the first valid metadata.
@@ -103,7 +103,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	return fi, partsMetadata, err
 }
 
-// Removes part.meta given by partName belonging to a mulitpart upload from minioMetaBucket
+// Removes part.meta given by partName belonging to a multipart upload from minioMetaBucket
 func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string, partNumber int) {
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
@@ -127,7 +127,7 @@ func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string
 	g.Wait()
 }
 
-// Removes part given by partName belonging to a mulitpart upload from minioMetaBucket
+// Removes part given by partName belonging to a multipart upload from minioMetaBucket
 func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir string, partNumber int) {
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
@@ -200,7 +200,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 	readDirFn(pathJoin(diskPath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
 		readDirFn(pathJoin(diskPath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
-			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
+			fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
 			if err != nil {
 				return nil
 			}
@@ -281,15 +281,14 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		if !disk.IsOnline() {
 			continue
 		}
-		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
+		uploadIDs, err = disk.ListDir(ctx, bucket, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if errors.Is(err, errDiskNotFound) {
 				continue
 			}
-			if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+			if errors.Is(err, errFileNotFound) {
 				return result, nil
 			}
-			logger.LogIf(ctx, err)
 			return result, toObjectErr(err, bucket, object)
 		}
 		break
@@ -303,10 +302,10 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 	// to read the metadata entry.
 	var uploads []MultipartInfo
 
-	populatedUploadIds := set.NewStringSet()
+	populatedUploadIDs := set.NewStringSet()
 
 	for _, uploadID := range uploadIDs {
-		if populatedUploadIds.Contains(uploadID) {
+		if populatedUploadIDs.Contains(uploadID) {
 			continue
 		}
 		// If present, use time stored in ID.
@@ -323,7 +322,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 			UploadID:  base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID(), uploadID))),
 			Initiated: startTime,
 		})
-		populatedUploadIds.Add(uploadID)
+		populatedUploadIDs.Add(uploadID)
 	}
 
 	sort.Slice(uploads, func(i int, j int) bool {
@@ -402,36 +401,33 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		parityDrives = er.defaultParityCount
 	}
 
-	// If we have offline disks upgrade the number of erasure codes for this object.
-	parityOrig := parityDrives
+	if globalStorageClass.AvailabilityOptimized() {
+		// If we have offline disks upgrade the number of erasure codes for this object.
+		parityOrig := parityDrives
 
-	var offlineDrives int
-	for _, disk := range onlineDisks {
-		if disk == nil {
-			parityDrives++
-			offlineDrives++
-			continue
+		var offlineDrives int
+		for _, disk := range onlineDisks {
+			if disk == nil || !disk.IsOnline() {
+				parityDrives++
+				offlineDrives++
+				continue
+			}
 		}
-		if !disk.IsOnline() {
-			parityDrives++
-			offlineDrives++
-			continue
+
+		if offlineDrives >= (len(onlineDisks)+1)/2 {
+			// if offline drives are more than 50% of the drives
+			// we have no quorum, we shouldn't proceed just
+			// fail at that point.
+			return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
-	}
 
-	if offlineDrives >= (len(onlineDisks)+1)/2 {
-		// if offline drives are more than 50% of the drives
-		// we have no quorum, we shouldn't proceed just
-		// fail at that point.
-		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
-	}
+		if parityDrives >= len(onlineDisks)/2 {
+			parityDrives = len(onlineDisks) / 2
+		}
 
-	if parityDrives >= len(onlineDisks)/2 {
-		parityDrives = len(onlineDisks) / 2
-	}
-
-	if parityOrig != parityDrives {
-		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+		if parityOrig != parityDrives {
+			userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+		}
 	}
 
 	dataDrives := len(onlineDisks) - parityDrives
@@ -463,6 +459,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
+	// if storageClass is standard no need to save it as part of metadata.
+	if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD {
+		delete(userDefined, xhttp.AmzStorageClass)
+	}
+
 	if opts.WantChecksum != nil && opts.WantChecksum.Type.IsSet() {
 		userDefined[hash.MinIOMultipartChecksum] = opts.WantChecksum.Type.String()
 	}
@@ -486,8 +487,8 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID)
 
 	// Write updated `xl.meta` to all disks.
-	if _, err := writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
-		return nil, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
+		return nil, toObjectErr(err, bucket, object)
 	}
 	return &NewMultipartUploadResult{
 		UploadID:     uploadID,
@@ -501,7 +502,9 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 //
 // Implements S3 compatible initiate multipart API.
 func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	return er.newMultipartUpload(ctx, bucket, object, opts)
 }
@@ -580,7 +583,16 @@ func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry
 //
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
+
+	data := r.Reader
+	// Validate input data size and it can never be less than zero.
+	if data.Size() < -1 {
+		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		return pi, toObjectErr(errInvalidArgument)
+	}
 
 	// Read lock for upload id.
 	// Only held while reading the upload metadata.
@@ -596,16 +608,12 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	// Validates if upload ID exists.
 	fi, _, err := er.checkUploadIDExists(rctx, bucket, object, uploadID, true)
 	if err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return pi, toObjectErr(err, bucket)
+		}
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	data := r.Reader
-	// Validate input data size and it can never be less than zero.
-	if data.Size() < -1 {
-		logger.LogIf(rctx, errInvalidArgument, logger.Application)
-		return pi, toObjectErr(errInvalidArgument)
-	}
-
 	// Write lock for this part ID, only hold it if we are planning to read from the
 	// streamto avoid any concurrent updates.
 	//
@@ -656,7 +664,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	var buffer []byte
 	switch size := data.Size(); {
 	case size == 0:
-		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+		buffer = make([]byte, 1) // Allocate at least a byte to reach EOF
 	case size == -1:
 		if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
 			// Account for padding and forced compression overhead and encryption.
@@ -681,7 +689,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	toEncode := io.Reader(data)
@@ -776,7 +784,9 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 // - compressed
 // Does not contain currently uploaded parts by design.
 func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	result := MultipartInfo{
 		Bucket:   bucket,
@@ -794,6 +804,9 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return result, toObjectErr(err, bucket)
+		}
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -809,7 +822,9 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
 	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
@@ -965,7 +980,9 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 //
 // Implements S3 compatible Complete multipart API.
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	// Hold write locks to verify uploaded parts, also disallows any
 	// parallel PutObjectPart() requests.
@@ -979,6 +996,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return oi, toObjectErr(err, bucket)
+		}
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -1237,6 +1257,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	// Save the consolidated actual size.
 	fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 
+	if opts.DataMovement {
+		fi.SetDataMov()
+	}
+
 	// Update all erasure metadata, make sure to not modify fields like
 	// checksum which are different on each disks.
 	for index := range partsMetadata {
@@ -1325,7 +1349,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // would be removed from the system, rollback is not possible on this
 // operation.
 func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -1337,6 +1363,9 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 
 	// Validates if upload ID exists.
 	if _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false); err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return toObjectErr(err, bucket)
+		}
 		return toObjectErr(err, bucket, object, uploadID)
 	}
 

cmd/erasure-object.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
@@ -46,7 +47,6 @@ import (
 	"github.com/minio/pkg/v2/mimedb"
 	"github.com/minio/pkg/v2/sync/errgroup"
 	"github.com/minio/pkg/v2/wildcard"
-	"github.com/tinylib/msgp/msgp"
 )
 
 // list all errors which can be ignored in object operations.
@@ -67,7 +67,9 @@ func countOnlineDisks(onlineDisks []StorageAPI) (online int) {
 // if source object and destination object are same we only
 // update metadata.
 func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, dstObject, &er)
+	if !dstOpts.NoAuditLog {
+		auditObjectErasureSet(ctx, dstObject, &er)
+	}
 
 	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
 	if !srcInfo.metadataOnly {
@@ -91,7 +93,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 
 	// Read metadata associated with the object from all disks.
 	if srcOpts.VersionID != "" {
-		metaArr, errs = readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, true, false)
+		metaArr, errs = readAllFileInfo(ctx, storageDisks, "", srcBucket, srcObject, srcOpts.VersionID, true, false)
 	} else {
 		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false, true)
 	}
@@ -179,7 +181,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	}
 
 	// Write unique `xl.meta` for each disk.
-	if _, err = writeUniqueFileInfo(ctx, onlineDisks, srcBucket, srcObject, metaArr, writeQuorum); err != nil {
+	if _, err = writeUniqueFileInfo(ctx, onlineDisks, "", srcBucket, srcObject, metaArr, writeQuorum); err != nil {
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
@@ -189,7 +191,9 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // GetObjectNInfo - returns object info and an object
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	var unlockOnDefer bool
 	nsUnlocker := func() {}
@@ -227,30 +231,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return nil, toObjectErr(err, bucket, object)
 	}
 
-	if !fi.DataShardFixed() {
-		diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks)
-		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
-			for index := range onlineDisks {
-				if onlineDisks[index] == OfflineDisk {
-					continue
-				}
-				if !metaArr[index].IsValid() {
-					continue
-				}
-				if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
-					// If disk mTime mismatches it is considered outdated
-					// https://github.com/minio/minio/pull/13803
-					//
-					// This check only is active if we could find maximally
-					// occurring disk mtimes that are somewhat same across
-					// the quorum. Allowing to skip those shards which we
-					// might think are wrong.
-					onlineDisks[index] = OfflineDisk
-				}
-			}
-		}
-	}
-
 	objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if objInfo.DeleteMarker {
 		if opts.VersionID == "" {
@@ -444,7 +424,9 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	if !opts.NoLock {
 		// Lock the object before reading.
@@ -511,7 +493,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		// count the number of offline disks
 		offline := 0
 		for i := 0; i < max(len(errs), len(dataErrs)); i++ {
-			if i < len(errs) && errs[i] == errDiskNotFound || i < len(dataErrs) && dataErrs[i] == errDiskNotFound {
+			if i < len(errs) && errors.Is(errs[i], errDiskNotFound) || i < len(dataErrs) && errors.Is(dataErrs[i], errDiskNotFound) {
 				offline++
 			}
 		}
@@ -570,29 +552,6 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 	return m, err
 }
 
-var readRawFileInfoErrs = append(objectOpIgnoredErrs,
-	errFileNotFound,
-	errFileNameTooLong,
-	errVolumeNotFound,
-	errFileVersionNotFound,
-	io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
-	io.EOF,              // some times we would read without locks, ignore these errors
-	msgp.ErrShortBytes,
-	context.DeadlineExceeded,
-	context.Canceled,
-)
-
-func readRawFileInfo(ctx context.Context, disk StorageAPI, bucket, object string, readData bool) (RawFileInfo, error) {
-	rf, err := disk.ReadXL(ctx, bucket, object, readData)
-
-	if err != nil && !IsErr(err, readRawFileInfoErrs...) {
-		logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
-			disk.String(), bucket, object, err),
-			disk.String())
-	}
-	return rf, err
-}
-
 func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers, allParts bool) (FileInfo, error) {
 	var xl xlMetaV2
 	if err := xl.LoadOrConvert(ri.Buf); err != nil {
@@ -622,7 +581,6 @@ func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVe
 		fileInfo.Data = xl.data.find(versionID)
 	}
 
-	fileInfo.DiskMTime = ri.DiskMTime
 	return fileInfo, nil
 }
 
@@ -635,7 +593,7 @@ func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			rf, err := readRawFileInfo(ctx, disks[index], bucket, object, readData)
+			rf, err := disks[index].ReadXL(ctx, bucket, object, readData)
 			if err != nil {
 				return err
 			}
@@ -673,9 +631,7 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 			continue
 		}
 		metadataArray[index] = &xl
-		metaFileInfos[index] = FileInfo{
-			DiskMTime: rf.DiskMTime,
-		}
+		metaFileInfos[index] = FileInfo{}
 	}
 
 	for index := range metadataArray {
@@ -715,7 +671,6 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 		}
 
 		// make sure to preserve this for diskmtime based healing bugfix.
-		diskMTime := metaFileInfos[index].DiskMTime
 		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers, allParts)
 		if errs[index] != nil {
 			continue
@@ -724,7 +679,6 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 		if readData {
 			metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
 		}
-		metaFileInfos[index].DiskMTime = diskMTime
 	}
 	if !readData {
 		for i := range v2bufs {
@@ -768,8 +722,6 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 }
 
 func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (FileInfo, []FileInfo, []StorageAPI, error) {
-	var mu sync.Mutex
-
 	rawArr := make([]RawFileInfo, er.setDriveCount)
 	metaArr := make([]FileInfo, er.setDriveCount)
 	errs := make([]error, er.setDriveCount)
@@ -780,14 +732,16 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	done := make(chan bool, er.setDriveCount)
 	disks := er.getDisks()
 
-	mrfCheck := make(chan FileInfo)
-	defer close(mrfCheck)
-
 	ropts := ReadOptions{
 		ReadData: readData,
 		Healing:  false,
 	}
 
+	mrfCheck := make(chan FileInfo)
+	defer xioutil.SafeClose(mrfCheck)
+
+	var rw sync.Mutex
+
 	// Ask for all disks first;
 	go func() {
 		ctx, cancel := context.WithCancel(ctx)
@@ -799,56 +753,69 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 				done <- false
 				continue
 			}
+			if !disk.IsOnline() {
+				done <- false
+				continue
+			}
 			wg.Add(1)
 			go func(i int, disk StorageAPI) {
 				defer wg.Done()
-				var fi FileInfo
-				var err error
+
+				var (
+					fi  FileInfo
+					rfi RawFileInfo
+					err error
+				)
+
 				if opts.VersionID != "" {
 					// Read a specific version ID
-					fi, err = readFileInfo(ctx, disk, bucket, object, opts.VersionID, ropts)
-					mu.Lock()
-					metaArr[i], errs[i] = fi, err
-					mu.Unlock()
+					fi, err = disk.ReadVersion(ctx, "", bucket, object, opts.VersionID, ropts)
 				} else {
 					// Read the latest version
-					var ri RawFileInfo
-					ri, err = readRawFileInfo(ctx, disk, bucket, object, readData)
-					mu.Lock()
-					rawArr[i], errs[i] = ri, err
-					mu.Unlock()
+					rfi, err = disk.ReadXL(ctx, bucket, object, readData)
 					if err == nil {
-						fi, err = fileInfoFromRaw(ri, bucket, object, readData, opts.InclFreeVersions, true)
-						mu.Lock()
-						metaArr[i], errs[i] = fi, err
-						mu.Unlock()
+						fi, err = fileInfoFromRaw(rfi, bucket, object, readData, opts.InclFreeVersions, true)
 					}
 				}
+
+				rw.Lock()
+				rawArr[i] = rfi
+				metaArr[i], errs[i] = fi, err
+				rw.Unlock()
+
 				done <- err == nil
 			}(i, disk)
 		}
 
 		wg.Wait()
-		close(done)
+		xioutil.SafeClose(done)
 
 		fi, ok := <-mrfCheck
 		if !ok {
 			return
 		}
+
 		if fi.Deleted {
 			return
 		}
+
 		// if one of the disk is offline, return right here no need
 		// to attempt a heal on the object.
 		if countErrs(errs, errDiskNotFound) > 0 {
 			return
 		}
+
 		var missingBlocks int
 		for i := range errs {
-			if errors.Is(errs[i], errFileNotFound) {
+			if IsErr(errs[i],
+				errFileNotFound,
+				errFileVersionNotFound,
+				errFileCorrupt,
+			) {
 				missingBlocks++
 			}
 		}
+
 		// if missing metadata can be reconstructed, attempt to reconstruct.
 		// additionally do not heal delete markers inline, let them be
 		// healed upon regular heal process.
@@ -879,13 +846,12 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		minDisks = er.setDriveCount - er.defaultParityCount
 	}
 
-	calcQuorum := func() (FileInfo, []FileInfo, []StorageAPI, time.Time, string, error) {
+	calcQuorum := func(metaArr []FileInfo, errs []error) (FileInfo, []FileInfo, []StorageAPI, time.Time, string, error) {
 		readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 		if err != nil {
 			return FileInfo{}, nil, nil, time.Time{}, "", err
 		}
-		err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
-		if err != nil {
+		if err := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil {
 			return FileInfo{}, nil, nil, time.Time{}, "", err
 		}
 		onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum)
@@ -895,7 +861,11 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		}
 
 		onlineMeta := make([]FileInfo, len(metaArr))
-		copy(onlineMeta, metaArr)
+		for i, disk := range onlineDisks {
+			if disk != nil {
+				onlineMeta[i] = metaArr[i]
+			}
+		}
 
 		return fi, onlineMeta, onlineDisks, modTime, etag, nil
 	}
@@ -922,20 +892,24 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 				continue
 			}
 		}
+
+		rw.Lock()
 		if opts.VersionID == "" && totalResp == er.setDriveCount {
-			// Disks cannot agree about the latest version, pass this to a more advanced code
-			metaArr, errs = pickLatestQuorumFilesInfo(ctx, rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true)
+			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
+				rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true))
+		} else {
+			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
 		}
-		mu.Lock()
-		fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum()
-		mu.Unlock()
+		rw.Unlock()
 		if err == nil && fi.InlineData() {
 			break
 		}
 	}
 
 	if err != nil {
-		if shouldCheckForDangling(err, errs, bucket) {
+		// We can only look for dangling if we received all the responses, if we did
+		// not we simply ignore it, since we can't tell for sure if its dangling object.
+		if totalResp == er.setDriveCount && shouldCheckForDangling(err, errs, bucket) {
 			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
@@ -955,7 +929,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	for i := range onlineMeta {
 		// verify metadata is valid, it has similar erasure info
 		// as well as common modtime, if modtime is not possible
-		// verify if it has common "etag" atleast.
+		// verify if it has common "etag" at least.
 		if onlineMeta[i].IsValid() && onlineMeta[i].Erasure.Equal(fi.Erasure) {
 			ok := onlineMeta[i].ModTime.Equal(modTime)
 			if modTime.IsZero() || modTime.Equal(timeSentinel) {
@@ -970,7 +944,11 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		onlineDisks[i] = nil
 	}
 
-	mrfCheck <- fi.ShallowCopy()
+	select {
+	case mrfCheck <- fi.ShallowCopy():
+	case <-ctx.Done():
+		return fi, onlineMeta, onlineDisks, toObjectErr(ctx.Err(), bucket, object)
+	}
 
 	return fi, onlineMeta, onlineDisks, nil
 }
@@ -993,7 +971,7 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 	return objInfo, nil
 }
 
-// getObjectInfoAndQuroum - wrapper for reading object metadata and constructs ObjectInfo, additionally returns write quorum for the object.
+// getObjectInfoAndQuorum - wrapper for reading object metadata and constructs ObjectInfo, additionally returns write quorum for the object.
 func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, wquorum int, err error) {
 	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
@@ -1122,7 +1100,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.Application)
+		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1150,7 +1128,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	var buffer []byte
 	switch size := data.Size(); {
 	case size == 0:
-		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+		buffer = make([]byte, 1) // Allocate at least a byte to reach EOF
 	case size >= fi.Erasure.BlockSize:
 		buffer = globalBytePoolCap.Get()
 		defer globalBytePoolCap.Put(buffer)
@@ -1227,7 +1205,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		}
 	}
 
-	if _, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaBucket, key, partsMetadata, writeQuorum); err != nil {
+	if _, err = writeUniqueFileInfo(ctx, onlineDisks, "", minioMetaBucket, key, partsMetadata, writeQuorum); err != nil {
 		return ObjectInfo{}, toObjectErr(err, minioMetaBucket, key)
 	}
 
@@ -1282,7 +1260,9 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry, scanMode m
 
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	data := r.Reader
 
@@ -1309,7 +1289,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// Validate input data size and it can never be less than -1.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.Application)
+		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1317,25 +1297,21 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	storageDisks := er.getDisks()
 
-	parityDrives := len(storageDisks) / 2
-	if !opts.MaxParity {
-		// Get parity and data drive count based on storage class metadata
-		parityDrives = globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
-		if parityDrives < 0 {
-			parityDrives = er.defaultParityCount
-		}
-
+	// Get parity and data drive count based on storage class metadata
+	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
+	if parityDrives < 0 {
+		parityDrives = er.defaultParityCount
+	}
+	if opts.MaxParity {
+		parityDrives = len(storageDisks) / 2
+	}
+	if !opts.MaxParity && globalStorageClass.AvailabilityOptimized() {
 		// If we have offline disks upgrade the number of erasure codes for this object.
 		parityOrig := parityDrives
 
 		var offlineDrives int
 		for _, disk := range storageDisks {
-			if disk == nil {
-				parityDrives++
-				offlineDrives++
-				continue
-			}
-			if !disk.IsOnline() {
+			if disk == nil || !disk.IsOnline() {
 				parityDrives++
 				offlineDrives++
 				continue
@@ -1401,7 +1377,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	var buffer []byte
 	switch size := data.Size(); {
 	case size == 0:
-		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+		buffer = make([]byte, 1) // Allocate at least a byte to reach EOF
 	case size >= fi.Erasure.BlockSize || size == -1:
 		buffer = globalBytePoolCap.Get()
 		defer globalBytePoolCap.Put(buffer)
@@ -1457,7 +1433,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			continue
 		}
 
-		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	toEncode := io.Reader(data)
@@ -1477,7 +1453,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	n, erasureErr := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if erasureErr != nil {
-		return ObjectInfo{}, toObjectErr(erasureErr, minioMetaTmpBucket, tempErasureObj)
+		return ObjectInfo{}, toObjectErr(erasureErr, bucket, object)
 	}
 
 	// Should return IncompleteBody{} error when reader has fewer bytes
@@ -1538,19 +1514,23 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
+	// if storageClass is standard no need to save it as part of metadata.
+	if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD {
+		delete(userDefined, xhttp.AmzStorageClass)
+	}
+
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
 		partsMetadata[index].Metadata = userDefined
 		partsMetadata[index].Size = n
 		partsMetadata[index].ModTime = modTime
-	}
-
-	if len(inlineBuffers) > 0 {
-		// Set an additional header when data is inlined.
-		for index := range partsMetadata {
+		if len(inlineBuffers) > 0 {
 			partsMetadata[index].SetInlineData()
 		}
+		if opts.DataMovement {
+			partsMetadata[index].SetDataMov()
+		}
 	}
 
 	// Rename the successfully written temporary object to final location.
@@ -1635,8 +1615,10 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
 func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
-	for _, obj := range objects {
-		auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
+	if !opts.NoAuditLog {
+		for _, obj := range objects {
+			auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
+		}
 	}
 
 	errs := make([]error, len(objects))
@@ -1805,20 +1787,12 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string) error {
 	disks := er.getDisks()
 	g := errgroup.WithNErrs(len(disks))
-	dirPrefix := encodeDirObject(prefix)
 	for index := range disks {
 		index := index
 		g.Go(func() error {
 			if disks[index] == nil {
 				return nil
 			}
-			// Deletes
-			// - The prefix and its children
-			// - The prefix__XLDIR__
-			defer disks[index].Delete(ctx, bucket, dirPrefix, DeleteOptions{
-				Recursive: true,
-				Immediate: true,
-			})
 			return disks[index].Delete(ctx, bucket, prefix, DeleteOptions{
 				Recursive: true,
 				Immediate: true,
@@ -1837,12 +1811,11 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	if opts.DeletePrefix {
-		if globalCacheConfig.Enabled() {
-			return ObjectInfo{}, toObjectErr(errMethodNotAllowed, bucket, object)
-		}
 		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
 	}
 
@@ -1999,6 +1972,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			ExpireRestored:   opts.Transition.ExpireRestored,
 		}
 		fi.SetTierFreeVersionID(fvID)
+		if opts.SkipFreeVersion {
+			fi.SetSkipTierFreeVersion()
+		}
 		if opts.VersionID != "" {
 			fi.VersionID = opts.VersionID
 		} else if opts.Versioned {
@@ -2028,6 +2004,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		ExpireRestored:   opts.Transition.ExpireRestored,
 	}
 	dfi.SetTierFreeVersionID(fvID)
+	if opts.SkipFreeVersion {
+		dfi.SetSkipTierFreeVersion()
+	}
 	if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}
@@ -2065,7 +2044,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 	// Read metadata associated with the object from all disks.
 	if opts.VersionID != "" {
-		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false, false)
+		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
 		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
 	}
@@ -2138,7 +2117,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 
 	// Read metadata associated with the object from all disks.
 	if opts.VersionID != "" {
-		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false, false)
+		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
 		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
 	}
@@ -2205,7 +2184,7 @@ func (er erasureObjects) updateObjectMetaWithOpts(ctx context.Context, bucket, o
 	// Wait for all the routines.
 	mErrs := g.Wait()
 
-	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, er.defaultWQuorum())
+	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, fi.WriteQuorum(er.defaultWQuorum()))
 }
 
 // updateObjectMeta will update the metadata of a file.
@@ -2466,7 +2445,7 @@ func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object s
 	var onlineDisks []StorageAPI
 	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi)
 
-	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, object, partsMetadata, writeQuorum); err != nil {
+	if _, err := writeUniqueFileInfo(ctx, onlineDisks, "", bucket, object, partsMetadata, writeQuorum); err != nil {
 		return toObjectErr(err, bucket, object)
 	}
 

cmd/erasure-object_test.go

@@ -521,7 +521,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	}
 
 	for _, disk := range xl.getDisks() {
-		files, _ := disk.ListDir(ctx, bucket, object, -1)
+		files, _ := disk.ListDir(ctx, "", bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
 				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
@@ -625,7 +625,7 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, disk := range xl.getDisks() {
-		files, _ := disk.ListDir(ctx, bucket, object, -1)
+		files, _ := disk.ListDir(ctx, "", bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
 				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
@@ -799,7 +799,7 @@ func TestPutObjectNoQuorumSmall(t *testing.T) {
 }
 
 // Test PutObject twice, one small and another bigger
-// than small data thresold and checks reading them again
+// than small data threshold and checks reading them again
 func TestPutObjectSmallInlineData(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -908,7 +908,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "", false, false)
+	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, "", bucket, object1, "", false, false)
 	parts1SC := globalStorageClass
 
 	// Object for test case 2 - No StorageClass defined, MetaData in PutObject requesting RRS Class
@@ -920,7 +920,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "", false, false)
+	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, "", bucket, object2, "", false, false)
 	parts2SC := globalStorageClass
 
 	// Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class
@@ -932,7 +932,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "", false, false)
+	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, "", bucket, object3, "", false, false)
 	parts3SC := globalStorageClass
 
 	// Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class
@@ -950,7 +950,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "", false, false)
+	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, "", bucket, object4, "", false, false)
 	parts4SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 6,
@@ -973,7 +973,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "", false, false)
+	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, "", bucket, object5, "", false, false)
 	parts5SC := globalStorageClass
 
 	// Object for test case 6 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting Standard Storage Class
@@ -994,7 +994,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "", false, false)
+	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, "", bucket, object6, "", false, false)
 	parts6SC := storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
@@ -1017,7 +1017,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "", false, false)
+	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, "", bucket, object7, "", false, false)
 	parts7SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 5,

cmd/erasure-server-pool-decom.go

@@ -26,7 +26,6 @@ import (
 	"math/rand"
 	"net/http"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
@@ -73,9 +72,6 @@ func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
 	if pd == nil {
 		return nil
 	}
-	if pd.StartTime.IsZero() {
-		return nil
-	}
 	return &PoolDecommissionInfo{
 		StartTime:               pd.StartTime,
 		StartSize:               pd.StartSize,
@@ -98,7 +94,7 @@ func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
 
 // bucketPop should be called when a bucket is done decommissioning.
 // Adds the bucket to the list of decommissioned buckets and updates resume numbers.
-func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
+func (pd *PoolDecommissionInfo) bucketPop(bucket string) bool {
 	pd.DecommissionedBuckets = append(pd.DecommissionedBuckets, bucket)
 	for i, b := range pd.QueuedBuckets {
 		if b == bucket {
@@ -110,9 +106,10 @@ func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
 				pd.Prefix = "" // empty this out for the next bucket
 				pd.Object = "" // empty this out for next object
 			}
-			return
+			return true
 		}
 	}
+	return false
 }
 
 func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
@@ -223,12 +220,12 @@ func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool {
 	return p.Pools[idx].Decommission.isBucketDecommissioned(bucket)
 }
 
-func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) {
+func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) bool {
 	if p.Pools[idx].Decommission == nil {
 		// Decommission not in progress.
-		return
+		return false
 	}
-	p.Pools[idx].Decommission.bucketPop(bucket.String())
+	return p.Pools[idx].Decommission.bucketPop(bucket.String())
 }
 
 func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) {
@@ -620,11 +617,12 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:   objInfo.VersionID,
 			UserDefined: objInfo.UserDefined,
+			NoAuditLog:  true,
 		})
 		if err != nil {
 			return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err)
 		}
-		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{})
+		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{NoAuditLog: true})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
 			hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
@@ -639,6 +637,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 					IndexCB: func() []byte {
 						return part.Index // Preserve part Index to ensure decompression works.
 					},
+					NoAuditLog: true,
 				})
 			if err != nil {
 				return fmt.Errorf("decommissionObject: PutObjectPart() %w", err)
@@ -653,7 +652,9 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			}
 		}
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
-			MTime: objInfo.ModTime,
+			DataMovement: true,
+			MTime:        objInfo.ModTime,
+			NoAuditLog:   true,
 		})
 		if err != nil {
 			err = fmt.Errorf("decommissionObject: CompleteMultipartUpload() %w", err)
@@ -665,11 +666,13 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 	if err != nil {
 		return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 	}
+
 	_, err = z.PutObject(ctx,
 		bucket,
 		objInfo.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
+			DataMovement: true,
 			VersionID:    objInfo.VersionID,
 			MTime:        objInfo.ModTime,
 			UserDefined:  objInfo.UserDefined,
@@ -677,6 +680,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			IndexCB: func() []byte {
 				return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works.
 			},
+			NoAuditLog: true,
 		})
 	if err != nil {
 		err = fmt.Errorf("decommissionObject: PutObject() %w", err)
@@ -696,17 +700,18 @@ func (v versionsSorter) reverse() {
 }
 
 func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi decomBucketInfo, fn func(entry metaCacheEntry)) error {
-	disks := set.getOnlineDisks()
+	disks, _ := set.getOnlineDisksWithHealing(false)
 	if len(disks) == 0 {
 		return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints())
 	}
 
-	listQuorum := (len(disks) + 1) / 2
+	// However many we ask, versions must exist on ~50%
+	listingQuorum := (set.setDriveCount + 1) / 2
 
 	// How to resolve partial results.
 	resolver := metadataResolutionParams{
-		dirQuorum: listQuorum, // make sure to capture all quorum ratios
-		objQuorum: listQuorum, // make sure to capture all quorum ratios
+		dirQuorum: listingQuorum, // make sure to capture all quorum ratios
+		objQuorum: listingQuorum, // make sure to capture all quorum ratios
 		bucket:    bi.Name,
 	}
 
@@ -716,7 +721,7 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
 		path:           bi.Prefix,
 		recursive:      true,
 		forwardTo:      "",
-		minDisks:       listQuorum,
+		minDisks:       listingQuorum,
 		reportNotFound: false,
 		agreed:         fn,
 		partial: func(entries metaCacheEntries, _ []error) {
@@ -733,14 +738,15 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
 
-	wStr := env.Get("_MINIO_DECOMMISSION_WORKERS", strconv.Itoa(len(pool.sets)))
-	workerSize, err := strconv.Atoi(wStr)
+	const envDecomWorkers = "_MINIO_DECOMMISSION_WORKERS"
+	workerSize, err := env.GetInt(envDecomWorkers, len(pool.sets))
 	if err != nil {
-		return err
+		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
+		workerSize = len(pool.sets)
 	}
 
-	// each set get its own thread separate from the concurrent
-	// objects/versions being decommissioned.
+	// Each decom worker needs one List() goroutine/worker
+	// add that many extra workers.
 	workerSize += len(pool.sets)
 
 	wk, err := workers.New(workerSize)
@@ -836,6 +842,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							DeleteReplication:  version.ReplicationState,
 							DeleteMarker:       true, // make sure we create a delete marker
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
+							NoAuditLog:         true,
 						})
 					var failure bool
 					if err != nil {
@@ -855,6 +862,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						// Success keep a count.
 						decommissioned++
 					}
+					auditLogDecom(ctx, "DecomCopyDeleteMarker", bi.Name, version.Name, versionID, err)
 					continue
 				}
 
@@ -885,6 +893,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							VersionID:    versionID,
 							NoDecryption: true,
 							NoLock:       true,
+							NoAuditLog:   true,
 						})
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
@@ -936,7 +945,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					bi.Name,
 					encodeDirObject(entry.name),
 					ObjectOptions{
-						DeletePrefix: true, // use prefix delete to delete all versions at once.
+						DeletePrefix:       true, // use prefix delete to delete all versions at once.
+						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
+						NoAuditLog:         true,
 					},
 				)
 				stopFn(err)
@@ -961,6 +972,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// We will perpetually retry listing if it fails, since we cannot
 			// possibly give up in this matter
 			for {
+				if contextCanceled(ctx) {
+					break
+				}
+
 				err := set.listObjectsToDecommission(ctx, bi,
 					func(entry metaCacheEntry) {
 						wk.Take()
@@ -1038,8 +1053,10 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 			}
 
 			z.poolMetaMutex.Lock()
-			z.poolMeta.BucketDone(idx, bucket) // remove from pendingBuckets and persist.
-			z.poolMeta.save(ctx, z.serverPools)
+			if z.poolMeta.BucketDone(idx, bucket) {
+				// remove from pendingBuckets and persist.
+				logger.LogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+			}
 			z.poolMetaMutex.Unlock()
 			continue
 		}
@@ -1054,8 +1071,9 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		stopFn(nil)
 
 		z.poolMetaMutex.Lock()
-		z.poolMeta.BucketDone(idx, bucket)
-		z.poolMeta.save(ctx, z.serverPools)
+		if z.poolMeta.BucketDone(idx, bucket) {
+			logger.LogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+		}
 		z.poolMetaMutex.Unlock()
 	}
 	return nil
@@ -1163,7 +1181,7 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	z.poolMetaMutex.Unlock()
 
 	if !failed {
-		logger.Info("Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine)
+		logger.Event(dctx, "Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine)
 		err := z.checkAfterDecom(dctx, idx)
 		if err != nil {
 			logger.LogIf(ctx, err)

cmd/erasure-server-pool-rebalance.go

@@ -26,16 +26,17 @@ import (
 	"math"
 	"math/rand"
 	"net/http"
-	"strconv"
 	"strings"
-	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/hash"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
+	"github.com/minio/pkg/v2/workers"
 )
 
 //go:generate msgp -file $GOFILE -unexported
@@ -114,12 +115,60 @@ func (z *erasureServerPools) loadRebalanceMeta(ctx context.Context) error {
 	}
 
 	z.rebalMu.Lock()
-	z.rebalMeta = r
+	if len(r.PoolStats) == len(z.serverPools) {
+		z.rebalMeta = r
+	} else {
+		z.updateRebalanceStats(ctx)
+	}
 	z.rebalMu.Unlock()
 
 	return nil
 }
 
+// updates rebalance.bin from let's say 2 pool setup in the middle
+// of a rebalance, was expanded can cause z.rebalMeta to be outdated
+// due to a missing new pool. This function tries to handle this
+// scenario, albeit rare it seems to have occurred in the wild.
+//
+// since we do not explicitly disallow it, but it is okay for them
+// expand and then we continue to rebalance.
+func (z *erasureServerPools) updateRebalanceStats(ctx context.Context) error {
+	var ok bool
+	for i := range z.serverPools {
+		if z.findIndex(i) == -1 {
+			// Also ensure to initialize rebalanceStats to indicate
+			// its a new pool that can receive rebalanced data.
+			z.rebalMeta.PoolStats = append(z.rebalMeta.PoolStats, &rebalanceStats{})
+			ok = true
+		}
+	}
+	if ok {
+		lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
+		lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
+			return err
+		}
+		defer lock.Unlock(lkCtx)
+
+		ctx = lkCtx.Context()
+
+		noLockOpts := ObjectOptions{NoLock: true}
+		return z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
+	}
+
+	return nil
+}
+
+func (z *erasureServerPools) findIndex(index int) int {
+	for i := 0; i < len(z.rebalMeta.PoolStats); i++ {
+		if i == index {
+			return index
+		}
+	}
+	return -1
+}
+
 // initRebalanceMeta initializes rebalance metadata for a new rebalance
 // operation and saves it in the object store.
 func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []string) (arn string, err error) {
@@ -324,7 +373,7 @@ func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool {
 
 func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) (err error) {
 	doneCh := make(chan struct{})
-	defer close(doneCh)
+	defer xioutil.SafeClose(doneCh)
 
 	// Save rebalance.bin periodically.
 	go func() {
@@ -383,6 +432,8 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 		}
 	}()
 
+	logger.Event(ctx, "Pool %d rebalancing is started", poolIdx+1)
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -407,6 +458,8 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 		z.bucketRebalanceDone(bucket, poolIdx)
 	}
 
+	logger.Event(ctx, "Pool %d rebalancing is done", poolIdx+1)
+
 	return err
 }
 
@@ -432,6 +485,41 @@ func (z *erasureServerPools) checkIfRebalanceDone(poolIdx int) bool {
 	return false
 }
 
+func (set *erasureObjects) listObjectsToRebalance(ctx context.Context, bucketName string, fn func(entry metaCacheEntry)) error {
+	disks, _ := set.getOnlineDisksWithHealing(false)
+	if len(disks) == 0 {
+		return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints())
+	}
+
+	// However many we ask, versions must exist on ~50%
+	listingQuorum := (set.setDriveCount + 1) / 2
+
+	// How to resolve partial results.
+	resolver := metadataResolutionParams{
+		dirQuorum: listingQuorum, // make sure to capture all quorum ratios
+		objQuorum: listingQuorum, // make sure to capture all quorum ratios
+		bucket:    bucketName,
+	}
+
+	err := listPathRaw(ctx, listPathRawOptions{
+		disks:          disks,
+		bucket:         bucketName,
+		recursive:      true,
+		forwardTo:      "",
+		minDisks:       listingQuorum,
+		reportNotFound: false,
+		agreed:         fn,
+		partial: func(entries metaCacheEntries, _ []error) {
+			entry, ok := entries.resolve(&resolver)
+			if ok {
+				fn(*entry)
+			}
+		},
+		finished: nil,
+	})
+	return err
+}
+
 // rebalanceBucket rebalances objects under bucket in poolIdx pool
 func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
@@ -443,23 +531,25 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 	rcfg, _ := getReplicationConfig(ctx, bucket)
 
 	pool := z.serverPools[poolIdx]
+
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
-	wStr := env.Get(envRebalanceWorkers, strconv.Itoa(len(pool.sets)))
-	workerSize, err := strconv.Atoi(wStr)
+	workerSize, err := env.GetInt(envRebalanceWorkers, len(pool.sets))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("invalid %s value: %s err: %v, defaulting to %d", envRebalanceWorkers, wStr, err, len(pool.sets)))
+		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
 		workerSize = len(pool.sets)
 	}
-	workers := make(chan struct{}, workerSize)
-	var wg sync.WaitGroup
-	for _, set := range pool.sets {
+
+	// Each decom worker needs one List() goroutine/worker
+	// add that many extra workers.
+	workerSize += len(pool.sets)
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		return err
+	}
+
+	for setIdx, set := range pool.sets {
 		set := set
-		disks := set.getOnlineDisks()
-		if len(disks) == 0 {
-			logger.LogIf(ctx, fmt.Errorf("no online disks found for set with endpoints %s",
-				set.getEndpoints()))
-			continue
-		}
 
 		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 			if lc == nil {
@@ -478,10 +568,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 		}
 
 		rebalanceEntry := func(entry metaCacheEntry) {
-			defer func() {
-				<-workers
-				wg.Done()
-			}()
+			defer wk.Give()
 
 			if entry.isDir() {
 				return
@@ -510,7 +597,6 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bucket, version.Name, version) {
-					rebalanced++
 					expired++
 					continue
 				}
@@ -540,6 +626,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 							DeleteReplication: version.ReplicationState,
 							DeleteMarker:      true, // make sure we create a delete marker
 							SkipRebalancing:   true, // make sure we skip the decommissioned pool
+							NoAuditLog:        true,
 						})
 					var failure bool
 					if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
@@ -551,6 +638,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						z.updatePoolStats(poolIdx, bucket, version)
 						rebalanced++
 					}
+					auditLogRebalance(ctx, "Rebalance:DeleteMarker", bucket, version.Name, versionID, err)
 					continue
 				}
 
@@ -567,6 +655,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 							VersionID:    versionID,
 							NoDecryption: true,
 							NoLock:       true,
+							NoAuditLog:   true,
 						})
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
@@ -609,7 +698,9 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					bucket,
 					encodeDirObject(entry.name),
 					ObjectOptions{
-						DeletePrefix: true, // use prefix delete to delete all versions at once.
+						DeletePrefix:       true, // use prefix delete to delete all versions at once.
+						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
+						NoAuditLog:         true,
 					},
 				)
 				stopFn(err)
@@ -620,44 +711,24 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			}
 		}
 
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			listQuorum := (len(disks) + 1) / 2
-
-			// How to resolve partial results.
-			resolver := metadataResolutionParams{
-				dirQuorum: listQuorum, // make sure to capture all quorum ratios
-				objQuorum: listQuorum, // make sure to capture all quorum ratios
-				bucket:    bucket,
-			}
-			err := listPathRaw(ctx, listPathRawOptions{
-				disks:          disks,
-				bucket:         bucket,
-				recursive:      true,
-				forwardTo:      "",
-				minDisks:       listQuorum,
-				reportNotFound: false,
-				agreed: func(entry metaCacheEntry) {
-					workers <- struct{}{}
-					wg.Add(1)
+		wk.Take()
+		go func(setIdx int) {
+			defer wk.Give()
+			err := set.listObjectsToRebalance(ctx, bucket,
+				func(entry metaCacheEntry) {
+					wk.Take()
 					go rebalanceEntry(entry)
 				},
-				partial: func(entries metaCacheEntries, _ []error) {
-					entry, ok := entries.resolve(&resolver)
-					if ok {
-						workers <- struct{}{}
-						wg.Add(1)
-						go rebalanceEntry(*entry)
-					}
-				},
-				finished: nil,
-			})
-			logger.LogIf(ctx, err)
-		}()
+			)
+			if err == nil || errors.Is(err, context.Canceled) {
+				return
+			}
+			setN := humanize.Ordinal(setIdx + 1)
+			logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN)
+		}(setIdx)
 	}
-	wg.Wait()
+
+	wk.Wait()
 	return nil
 }
 
@@ -695,8 +766,7 @@ func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int
 	}
 	z.rebalMeta = r
 
-	err = z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
-	return err
+	return z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
 }
 
 func auditLogRebalance(ctx context.Context, apiName, bucket, object, versionID string, err error) {
@@ -731,11 +801,12 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 		res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
 			VersionID:   oi.VersionID,
 			UserDefined: oi.UserDefined,
+			NoAuditLog:  true,
 		})
 		if err != nil {
 			return fmt.Errorf("rebalanceObject: NewMultipartUpload() %w", err)
 		}
-		defer z.AbortMultipartUpload(ctx, bucket, oi.Name, res.UploadID, ObjectOptions{})
+		defer z.AbortMultipartUpload(ctx, bucket, oi.Name, res.UploadID, ObjectOptions{NoAuditLog: true})
 
 		parts := make([]CompletePart, len(oi.Parts))
 		for i, part := range oi.Parts {
@@ -751,6 +822,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 					IndexCB: func() []byte {
 						return part.Index // Preserve part Index to ensure decompression works.
 					},
+					NoAuditLog: true,
 				})
 			if err != nil {
 				return fmt.Errorf("rebalanceObject: PutObjectPart() %w", err)
@@ -761,7 +833,9 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 			}
 		}
 		_, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{
-			MTime: oi.ModTime,
+			DataMovement: true,
+			MTime:        oi.ModTime,
+			NoAuditLog:   true,
 		})
 		if err != nil {
 			err = fmt.Errorf("rebalanceObject: CompleteMultipartUpload() %w", err)
@@ -773,11 +847,13 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 	if err != nil {
 		return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
 	}
+
 	_, err = z.PutObject(ctx,
 		bucket,
 		oi.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
+			DataMovement: true,
 			VersionID:    oi.VersionID,
 			MTime:        oi.ModTime,
 			UserDefined:  oi.UserDefined,
@@ -785,6 +861,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 			IndexCB: func() []byte {
 				return oi.Parts[0].Index // Preserve part Index to ensure decompression works.
 			},
+			NoAuditLog: true,
 		})
 	if err != nil {
 		err = fmt.Errorf("rebalanceObject: PutObject() %w", err)