fix: precondition check for multipart with existing object replication (#19349)

Fix races in IAM cache

Fixes #19344

On the top level we only grab a read lock, but we write to the cache if we manage to fetch it.

https://github.com/minio/minio/blob/a03dac41ebd2c1b9d3f1110ef5d299ffebb8f87b/cmd/iam-store.go#L446 is also flipped to what it should be AFAICT.

Change the internal cache structure to a concurrency safe implementation.

Bonus: Also switch grid implementation.

.github/workflows/vulncheck.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21.5
+          go-version: 1.21.8
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

CREDITS

@@ -1125,6 +1125,39 @@ https://cloud.google.com/go/storage
 
 ================================================================
 
+filippo.io/edwards25519
+https://filippo.io/edwards25519
+----------------------------------------------------------------
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/Azure/azure-pipeline-go
 https://github.com/Azure/azure-pipeline-go
 ----------------------------------------------------------------
@@ -3353,213 +3386,6 @@ Redistribution and use in source and binary forms, with or without modification,
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ================================================================
 
-github.com/cncf/xds/go
-https://github.com/cncf/xds/go
-----------------------------------------------------------------
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/containerd/console
 https://github.com/containerd/console
 ----------------------------------------------------------------
@@ -5767,214 +5593,6 @@ https://github.com/elastic/go-elasticsearch/v7
 
 ================================================================
 
-github.com/envoyproxy/protoc-gen-validate
-https://github.com/envoyproxy/protoc-gen-validate
-----------------------------------------------------------------
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/fatih/color
 https://github.com/fatih/color
 ----------------------------------------------------------------
@@ -18064,8 +17682,8 @@ https://github.com/minio/highwayhash
 
 ================================================================
 
-github.com/minio/kes-go
-https://github.com/minio/kes-go
+github.com/minio/kms-go/kes
+https://github.com/minio/kms-go/kes
 ----------------------------------------------------------------
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
@@ -27445,33 +27063,6 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
       the Mozilla Public License, v. 2.0.
 
 
-================================================================
-
-github.com/sirupsen/logrus
-https://github.com/sirupsen/logrus
-----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) 2014 Simon Eskildsen
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
 ================================================================
 
 github.com/stretchr/testify
@@ -32432,36 +32023,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-gopkg.in/h2non/filetype.v1
-https://gopkg.in/h2non/filetype.v1
-----------------------------------------------------------------
-The MIT License
-
-Copyright (c) Tomas Aparicio
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-================================================================
-
 gopkg.in/ini.v1
 https://gopkg.in/ini.v1
 ----------------------------------------------------------------

Makefile

@@ -24,7 +24,7 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.10-0.20240227114326-6d6f813fff1b
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio

buildscripts/heal-manual.go

@@ -44,7 +44,6 @@ func main() {
 	opts := madmin.HealOpts{
 		Recursive: true,                  // recursively heal all objects at 'prefix'
 		Remove:    true,                  // remove content that has lost quorum and not recoverable
-		Recreate:  true,                  // rewrite all old non-inlined xl.meta to new xl.meta
 		ScanMode:  madmin.HealNormalScan, // by default do not do 'deep' scanning
 	}
 

buildscripts/verify-healing.sh

@@ -83,7 +83,7 @@ function start_minio_3_node() {
 }
 
 function check_online() {
-	if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
+	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
 		echo "1"
 	fi
 }
@@ -109,6 +109,7 @@ function perform_test() {
 
 	rm -rf ${WORK_DIR}/${1}/*/
 
+	set -x
 	start_minio_3_node 120 $2
 
 	rv=$(check_online)

cmd/admin-bucket-handlers.go

@@ -31,7 +31,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -687,7 +687,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 					rpt.SetStatus(bucket, fileName, err)
 					continue
 				}
-				v := newBucketMetadata(bucket)
+				v, _ := globalBucketMetadataSys.Get(bucket)
 				bucketMap[bucket] = &v
 			}
 
@@ -723,7 +723,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 					rpt.SetStatus(bucket, fileName, err)
 					continue
 				}
-				v := newBucketMetadata(bucket)
+				v, _ := globalBucketMetadataSys.Get(bucket)
 				bucketMap[bucket] = &v
 			}
 
@@ -776,7 +776,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				rpt.SetStatus(bucket, "", err)
 				continue
 			}
-			v := newBucketMetadata(bucket)
+			v, _ := globalBucketMetadataSys.Get(bucket)
 			bucketMap[bucket] = &v
 		}
 		if _, ok := bucketMap[bucket]; !ok {

cmd/admin-handler-utils.go

@@ -23,7 +23,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"

cmd/admin-handlers-idp-ldap.go

@@ -193,6 +193,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 	// fail if ldap is not enabled
 	if !globalIAMSys.LDAPConfig.Enabled() {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("LDAP not enabled")), r.URL)
+		return
 	}
 
 	// Find the user for the request sender (as it may be sent via a service

cmd/admin-handlers-users.go

@@ -33,10 +33,12 @@ import (
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
+	"github.com/puzpuzpuz/xsync/v3"
 )
 
 // RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
@@ -1197,20 +1199,15 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	bucketStorageCache.Once.Do(func() {
-		// Set this to 10 secs since its enough, as scanner
-		// does not update the bucket usage values frequently.
-		bucketStorageCache.TTL = 10 * time.Second
-
-		// Rely on older value if usage loading fails from disk.
-		bucketStorageCache.Relax = true
-		bucketStorageCache.Update = func() (DataUsageInfo, error) {
+	bucketStorageCache.InitOnce(10*time.Second,
+		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
+		func() (DataUsageInfo, error) {
 			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
 			defer done()
 
 			return loadDataUsageFromBackend(ctx, objectAPI)
-		}
-	})
+		},
+	)
 
 	dataUsageInfo, _ := bucketStorageCache.Get()
 
@@ -1940,13 +1937,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case userPolicyMappingsFile:
-			userPolicyMap := make(map[string]MappedPolicy)
+			userPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-			userPolData, err := json.Marshal(userPolicyMap)
+			userPolData, err := json.Marshal(mappedPoliciesToMap(userPolicyMap))
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -1957,13 +1954,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case groupPolicyMappingsFile:
-			groupPolicyMap := make(map[string]MappedPolicy)
+			groupPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-			grpPolData, err := json.Marshal(groupPolicyMap)
+			grpPolData, err := json.Marshal(mappedPoliciesToMap(groupPolicyMap))
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -1974,13 +1971,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case stsUserPolicyMappingsFile:
-			userPolicyMap := make(map[string]MappedPolicy)
+			userPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-			userPolData, err := json.Marshal(userPolicyMap)
+			userPolData, err := json.Marshal(mappedPoliciesToMap(userPolicyMap))
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -1990,13 +1987,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case stsGroupPolicyMappingsFile:
-			groupPolicyMap := make(map[string]MappedPolicy)
+			groupPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-			grpPolData, err := json.Marshal(groupPolicyMap)
+			grpPolData, err := json.Marshal(mappedPoliciesToMap(groupPolicyMap))
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return

cmd/admin-handlers.go

@@ -159,7 +159,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
 					Host:           nerr.Host.String(),
 					CurrentVersion: Version,
-					UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+					UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 				}
 			}
 		}
@@ -176,7 +176,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 			peerResults[local] = madmin.ServerPeerUpdateStatus{
 				Host:           local,
 				CurrentVersion: Version,
-				UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+				UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 			}
 		}
 	} else {
@@ -212,7 +212,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 							Host:           nerr.Host.String(),
 							Err:            nerr.Err.Error(),
 							CurrentVersion: Version,
-							UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+							UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 						}
 					}
 				}
@@ -404,7 +404,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	updateStatus := madmin.ServerUpdateStatus{
 		CurrentVersion: Version,
-		UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+		UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 	}
 
 	// Marshal API response
@@ -2280,8 +2280,6 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
 	servers := globalNotificationSys.ServerInfo(metrics)
 	servers = append(servers, local)
 
-	assignPoolNumbers(servers)
-
 	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
 	var backend madmin.ErasureBackend
 
@@ -2718,14 +2716,20 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
 				servers = append(servers, madmin.ServerInfo{
-					State:      server.State,
-					Endpoint:   anonEndpoint,
-					Uptime:     server.Uptime,
-					Version:    server.Version,
-					CommitID:   server.CommitID,
-					Network:    anonymizeNetwork(server.Network),
-					Drives:     anonymizeDrives(server.Disks),
-					PoolNumber: server.PoolNumber,
+					State:    server.State,
+					Endpoint: anonEndpoint,
+					Uptime:   server.Uptime,
+					Version:  server.Version,
+					CommitID: server.CommitID,
+					Network:  anonymizeNetwork(server.Network),
+					Drives:   anonymizeDrives(server.Disks),
+					PoolNumber: func() int {
+						if len(server.PoolNumbers) == 1 {
+							return server.PoolNumbers[0]
+						}
+						return math.MaxInt // this indicates that its unset.
+					}(),
+					PoolNumbers: server.PoolNumbers,
 					MemStats: madmin.MemStats{
 						Alloc:      server.MemStats.Alloc,
 						TotalAlloc: server.MemStats.TotalAlloc,
@@ -2915,22 +2919,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-func assignPoolNumbers(servers []madmin.ServerProperties) {
-	for i := range servers {
-		for idx, ge := range globalEndpoints {
-			for _, endpoint := range ge.Endpoints {
-				if servers[i].Endpoint == endpoint.Host {
-					servers[i].PoolNumber = idx + 1
-				} else if host, err := xnet.ParseHost(servers[i].Endpoint); err == nil {
-					if host.Name == endpoint.Hostname() {
-						servers[i].PoolNumber = idx + 1
-					}
-				}
-			}
-		}
-	}
-}
-
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 

cmd/admin-router.go

@@ -19,9 +19,6 @@ package cmd
 
 import (
 	"net/http"
-	"reflect"
-	"runtime"
-	"strings"
 
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
@@ -63,20 +60,12 @@ const (
 	noObjLayerFlag
 )
 
-// Has checks if the the given flag is enabled in `h`.
+// Has checks if the given flag is enabled in `h`.
 func (h hFlag) Has(flag hFlag) bool {
 	// Use bitwise-AND and check if the result is non-zero.
 	return h&flag != 0
 }
 
-func getHandlerName(f http.HandlerFunc) string {
-	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
-	name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.adminAPIHandlers.")
-	name = strings.TrimSuffix(name, "Handler-fm")
-	name = strings.TrimSuffix(name, "-fm")
-	return name
-}
-
 // adminMiddleware performs some common admin handler functionality for all
 // handlers:
 //
@@ -86,9 +75,13 @@ func getHandlerName(f http.HandlerFunc) string {
 //
 // - sets up call to send AuditLog
 //
-// Note that, while this is a middleware function (i.e. it takes a handler
-// function and returns one), due to flags being passed based on required
-// conditions, it is done per-"handler function registration" in the router.
+// While this is a middleware function (i.e. it takes a handler function and
+// returns one), due to flags being passed based on required conditions, it is
+// done per-"handler function registration" in the router.
+//
+// The passed in handler function must be a method of `adminAPIHandlers` for the
+// name displayed in logs and trace to be accurate. The name is extracted via
+// reflection.
 //
 // When no flags are passed, gzip compression, http tracing of headers and
 // checking of object layer availability are all enabled. Use flags to modify
@@ -100,10 +93,8 @@ func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc {
 		handlerFlags |= flag
 	}
 
-	// Get name of the handler using reflection. NOTE: The passed in handler
-	// function must be a method of `adminAPIHandlers` for this extraction to
-	// work as expected.
-	handlerName := getHandlerName(f)
+	// Get name of the handler using reflection.
+	handlerName := getHandlerName(f, "adminAPIHandlers")
 
 	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
 		// Update request context with `logger.ReqInfo`.

cmd/admin-server-info.go

@@ -19,10 +19,12 @@ package cmd
 
 import (
 	"context"
+	"math"
 	"net/http"
 	"os"
 	"runtime"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"time"
 
@@ -42,9 +44,13 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	if globalIsDistErasure {
 		addr = globalLocalNodeName
 	}
+	poolNumbers := make(map[int]struct{})
 	network := make(map[string]string)
 	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				poolNumbers[endpoint.PoolIdx+1] = struct{}{}
+			}
 			nodeName := endpoint.Host
 			if nodeName == "" {
 				nodeName = addr
@@ -112,6 +118,17 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		MinioEnvVars: make(map[string]string, 10),
 	}
 
+	for poolNumber := range poolNumbers {
+		props.PoolNumbers = append(props.PoolNumbers, poolNumber)
+	}
+	sort.Ints(props.PoolNumbers)
+	props.PoolNumber = func() int {
+		if len(props.PoolNumbers) == 1 {
+			return props.PoolNumbers[0]
+		}
+		return math.MaxInt // this indicates that its unset.
+	}()
+
 	sensitive := map[string]struct{}{
 		config.EnvAccessKey:         {},
 		config.EnvSecretKey:         {},

cmd/api-headers.go

@@ -50,7 +50,7 @@ func setEventStreamHeaders(w http.ResponseWriter) {
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	// Set the "Server" http header.
-	w.Header().Set(xhttp.ServerInfo, "MinIO")
+	w.Header().Set(xhttp.ServerInfo, MinioStoreName)
 
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.

cmd/api-router.go

@@ -18,14 +18,11 @@
 package cmd
 
 import (
-	"compress/gzip"
 	"net"
 	"net/http"
 
-	"github.com/klauspost/compress/gzhttp"
 	consoleapi "github.com/minio/console/api"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/wildcard"
 	"github.com/rs/cors"
@@ -171,6 +168,87 @@ var rejectedBucketAPIs = []rejectedAPI{
 	},
 }
 
+// Set of s3 handler options as bit flags.
+type s3HFlag uint8
+
+const (
+	// when provided, disables Gzip compression.
+	noGZS3HFlag = 1 << iota
+
+	// when provided, enables only tracing of headers. Otherwise, both headers
+	// and body are traced.
+	traceHdrsS3HFlag
+
+	// when provided, disables throttling via the `maxClients` middleware.
+	noThrottleS3HFlag
+)
+
+func (h s3HFlag) has(flag s3HFlag) bool {
+	// Use bitwise-AND and check if the result is non-zero.
+	return h&flag != 0
+}
+
+// s3APIMiddleware - performs some common handler functionality for S3 API
+// handlers.
+//
+// It is set per-"handler function registration" in the router to allow for
+// behavior modification via flags.
+//
+// This middleware always calls `collectAPIStats` to collect API stats.
+//
+// The passed in handler function must be a method of `objectAPIHandlers` for
+// the name displayed in logs and trace to be accurate. The name is extracted
+// via reflection.
+//
+// When **no** flags are passed, the behavior is to trace both headers and body,
+// gzip the response and throttle the handler via `maxClients`. Each of these
+// can be disabled via the corresponding `s3HFlag`.
+//
+// CAUTION: for requests involving large req/resp bodies ensure to pass the
+// `traceHdrsS3HFlag`, otherwise both headers and body will be traced, causing
+// high memory usage!
+func s3APIMiddleware(f http.HandlerFunc, flags ...s3HFlag) http.HandlerFunc {
+	// Collect all flags with bitwise-OR and assign operator
+	var handlerFlags s3HFlag
+	for _, flag := range flags {
+		handlerFlags |= flag
+	}
+
+	// Get name of the handler using reflection.
+	handlerName := getHandlerName(f, "objectAPIHandlers")
+
+	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		// Wrap the actual handler with the appropriate tracing middleware.
+		var tracedHandler http.HandlerFunc
+		if handlerFlags.has(traceHdrsS3HFlag) {
+			tracedHandler = httpTraceHdrs(f)
+		} else {
+			tracedHandler = httpTraceAll(f)
+		}
+
+		// Skip wrapping with the gzip middleware if specified.
+		var gzippedHandler http.HandlerFunc = tracedHandler
+		if !handlerFlags.has(noGZS3HFlag) {
+			gzippedHandler = gzipHandler(gzippedHandler)
+		}
+
+		// Skip wrapping with throttling middleware if specified.
+		var throttledHandler http.HandlerFunc = gzippedHandler
+		if !handlerFlags.has(noThrottleS3HFlag) {
+			throttledHandler = maxClients(throttledHandler)
+		}
+
+		// Collect API stats using the API name got from reflection in
+		// `getHandlerName`.
+		statsCollectedHandler := collectAPIStats(handlerName, throttledHandler)
+
+		// Call the final handler.
+		statsCollectedHandler(w, r)
+	}
+
+	return handler
+}
+
 // registerAPIRouter - registers S3 compatible APIs.
 func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
@@ -208,12 +286,6 @@ func registerAPIRouter(router *mux.Router) {
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
-	if err != nil {
-		// Static params, so this is very unlikely.
-		logger.Fatal(err, "Unable to initialize server")
-	}
-
 	for _, router := range routers {
 		// Register all rejected object APIs
 		for _, r := range rejectedObjAPIs {
@@ -225,240 +297,307 @@ func registerAPIRouter(router *mux.Router) {
 
 		// Object operations
 		// HeadObject
-		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
+		router.Methods(http.MethodHead).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.HeadObjectHandler))
+
 		// GetObjectAttributes
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectAttributesHandler, traceHdrsS3HFlag)).
+			Queries("attributes", "")
+
 		// CopyObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
+			HandlerFunc(s3APIMiddleware(api.CopyObjectPartHandler)).
 			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectPartHandler, traceHdrsS3HFlag)).
+			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.ListObjectPartsHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("completemultipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.CompleteMultipartUploadHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("newmultipartupload", maxClients(gz(httpTraceAll(api.NewMultipartUploadHandler))))).Queries("uploads", "")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.NewMultipartUploadHandler)).
+			Queries("uploads", "")
 		// AbortMultipartUpload
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("abortmultipartupload", maxClients(gz(httpTraceAll(api.AbortMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.AbortMultipartUploadHandler)).
+			Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectacl", maxClients(gz(httpTraceHdrs(api.GetObjectACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectACLHandler, traceHdrsS3HFlag)).
+			Queries("acl", "")
 		// PutObjectACL - this is a dummy call.
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectacl", maxClients(gz(httpTraceHdrs(api.PutObjectACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectACLHandler, traceHdrsS3HFlag)).
+			Queries("acl", "")
 		// GetObjectTagging
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjecttagging", maxClients(gz(httpTraceHdrs(api.GetObjectTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// PutObjectTagging
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjecttagging", maxClients(gz(httpTraceHdrs(api.PutObjectTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// DeleteObjectTagging
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("deleteobjecttagging", maxClients(gz(httpTraceHdrs(api.DeleteObjectTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.DeleteObjectTaggingHandler, traceHdrsS3HFlag)).
+			Queries("tagging", "")
 		// SelectObjectContent
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("selectobjectcontent", maxClients(gz(httpTraceHdrs(api.SelectObjectContentHandler))))).Queries("select", "").Queries("select-type", "2")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.SelectObjectContentHandler, traceHdrsS3HFlag)).
+			Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectretention", maxClients(gz(httpTraceAll(api.GetObjectRetentionHandler))))).Queries("retention", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectRetentionHandler)).
+			Queries("retention", "")
 		// GetObjectLegalHold
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectLegalHoldHandler)).
+			Queries("legal-hold", "")
 		// GetObject with lambda ARNs
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectLambdaHandler, traceHdrsS3HFlag)).
+			Queries("lambdaArn", "{lambdaArn:.*}")
 		// GetObject
-		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
+		router.Methods(http.MethodGet).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.GetObjectHandler, traceHdrsS3HFlag))
 		// CopyObject
-		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
-			collectAPIStats("copyobject", maxClients(gz(httpTraceAll(api.CopyObjectHandler)))))
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(s3APIMiddleware(api.CopyObjectHandler))
 		// PutObjectRetention
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectretention", maxClients(gz(httpTraceAll(api.PutObjectRetentionHandler))))).Queries("retention", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectRetentionHandler)).
+			Queries("retention", "")
 		// PutObjectLegalHold
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectlegalhold", maxClients(gz(httpTraceAll(api.PutObjectLegalHoldHandler))))).Queries("legal-hold", "")
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectLegalHoldHandler)).
+			Queries("legal-hold", "")
 
 		// PutObject with auto-extract support for zip
-		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc(
-			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler)))))
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzSnowballExtract, "true").
+			HandlerFunc(s3APIMiddleware(api.PutObjectExtractHandler, traceHdrsS3HFlag))
 
 		// PutObject
-		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler)))))
+		router.Methods(http.MethodPut).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PutObjectHandler, traceHdrsS3HFlag))
 
 		// DeleteObject
-		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("deleteobject", maxClients(gz(httpTraceAll(api.DeleteObjectHandler)))))
+		router.Methods(http.MethodDelete).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.DeleteObjectHandler))
 
 		// PostRestoreObject
-		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("restoreobject", maxClients(gz(httpTraceAll(api.PostRestoreObjectHandler))))).Queries("restore", "")
+		router.Methods(http.MethodPost).Path("/{object:.+}").
+			HandlerFunc(s3APIMiddleware(api.PostRestoreObjectHandler)).
+			Queries("restore", "")
 
 		// Bucket operations
 
 		// GetBucketLocation
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlocation", maxClients(gz(httpTraceAll(api.GetBucketLocationHandler))))).Queries("location", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketLocationHandler)).
+			Queries("location", "")
 		// GetBucketPolicy
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketpolicy", maxClients(gz(httpTraceAll(api.GetBucketPolicyHandler))))).Queries("policy", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketPolicyHandler)).
+			Queries("policy", "")
 		// GetBucketLifecycle
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlifecycle", maxClients(gz(httpTraceAll(api.GetBucketLifecycleHandler))))).Queries("lifecycle", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// GetBucketEncryption
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketencryption", maxClients(gz(httpTraceAll(api.GetBucketEncryptionHandler))))).Queries("encryption", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketEncryptionHandler)).
+			Queries("encryption", "")
 		// GetBucketObjectLockConfig
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketobjectlockconfiguration", maxClients(gz(httpTraceAll(api.GetBucketObjectLockConfigHandler))))).Queries("object-lock", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketObjectLockConfigHandler)).
+			Queries("object-lock", "")
 		// GetBucketReplicationConfig
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.GetBucketReplicationConfigHandler))))).Queries("replication", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// GetBucketVersioning
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketversioning", maxClients(gz(httpTraceAll(api.GetBucketVersioningHandler))))).Queries("versioning", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketVersioningHandler)).
+			Queries("versioning", "")
 		// GetBucketNotification
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketNotificationHandler)).
+			Queries("notification", "")
 		// ListenNotification
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)).
+			Queries("events", "{events:.*}")
 		// ResetBucketReplicationStatus - MinIO extension API
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStatusHandler)).
+			Queries("replication-reset-status", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketacl", maxClients(gz(httpTraceAll(api.GetBucketACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketACLHandler)).
+			Queries("acl", "")
 		// PutBucketACL -- this is a dummy call.
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketacl", maxClients(gz(httpTraceAll(api.PutBucketACLHandler))))).Queries("acl", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
+			Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketcors", maxClients(gz(httpTraceAll(api.GetBucketCorsHandler))))).Queries("cors", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
+			Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketwebsite", maxClients(gz(httpTraceAll(api.GetBucketWebsiteHandler))))).Queries("website", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)).
+			Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketaccelerate", maxClients(gz(httpTraceAll(api.GetBucketAccelerateHandler))))).Queries("accelerate", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketAccelerateHandler)).
+			Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketrequestpayment", maxClients(gz(httpTraceAll(api.GetBucketRequestPaymentHandler))))).Queries("requestPayment", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketRequestPaymentHandler)).
+			Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlogging", maxClients(gz(httpTraceAll(api.GetBucketLoggingHandler))))).Queries("logging", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)).
+			Queries("logging", "")
 		// GetBucketTaggingHandler
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbuckettagging", maxClients(gz(httpTraceAll(api.GetBucketTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)).
+			Queries("tagging", "")
 		// DeleteBucketWebsiteHandler
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketwebsite", maxClients(gz(httpTraceAll(api.DeleteBucketWebsiteHandler))))).Queries("website", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketWebsiteHandler)).
+			Queries("website", "")
 		// DeleteBucketTaggingHandler
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebuckettagging", maxClients(gz(httpTraceAll(api.DeleteBucketTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketTaggingHandler)).
+			Queries("tagging", "")
 
 		// ListMultipartUploads
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listmultipartuploads", maxClients(gz(httpTraceAll(api.ListMultipartUploadsHandler))))).Queries("uploads", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListMultipartUploadsHandler)).
+			Queries("uploads", "")
 		// ListObjectsV2M
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv2M", maxClients(gz(httpTraceAll(api.ListObjectsV2MHandler))))).Queries("list-type", "2", "metadata", "true")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV2MHandler)).
+			Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV2Handler)).
+			Queries("list-type", "2")
 		// ListObjectVersions
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectVersionsMHandler)).
+			Queries("versions", "", "metadata", "true")
 		// ListObjectVersions
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectVersionsHandler)).
+			Queries("versions", "")
 		// GetBucketPolicyStatus
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getpolicystatus", maxClients(gz(httpTraceAll(api.GetBucketPolicyStatusHandler))))).Queries("policyStatus", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketPolicyStatusHandler)).
+			Queries("policyStatus", "")
 		// PutBucketLifecycle
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketlifecycle", maxClients(gz(httpTraceAll(api.PutBucketLifecycleHandler))))).Queries("lifecycle", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// PutBucketReplicationConfig
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.PutBucketReplicationConfigHandler))))).Queries("replication", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// PutBucketEncryption
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketencryption", maxClients(gz(httpTraceAll(api.PutBucketEncryptionHandler))))).Queries("encryption", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketEncryptionHandler)).
+			Queries("encryption", "")
 
 		// PutBucketPolicy
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketpolicy", maxClients(gz(httpTraceAll(api.PutBucketPolicyHandler))))).Queries("policy", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketPolicyHandler)).
+			Queries("policy", "")
 
 		// PutBucketObjectLockConfig
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.PutBucketObjectLockConfigHandler))))).Queries("object-lock", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketObjectLockConfigHandler)).
+			Queries("object-lock", "")
 		// PutBucketTaggingHandler
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbuckettagging", maxClients(gz(httpTraceAll(api.PutBucketTaggingHandler))))).Queries("tagging", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketTaggingHandler)).
+			Queries("tagging", "")
 		// PutBucketVersioning
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketversioning", maxClients(gz(httpTraceAll(api.PutBucketVersioningHandler))))).Queries("versioning", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketVersioningHandler)).
+			Queries("versioning", "")
 		// PutBucketNotification
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketnotification", maxClients(gz(httpTraceAll(api.PutBucketNotificationHandler))))).Queries("notification", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketNotificationHandler)).
+			Queries("notification", "")
 		// ResetBucketReplicationStart - MinIO extension API
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("resetbucketreplicationstart", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStartHandler))))).Queries("replication-reset", "")
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStartHandler)).
+			Queries("replication-reset", "")
 
 		// PutBucket
-		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucket", maxClients(gz(httpTraceAll(api.PutBucketHandler)))))
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketHandler))
 		// HeadBucket
-		router.Methods(http.MethodHead).HandlerFunc(
-			collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
+		router.Methods(http.MethodHead).
+			HandlerFunc(s3APIMiddleware(api.HeadBucketHandler))
 		// PostPolicy
-		router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
-			return isRequestPostPolicySignatureV4(r)
-		}).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
+		router.Methods(http.MethodPost).
+			MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
+				return isRequestPostPolicySignatureV4(r)
+			}).
+			HandlerFunc(s3APIMiddleware(api.PostPolicyBucketHandler, traceHdrsS3HFlag))
 		// DeleteMultipleObjects
-		router.Methods(http.MethodPost).HandlerFunc(
-			collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")
+		router.Methods(http.MethodPost).
+			HandlerFunc(s3APIMiddleware(api.DeleteMultipleObjectsHandler)).
+			Queries("delete", "")
 		// DeleteBucketPolicy
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketpolicy", maxClients(gz(httpTraceAll(api.DeleteBucketPolicyHandler))))).Queries("policy", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketPolicyHandler)).
+			Queries("policy", "")
 		// DeleteBucketReplication
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.DeleteBucketReplicationConfigHandler))))).Queries("replication", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketReplicationConfigHandler)).
+			Queries("replication", "")
 		// DeleteBucketLifecycle
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketlifecycle", maxClients(gz(httpTraceAll(api.DeleteBucketLifecycleHandler))))).Queries("lifecycle", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketLifecycleHandler)).
+			Queries("lifecycle", "")
 		// DeleteBucketEncryption
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketencryption", maxClients(gz(httpTraceAll(api.DeleteBucketEncryptionHandler))))).Queries("encryption", "")
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketEncryptionHandler)).
+			Queries("encryption", "")
 		// DeleteBucket
-		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucket", maxClients(gz(httpTraceAll(api.DeleteBucketHandler)))))
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketHandler))
 
 		// MinIO extension API for replication.
 		//
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsV2Handler)).
+			Queries("replication-metrics", "2")
 		// deprecated handler
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsHandler)).
+			Queries("replication-metrics", "")
 
 		// ValidateBucketReplicationCreds
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ValidateBucketReplicationCredsHandler)).
+			Queries("replication-check", "")
 
 		// Register rejected bucket APIs
 		for _, r := range rejectedBucketAPIs {
@@ -468,24 +607,25 @@ func registerAPIRouter(router *mux.Router) {
 		}
 
 		// S3 ListObjectsV1 (Legacy)
-		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv1", maxClients(gz(httpTraceAll(api.ListObjectsV1Handler)))))
+		router.Methods(http.MethodGet).
+			HandlerFunc(s3APIMiddleware(api.ListObjectsV1Handler))
 	}
 
 	// Root operation
 
 	// ListenNotification
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).
+		HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)).
+		Queries("events", "{events:.*}")
 
 	// ListBuckets
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).
+		HandlerFunc(s3APIMiddleware(api.ListBucketsHandler))
 
 	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
 	// than failing with UnknownAPIRequest we simply handle it for now.
-	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
-		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).
+		HandlerFunc(s3APIMiddleware(api.ListBucketsHandler))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))

cmd/api-utils.go

@@ -18,6 +18,10 @@
 package cmd
 
 import (
+	"fmt"
+	"net/http"
+	"reflect"
+	"runtime"
 	"strings"
 )
 
@@ -100,3 +104,16 @@ func s3EncodeName(name, encodingType string) string {
 	}
 	return name
 }
+
+// getHandlerName returns the name of the handler function. It takes the type
+// name as a string to clean up the name retrieved via reflection. This function
+// only works correctly when the type is present in the cmd package.
+func getHandlerName(f http.HandlerFunc, cmdType string) string {
+	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
+
+	packageName := fmt.Sprintf("github.com/minio/minio/cmd.%s.", cmdType)
+	name = strings.TrimPrefix(name, packageName)
+	name = strings.TrimSuffix(name, "Handler-fm")
+	name = strings.TrimSuffix(name, "-fm")
+	return name
+}

cmd/auth-handler.go

@@ -298,8 +298,17 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 	if cred.IsTemp() && cred.IsExpired() {
 		return nil, toAPIErrorCode(r.Context(), errInvalidAccessKeyID)
 	}
-
 	secret := globalActiveCred.SecretKey
+	if globalSiteReplicationSys.isEnabled() && cred.AccessKey != siteReplicatorSvcAcc {
+		nsecret, err := getTokenSigningKey()
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		// sign root's temporary accounts also with site replicator creds
+		if cred.ParentUser != globalActiveCred.AccessKey || cred.IsTemp() {
+			secret = nsecret
+		}
+	}
 	if cred.IsServiceAccount() {
 		token = cred.SessionToken
 		secret = cred.SecretKey

cmd/background-heal-ops.go

@@ -102,7 +102,6 @@ func waitForLowHTTPReq() {
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
 	for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
 		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI)
 	}

cmd/background-newdisks-heal-ops.go

@@ -87,6 +87,8 @@ type healingTracker struct {
 	// ID of the current healing operation
 	HealID string
 
+	ItemsSkipped uint64
+	BytesSkipped uint64
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -175,14 +177,18 @@ func (h *healingTracker) setObject(object string) {
 	h.Object = object
 }
 
-func (h *healingTracker) updateProgress(success bool, bytes uint64) {
+func (h *healingTracker) updateProgress(success, skipped bool, bytes uint64) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
-	if success {
+	switch {
+	case success:
 		h.ItemsHealed++
 		h.BytesDone += bytes
-	} else {
+	case skipped:
+		h.ItemsSkipped++
+		h.BytesSkipped += bytes
+	default:
 		h.ItemsFailed++
 		h.BytesFailed += bytes
 	}
@@ -323,8 +329,10 @@ func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 		ObjectsTotalCount: h.ObjectsTotalCount,
 		ObjectsTotalSize:  h.ObjectsTotalSize,
 		ItemsHealed:       h.ItemsHealed,
+		ItemsSkipped:      h.ItemsSkipped,
 		ItemsFailed:       h.ItemsFailed,
 		BytesDone:         h.BytesDone,
+		BytesSkipped:      h.BytesSkipped,
 		BytesFailed:       h.BytesFailed,
 		Bucket:            h.Bucket,
 		Object:            h.Object,
@@ -376,25 +384,10 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
 func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
-	disk, format, _, err := connectEndpoint(endpoint)
-	if err != nil {
-		return fmt.Errorf("Error: %w, %s", err, endpoint)
-	}
-	defer disk.Close()
-	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
-	if poolIdx < 0 {
-		return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
-	}
-
-	// Calculate the set index where the current endpoint belongs
-	z.serverPools[poolIdx].erasureDisksMu.RLock()
-	setIdx, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
-	z.serverPools[poolIdx].erasureDisksMu.RUnlock()
-	if err != nil {
-		return err
-	}
-	if setIdx < 0 {
-		return fmt.Errorf("unexpected set index (%d) found for  %s", setIdx, disk.Endpoint())
+	poolIdx, setIdx := endpoint.PoolIdx, endpoint.SetIdx
+	disk := getStorageViaEndpoint(endpoint)
+	if disk == nil {
+		return fmt.Errorf("Unexpected error disk must be initialized by now after formatting: %s", endpoint)
 	}
 
 	// Prevent parallel erasure set healing
@@ -459,11 +452,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 
-	if tracker.ItemsFailed > 0 {
-		logger.Event(ctx, "Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
-	} else {
-		logger.Event(ctx, "Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
-	}
+	logger.Event(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 
 	if len(tracker.QueuedBuckets) > 0 {
 		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)

cmd/background-newdisks-heal-ops_gen.go

@@ -188,6 +188,18 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "HealID")
 				return
 			}
+		case "ItemsSkipped":
+			z.ItemsSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ItemsSkipped")
+				return
+			}
+		case "BytesSkipped":
+			z.BytesSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "BytesSkipped")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -201,9 +213,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 23
+	// map header, size 25
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x19, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -446,15 +458,35 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "HealID")
 		return
 	}
+	// write "ItemsSkipped"
+	err = en.Append(0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ItemsSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ItemsSkipped")
+		return
+	}
+	// write "BytesSkipped"
+	err = en.Append(0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.BytesSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "BytesSkipped")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 23
+	// map header, size 25
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x19, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -528,6 +560,12 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "HealID"
 	o = append(o, 0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
 	o = msgp.AppendString(o, z.HealID)
+	// string "ItemsSkipped"
+	o = append(o, 0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ItemsSkipped)
+	// string "BytesSkipped"
+	o = append(o, 0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.BytesSkipped)
 	return
 }
 
@@ -713,6 +751,18 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "HealID")
 				return
 			}
+		case "ItemsSkipped":
+			z.ItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ItemsSkipped")
+				return
+			}
+		case "BytesSkipped":
+			z.BytesSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BytesSkipped")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -735,6 +785,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
-	s += 7 + msgp.StringPrefixSize + len(z.HealID)
+	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size
 	return
 }

cmd/batch-expire.go

@@ -427,7 +427,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 					default:
 					}
 					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-					_, err := api.DeleteObject(ctx, exp.Bucket, exp.Name, ObjectOptions{
+					_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
 						DeletePrefix: true,
 					})
 					if err != nil {

cmd/batch-handlers.go

@@ -63,7 +63,6 @@ type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
 	User      string               `yaml:"-" json:"user"`
 	Started   time.Time            `yaml:"-" json:"started"`
-	Location  string               `yaml:"-" json:"location"`
 	Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
 	KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
 	Expire    *BatchJobExpire      `yaml:"expire" json:"expire"`
@@ -278,7 +277,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	isTags := len(r.Flags.Filter.Tags) != 0
+	hasTags := len(r.Flags.Filter.Tags) != 0
 	isMetadata := len(r.Flags.Filter.Metadata) != 0
 	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
 
@@ -303,7 +302,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return true
 		}
 
-		if isTags {
+		if hasTags {
 			// Only parse object tags if tags filter is specified.
 			tagMap := map[string]string{}
 			tagStr := oi.UserTags
@@ -408,7 +407,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 						continue
 					}
 				}
-				if isTags {
+				if hasTags {
 					tags, err := c.GetObjectTagging(ctx, r.Source.Bucket, obj.Key, minio.GetObjectTaggingOptions{})
 					if err == nil {
 						oi.UserTags = tags.String()
@@ -710,38 +709,52 @@ type batchJobInfo struct {
 }
 
 const (
-	batchReplName      = "batch-replicate.bin"
-	batchReplFormat    = 1
-	batchReplVersionV1 = 1
-	batchReplVersion   = batchReplVersionV1
-	batchJobName       = "job.bin"
-	batchJobPrefix     = "batch-jobs"
+	batchReplName         = "batch-replicate.bin"
+	batchReplFormat       = 1
+	batchReplVersionV1    = 1
+	batchReplVersion      = batchReplVersionV1
+	batchJobName          = "job.bin"
+	batchJobPrefix        = "batch-jobs"
+	batchJobReportsPrefix = batchJobPrefix + "/reports"
 
 	batchReplJobAPIVersion        = "v1"
 	batchReplJobDefaultRetries    = 3
 	batchReplJobDefaultRetryDelay = 250 * time.Millisecond
 )
 
-func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+func getJobReportPath(job BatchJobRequest) string {
 	var fileName string
-	var format, version uint16
 	switch {
 	case job.Replicate != nil:
 		fileName = batchReplName
+	case job.KeyRotate != nil:
+		fileName = batchKeyRotationName
+	case job.Expire != nil:
+		fileName = batchExpireName
+	}
+	return pathJoin(batchJobReportsPrefix, job.ID, fileName)
+}
+
+func getJobPath(job BatchJobRequest) string {
+	return pathJoin(batchJobPrefix, job.ID)
+}
+
+func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	var format, version uint16
+	switch {
+	case job.Replicate != nil:
 		version = batchReplVersionV1
 		format = batchReplFormat
 	case job.KeyRotate != nil:
-		fileName = batchKeyRotationName
 		version = batchKeyRotateVersionV1
 		format = batchKeyRotationFormat
 	case job.Expire != nil:
-		fileName = batchExpireName
 		version = batchExpireVersionV1
 		format = batchExpireFormat
 	default:
 		return errors.New("no supported batch job request specified")
 	}
-	data, err := readConfig(ctx, api, pathJoin(job.Location, fileName))
+	data, err := readConfig(ctx, api, getJobReportPath(job))
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
 			ri.Version = int(version)
@@ -852,8 +865,8 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 	now := UTCNow()
 	ri.mu.Lock()
 	var (
-		format, version  uint16
-		jobTyp, fileName string
+		format, version uint16
+		jobTyp          string
 	)
 
 	if now.Sub(ri.LastUpdate) >= duration {
@@ -862,19 +875,16 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 			format = batchReplFormat
 			version = batchReplVersion
 			jobTyp = string(job.Type())
-			fileName = batchReplName
 			ri.Version = batchReplVersionV1
 		case madmin.BatchJobKeyRotate:
 			format = batchKeyRotationFormat
 			version = batchKeyRotateVersion
 			jobTyp = string(job.Type())
-			fileName = batchKeyRotationName
 			ri.Version = batchKeyRotateVersionV1
 		case madmin.BatchJobExpire:
 			format = batchExpireFormat
 			version = batchExpireVersion
 			jobTyp = string(job.Type())
-			fileName = batchExpireName
 			ri.Version = batchExpireVersionV1
 		default:
 			return errInvalidArgument
@@ -895,7 +905,7 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 		if err != nil {
 			return err
 		}
-		return saveConfig(ctx, api, pathJoin(job.Location, fileName), buf)
+		return saveConfig(ctx, api, getJobReportPath(job), buf)
 	}
 	ri.mu.Unlock()
 	return nil
@@ -1054,8 +1064,6 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
 		go func() {
-			defer xioutil.SafeClose(slowCh)
-
 			// Snowball currently needs the high level minio-go Client, not the Core one
 			cl, err := miniogo.New(u.Host, &miniogo.Options{
 				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
@@ -1071,31 +1079,8 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			// Already validated before arriving here
 			smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
 
-			var (
-				obj   = ObjectInfo{}
-				batch = make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
-				valid = true
-			)
-
-			for valid {
-				obj, valid = <-walkCh
-
-				if !valid {
-					goto write
-				}
-
-				if obj.DeleteMarker || !obj.VersionPurgeStatus.Empty() || obj.Size >= int64(smallerThan) {
-					slowCh <- obj
-					continue
-				}
-
-				batch = append(batch, obj)
-
-				if len(batch) < *r.Source.Snowball.Batch {
-					continue
-				}
-
-			write:
+			batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
+			writeFn := func(batch []ObjectInfo) {
 				if len(batch) > 0 {
 					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
 						logger.LogIf(ctx, err)
@@ -1108,9 +1093,24 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						// persist in-memory state to disk after every 10secs.
 						logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 					}
-					batch = batch[:0]
 				}
 			}
+			for obj := range walkCh {
+				if obj.DeleteMarker || !obj.VersionPurgeStatus.Empty() || obj.Size >= int64(smallerThan) {
+					slowCh <- obj
+					continue
+				}
+
+				batch = append(batch, obj)
+
+				if len(batch) < *r.Source.Snowball.Batch {
+					continue
+				}
+				writeFn(batch)
+				batch = batch[:0]
+			}
+			writeFn(batch)
+			xioutil.SafeClose(slowCh)
 		}()
 	} else {
 		slowCh = walkCh
@@ -1417,15 +1417,8 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 }
 
 func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
-	switch {
-	case j.Replicate != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchReplName))
-	case j.KeyRotate != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
-	case j.Expire != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchExpireName))
-	}
-	deleteConfig(ctx, api, j.Location)
+	deleteConfig(ctx, api, getJobReportPath(j))
+	deleteConfig(ctx, api, getJobPath(j))
 }
 
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
@@ -1437,13 +1430,12 @@ func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
 		return err
 	}
 
-	j.Location = pathJoin(batchJobPrefix, j.ID)
 	job, err := j.MarshalMsg(nil)
 	if err != nil {
 		return err
 	}
 
-	return saveConfig(ctx, api, j.Location, job)
+	return saveConfig(ctx, api, getJobPath(*j), job)
 }
 
 func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string) error {
@@ -1671,8 +1663,7 @@ func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request)
 	}
 
 	j := BatchJobRequest{
-		ID:       jobID,
-		Location: pathJoin(batchJobPrefix, jobID),
+		ID: jobID,
 	}
 
 	j.delete(ctx, objectAPI)

cmd/batch-handlers_gen.go

@@ -42,12 +42,6 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if dc.IsNil() {
 				err = dc.ReadNil()
@@ -115,9 +109,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 7
+	// map header, size 6
 	// write "ID"
-	err = en.Append(0x87, 0xa2, 0x49, 0x44)
+	err = en.Append(0x86, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -146,16 +140,6 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Started")
 		return
 	}
-	// write "Location"
-	err = en.Append(0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Location)
-	if err != nil {
-		err = msgp.WrapError(err, "Location")
-		return
-	}
 	// write "Replicate"
 	err = en.Append(0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if err != nil {
@@ -213,9 +197,9 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 7
+	// map header, size 6
 	// string "ID"
-	o = append(o, 0x87, 0xa2, 0x49, 0x44)
+	o = append(o, 0x86, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -223,9 +207,6 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Started"
 	o = append(o, 0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
 	o = msgp.AppendTime(o, z.Started)
-	// string "Location"
-	o = append(o, 0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	o = msgp.AppendString(o, z.Location)
 	// string "Replicate"
 	o = append(o, 0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if z.Replicate == nil {
@@ -298,12 +279,6 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
@@ -369,7 +344,7 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobRequest) Msgsize() (s int) {
-	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 9 + msgp.StringPrefixSize + len(z.Location) + 10
+	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 10
 	if z.Replicate == nil {
 		s += msgp.NilSize
 	} else {

cmd/batch-rotate.go

@@ -274,7 +274,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	skip := func(info FileInfo) (ok bool) {
+	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
@@ -360,7 +360,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	results := make(chan ObjectInfo, 100)
 	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
 		Marker: lastObject,
-		Filter: skip,
+		Filter: selectObj,
 	}); err != nil {
 		cancel()
 		// Do not need to retry if we can't list objects on source.

cmd/bucket-encryption-handlers.go

@@ -25,7 +25,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"

cmd/bucket-lifecycle.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -39,12 +38,9 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
-	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
-	"github.com/minio/pkg/v2/env"
 	xnet "github.com/minio/pkg/v2/net"
-	"github.com/minio/pkg/v2/workers"
 	"github.com/zeebo/xxh3"
 )
 
@@ -105,96 +101,284 @@ type expiryTask struct {
 	src     lcEventSrc
 }
 
+// expiryStats records metrics related to ILM expiry activities
+type expiryStats struct {
+	missedExpiryTasks      atomic.Int64
+	missedFreeVersTasks    atomic.Int64
+	missedTierJournalTasks atomic.Int64
+	workers                atomic.Int32
+}
+
+// MissedTasks returns the number of ILM expiry tasks that were missed since
+// there were no available workers.
+func (e *expiryStats) MissedTasks() int64 {
+	return e.missedExpiryTasks.Load()
+}
+
+// MissedFreeVersTasks returns the number of free version collection tasks that
+// were missed since there were no available workers.
+func (e *expiryStats) MissedFreeVersTasks() int64 {
+	return e.missedFreeVersTasks.Load()
+}
+
+// MissedTierJournalTasks returns the number of tasks to remove tiered objects
+// that were missed since there were no available workers.
+func (e *expiryStats) MissedTierJournalTasks() int64 {
+	return e.missedTierJournalTasks.Load()
+}
+
+// NumWorkers returns the number of active workers executing one of ILM expiry
+// tasks or free version collection tasks.
+func (e *expiryStats) NumWorkers() int32 {
+	return e.workers.Load()
+}
+
+type expiryOp interface {
+	OpHash() uint64
+}
+
+type freeVersionTask struct {
+	ObjectInfo
+}
+
+func (f freeVersionTask) OpHash() uint64 {
+	return xxh3.HashString(f.TransitionedObject.Tier + f.TransitionedObject.Name)
+}
+
+func (n newerNoncurrentTask) OpHash() uint64 {
+	return xxh3.HashString(n.bucket + n.versions[0].ObjectV.ObjectName)
+}
+
+func (j jentry) OpHash() uint64 {
+	return xxh3.HashString(j.TierName + j.ObjName)
+}
+
+func (e expiryTask) OpHash() uint64 {
+	return xxh3.HashString(e.objInfo.Bucket + e.objInfo.Name)
+}
+
+// expiryState manages all ILM related expiration activities.
 type expiryState struct {
-	once                sync.Once
-	byDaysCh            chan expiryTask
-	byNewerNoncurrentCh chan newerNoncurrentTask
+	mu      sync.RWMutex
+	workers atomic.Pointer[[]chan expiryOp]
+
+	ctx    context.Context
+	objAPI ObjectLayer
+
+	stats expiryStats
 }
 
 // PendingTasks returns the number of pending ILM expiry tasks.
 func (es *expiryState) PendingTasks() int {
-	return len(es.byDaysCh) + len(es.byNewerNoncurrentCh)
+	w := es.workers.Load()
+	if w == nil || len(*w) == 0 {
+		return 0
+	}
+	var tasks int
+	for _, wrkr := range *w {
+		tasks += len(wrkr)
+	}
+	return tasks
 }
 
-// close closes work channels exactly once.
-func (es *expiryState) close() {
-	es.once.Do(func() {
-		xioutil.SafeClose(es.byDaysCh)
-		xioutil.SafeClose(es.byNewerNoncurrentCh)
-	})
+// enqueueTierJournalEntry enqueues a tier journal entry referring to a remote
+// object corresponding to a 'replaced' object versions. This applies only to
+// non-versioned or version suspended buckets.
+func (es *expiryState) enqueueTierJournalEntry(je jentry) {
+	wrkr := es.getWorkerCh(je.OpHash())
+	if wrkr == nil {
+		es.stats.missedTierJournalTasks.Add(1)
+		return
+	}
+	select {
+	case <-GlobalContext.Done():
+	case wrkr <- je:
+	default:
+		es.stats.missedTierJournalTasks.Add(1)
+	}
+}
+
+// enqueueFreeVersion enqueues a free version to be deleted
+func (es *expiryState) enqueueFreeVersion(oi ObjectInfo) {
+	task := freeVersionTask{ObjectInfo: oi}
+	wrkr := es.getWorkerCh(task.OpHash())
+	if wrkr == nil {
+		es.stats.missedFreeVersTasks.Add(1)
+		return
+	}
+	select {
+	case <-GlobalContext.Done():
+	case wrkr <- task:
+	default:
+		es.stats.missedFreeVersTasks.Add(1)
+	}
 }
 
 // enqueueByDays enqueues object versions expired by days for expiry.
 func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
+	task := expiryTask{objInfo: oi, event: event, src: src}
+	wrkr := es.getWorkerCh(task.OpHash())
+	if wrkr == nil {
+		es.stats.missedExpiryTasks.Add(1)
+		return
+	}
 	select {
 	case <-GlobalContext.Done():
-		es.close()
-	case es.byDaysCh <- expiryTask{objInfo: oi, event: event, src: src}:
+	case wrkr <- task:
 	default:
+		es.stats.missedExpiryTasks.Add(1)
 	}
 }
 
 // enqueueByNewerNoncurrent enqueues object versions expired by
 // NewerNoncurrentVersions limit for expiry.
 func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete, lcEvent lifecycle.Event) {
+	if len(versions) == 0 {
+		return
+	}
+
+	task := newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}
+	wrkr := es.getWorkerCh(task.OpHash())
+	if wrkr == nil {
+		es.stats.missedExpiryTasks.Add(1)
+		return
+	}
 	select {
 	case <-GlobalContext.Done():
-		es.close()
-	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}:
+	case wrkr <- task:
 	default:
+		es.stats.missedExpiryTasks.Add(1)
 	}
 }
 
+// globalExpiryState is the per-node instance which manages all ILM expiry tasks.
 var globalExpiryState *expiryState
 
-func newExpiryState() *expiryState {
-	return &expiryState{
-		byDaysCh:            make(chan expiryTask, 100000),
-		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 100000),
+// newExpiryState creates an expiryState with buffered channels allocated for
+// each ILM expiry task type.
+func newExpiryState(ctx context.Context, objAPI ObjectLayer, n int) *expiryState {
+	es := &expiryState{
+		ctx:    ctx,
+		objAPI: objAPI,
+	}
+	workers := make([]chan expiryOp, 0, n)
+	es.workers.Store(&workers)
+	es.ResizeWorkers(n)
+	return es
+}
+
+func (es *expiryState) getWorkerCh(h uint64) chan<- expiryOp {
+	w := es.workers.Load()
+	if w == nil || len(*w) == 0 {
+		return nil
+	}
+	workers := *w
+	return workers[h%uint64(len(workers))]
+}
+
+func (es *expiryState) ResizeWorkers(n int) {
+	// Lock to avoid multiple resizes to happen at the same time.
+	es.mu.Lock()
+	defer es.mu.Unlock()
+	var workers []chan expiryOp
+	if v := es.workers.Load(); v != nil {
+		// Copy to new array.
+		workers = append(workers, *v...)
+	}
+
+	if n == len(workers) || n < 1 {
+		return
+	}
+
+	for len(workers) < n {
+		input := make(chan expiryOp, 10000)
+		workers = append(workers, input)
+		go es.Worker(input)
+		es.stats.workers.Add(1)
+	}
+
+	for len(workers) > n {
+		worker := workers[len(workers)-1]
+		workers = workers[:len(workers)-1]
+		worker <- expiryOp(nil)
+		es.stats.workers.Add(-1)
+	}
+	// Atomically replace workers.
+	es.workers.Store(&workers)
+}
+
+// Worker handles 4 types of expiration tasks.
+// 1. Expiry of objects, includes regular and transitioned objects
+// 2. Expiry of noncurrent versions due to NewerNoncurrentVersions
+// 3. Expiry of free-versions, for remote objects of transitioned object which have been expired since.
+// 4. Expiry of remote objects corresponding to objects in a
+// non-versioned/version suspended buckets
+func (es *expiryState) Worker(input <-chan expiryOp) {
+	for {
+		select {
+		case <-es.ctx.Done():
+			return
+		case v, ok := <-input:
+			if !ok {
+				return
+			}
+			if v == nil {
+				// ResizeWorkers signaling worker to quit
+				return
+			}
+			switch v := v.(type) {
+			case expiryTask:
+				if v.objInfo.TransitionedObject.Status != "" {
+					applyExpiryOnTransitionedObject(es.ctx, es.objAPI, v.objInfo, v.event, v.src)
+				} else {
+					applyExpiryOnNonTransitionedObjects(es.ctx, es.objAPI, v.objInfo, v.event, v.src)
+				}
+			case newerNoncurrentTask:
+				deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.event)
+			case jentry:
+				logger.LogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+			case freeVersionTask:
+				oi := v.ObjectInfo
+				traceFn := globalLifecycleSys.trace(oi)
+				if !oi.TransitionedObject.FreeVersion {
+					// nothing to be done
+					return
+				}
+
+				ignoreNotFoundErr := func(err error) error {
+					switch {
+					case isErrVersionNotFound(err), isErrObjectNotFound(err):
+						return nil
+					}
+					return err
+				}
+				// Remove the remote object
+				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
+				if ignoreNotFoundErr(err) != nil {
+					logger.LogIf(es.ctx, err)
+					return
+				}
+
+				// Remove this free version
+				_, err = es.objAPI.DeleteObject(es.ctx, oi.Bucket, oi.Name, ObjectOptions{
+					VersionID:        oi.VersionID,
+					InclFreeVersions: true,
+				})
+				if err == nil {
+					auditLogLifecycle(es.ctx, oi, ILMFreeVersionDelete, nil, traceFn)
+				}
+				if ignoreNotFoundErr(err) != nil {
+					logger.LogIf(es.ctx, err)
+				}
+			default:
+				logger.LogIf(es.ctx, fmt.Errorf("Invalid work type - %v", v))
+			}
+		}
 	}
 }
 
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
-	globalExpiryState = newExpiryState()
-
-	workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
-	if workerSize == 0 {
-		workerSize = 4
-	}
-	ewk, err := workers.New(workerSize)
-	if err != nil {
-		logger.LogIf(ctx, err)
-	}
-
-	nwk, err := workers.New(workerSize)
-	if err != nil {
-		logger.LogIf(ctx, err)
-	}
-
-	go func() {
-		for t := range globalExpiryState.byDaysCh {
-			ewk.Take()
-			go func(t expiryTask) {
-				defer ewk.Give()
-				if t.objInfo.TransitionedObject.Status != "" {
-					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.event, t.src)
-				} else {
-					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.event, t.src)
-				}
-			}(t)
-		}
-		ewk.Wait()
-	}()
-	go func() {
-		for t := range globalExpiryState.byNewerNoncurrentCh {
-			nwk.Take()
-			go func(t newerNoncurrentTask) {
-				defer nwk.Give()
-				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions, t.event)
-			}(t)
-		}
-		nwk.Wait()
-	}()
+	globalExpiryState = newExpiryState(ctx, objectAPI, globalILMConfig.getExpirationWorkers())
 }
 
 // newerNoncurrentTask encapsulates arguments required by worker to expire objects
@@ -258,6 +442,10 @@ func newTransitionState(ctx context.Context) *transitionState {
 // of transition workers.
 func (t *transitionState) Init(objAPI ObjectLayer) {
 	n := globalAPIConfig.getTransitionWorkers()
+	// Prefer ilm.transition_workers over now deprecated api.transition_workers
+	if tw := globalILMConfig.getTransitionWorkers(); tw > 0 {
+		n = tw
+	}
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
@@ -418,18 +606,18 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		}
 		return err
 	}
-	// When an object is past expiry or when a transitioned object is being
-	// deleted, 'mark' the data in the remote tier for delete.
-	entry := jentry{
-		ObjName:   oi.TransitionedObject.Name,
-		VersionID: oi.TransitionedObject.VersionID,
-		TierName:  oi.TransitionedObject.Tier,
+
+	// Delete remote object from warm-tier
+	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
+	if err == nil {
+		// Skip adding free version since we successfully deleted the
+		// remote object
+		opts.SkipFreeVersion = true
+	} else {
+		logger.LogIf(ctx, err)
 	}
-	if err := globalTierJournal.AddEntry(entry); err != nil {
-		return err
-	}
-	// Delete metadata on source, now that data in remote tier has been
-	// marked for deletion.
+
+	// Now, delete object from hot-tier namespace
 	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
 		return err
 	}

cmd/bucket-quota.go

@@ -47,19 +47,15 @@ var bucketStorageCache = cachevalue.New[DataUsageInfo]()
 
 // Init initialize bucket quota.
 func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
-	bucketStorageCache.Once.Do(func() {
-		// Set this to 10 secs since its enough, as scanner
-		// does not update the bucket usage values frequently.
-		bucketStorageCache.TTL = 10 * time.Second
-		// Rely on older value if usage loading fails from disk.
-		bucketStorageCache.Relax = true
-		bucketStorageCache.Update = func() (DataUsageInfo, error) {
+	bucketStorageCache.InitOnce(10*time.Second,
+		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
+		func() (DataUsageInfo, error) {
 			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
 			defer done()
 
 			return loadDataUsageFromBackend(ctx, objAPI)
-		}
-	})
+		},
+	)
 }
 
 // GetBucketUsageInfo return bucket usage info for a given bucket

cmd/bucket-replication-metrics_gen.go

@@ -584,6 +584,7 @@ func (z *InQueueStats) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -595,6 +596,7 @@ func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) {
 func (z InQueueStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }

cmd/bucket-replication-utils_gen.go

@@ -749,6 +749,7 @@ func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -760,6 +761,7 @@ func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) {
 func (z ReplicateDecision) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }
@@ -1388,6 +1390,7 @@ func (z *ResyncDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -1399,6 +1402,7 @@ func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) {
 func (z ResyncDecision) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }

cmd/bucket-replication.go

@@ -1563,6 +1563,9 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		if err == nil {
 			break
 		}
+		if minio.ToErrorResponse(err).Code == "PreconditionFailed" {
+			return err
+		}
 		attempts++
 		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 	}

cmd/build-constants.go

@@ -17,6 +17,8 @@
 
 package cmd
 
+import "runtime"
+
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
@@ -40,4 +42,28 @@ var (
 
 	// CopyrightYear - dynamic value of the copyright end year
 	CopyrightYear = "0000"
+
+	// MinioReleaseTagTimeLayout - release tag time layout.
+	MinioReleaseTagTimeLayout = "2006-01-02T15-04-05Z"
+
+	// MinioOSARCH - OS and ARCH.
+	minioOSARCH = runtime.GOOS + "-" + runtime.GOARCH
+
+	// MinioReleaseBaseURL - release url without os and arch.
+	MinioReleaseBaseURL = "https://dl.min.io/server/minio/release/"
+
+	// MinioReleaseURL - release URL.
+	MinioReleaseURL = MinioReleaseBaseURL + minioOSARCH + SlashSeparator
+
+	// MinioStoreName - MinIO store name.
+	MinioStoreName = "MinIO"
+
+	// MinioUAName - MinIO user agent name.
+	MinioUAName = "MinIO"
+
+	// MinioBannerName - MinIO banner name for startup message.
+	MinioBannerName = "MinIO Object Storage Server"
+
+	// MinioLicense - MinIO server license.
+	MinioLicense = "GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"
 )

cmd/common-main.go

@@ -27,7 +27,6 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
-	"math/rand"
 	"net"
 	"net/url"
 	"os"
@@ -50,7 +49,7 @@ import (
 	"github.com/minio/console/api/operations"
 	consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2"
 	consoleCerts "github.com/minio/console/pkg/certs"
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -67,9 +66,11 @@ import (
 )
 
 // serverDebugLog will enable debug printing
-var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
-
-var currentReleaseTime time.Time
+var (
+	serverDebugLog     = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+	currentReleaseTime time.Time
+	orchestrated       = IsKubernetes() || IsDocker()
+)
 
 func init() {
 	if runtime.GOOS == "windows" {
@@ -83,13 +84,9 @@ func init() {
 		}
 	}
 
-	rand.Seed(time.Now().UTC().UnixNano())
-
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	initGlobalContext()
-
 	globalBatchJobsMetrics = batchJobMetrics{metrics: make(map[string]*batchJobInfo)}
 	go globalBatchJobsMetrics.purgeJobMetrics()
 
@@ -366,6 +363,14 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 		ctxt.ConsoleAddr = ctx.String("console-address")
 	}
 
+	if cxml := ctx.String("crossdomain-xml"); cxml != "" {
+		buf, err := os.ReadFile(cxml)
+		if err != nil {
+			return err
+		}
+		ctxt.CrossDomainXML = string(buf)
+	}
+
 	// Check "no-compat" flag from command line argument.
 	ctxt.StrictS3Compat = !(ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat"))
 
@@ -395,6 +400,7 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.ConnReadDeadline = ctx.Duration("conn-read-deadline")
 	ctxt.ConnWriteDeadline = ctx.Duration("conn-write-deadline")
 	ctxt.ConnClientReadDeadline = ctx.Duration("conn-client-read-deadline")
+	ctxt.ConnClientWriteDeadline = ctx.Duration("conn-client-write-deadline")
 
 	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
@@ -490,7 +496,11 @@ func runDNSCache(ctx *cli.Context) {
 	dnsTTL := ctx.Duration("dns-cache-ttl")
 	// Check if we have configured a custom DNS cache TTL.
 	if dnsTTL <= 0 {
-		dnsTTL = 10 * time.Minute
+		if orchestrated {
+			dnsTTL = 30 * time.Second
+		} else {
+			dnsTTL = 10 * time.Minute
+		}
 	}
 
 	// Call to refresh will refresh names in cache.
@@ -896,7 +906,6 @@ func handleKMSConfig() {
 			}
 			kmsConf = kms.Config{
 				Endpoints:    endpoints,
-				Enclave:      env.Get(kms.EnvKESEnclave, ""),
 				DefaultKeyID: env.Get(kms.EnvKESKeyName, ""),
 				APIKey:       key,
 				RootCAs:      rootCAs,
@@ -942,7 +951,6 @@ func handleKMSConfig() {
 
 			kmsConf = kms.Config{
 				Endpoints:        endpoints,
-				Enclave:          env.Get(kms.EnvKESEnclave, ""),
 				DefaultKeyID:     env.Get(kms.EnvKESKeyName, ""),
 				Certificate:      certificate,
 				ReloadCertEvents: reloadCertEvents,

cmd/config-current.go

@@ -18,13 +18,17 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"strings"
 	"sync"
 
+	"github.com/minio/kms-go/kes"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/browser"
+	"github.com/minio/minio/internal/kms"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
@@ -41,6 +45,7 @@ import (
 	"github.com/minio/minio/internal/config/identity/openid"
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
+	"github.com/minio/minio/internal/config/ilm"
 	"github.com/minio/minio/internal/config/lambda"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
@@ -74,6 +79,7 @@ func initHelp() {
 		config.SubnetSubSys:         subnet.DefaultKVS,
 		config.CallhomeSubSys:       callhome.DefaultKVS,
 		config.DriveSubSys:          drive.DefaultKVS,
+		config.ILMSubSys:            ilm.DefaultKVS,
 		config.CacheSubSys:          cache.DefaultKVS,
 		config.BatchSubSys:          batch.DefaultKVS,
 		config.BrowserSubSys:        browser.DefaultKVS,
@@ -569,6 +575,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		}
 
 		globalAPIConfig.init(apiConfig, setDriveCounts)
+		autoGenerateRootCredentials() // Generate the KMS root credentials here since we don't know whether API root access is disabled until now.
 
 		// Initialize remote instance transport once.
 		getRemoteInstanceTransportOnce.Do(func() {
@@ -613,13 +620,13 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.HTTP {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceConsoleIf
+				l.LogOnceIf = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
 			loggerCfg.HTTP[n] = l
 		}
-		if errs := logger.UpdateSystemTargets(ctx, loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateHTTPWebhooks(ctx, loggerCfg.HTTP); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
@@ -630,14 +637,14 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.AuditWebhook {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceConsoleIf
+				l.LogOnceIf = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
 			loggerCfg.AuditWebhook[n] = l
 		}
 
-		if errs := logger.UpdateAuditWebhookTargets(ctx, loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateAuditWebhooks(ctx, loggerCfg.AuditWebhook); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
@@ -711,6 +718,18 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			return fmt.Errorf("Unable to apply browser config: %w", err)
 		}
 		globalBrowserConfig.Update(browserCfg)
+	case config.ILMSubSys:
+		ilmCfg, err := ilm.LookupConfig(s[config.ILMSubSys][config.Default])
+		if err != nil {
+			return fmt.Errorf("Unable to apply ilm config: %w", err)
+		}
+		if globalTransitionState != nil {
+			globalTransitionState.UpdateWorkers(ilmCfg.TransitionWorkers)
+		}
+		if globalExpiryState != nil {
+			globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
+		}
+		globalILMConfig.update(ilmCfg)
 	}
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()
@@ -720,6 +739,55 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 	return nil
 }
 
+// autoGenerateRootCredentials generates root credentials deterministically if
+// a KMS is configured, no manual credentials have been specified and if root
+// access is disabled.
+func autoGenerateRootCredentials() {
+	if GlobalKMS == nil {
+		return
+	}
+	if globalAPIConfig.permitRootAccess() || !globalActiveCred.Equal(auth.DefaultCredentials) {
+		return
+	}
+
+	if manager, ok := GlobalKMS.(kms.KeyManager); ok {
+		stat, err := GlobalKMS.Stat(GlobalContext)
+		if err != nil {
+			logger.LogIf(GlobalContext, err, "Unable to generate root credentials using KMS")
+			return
+		}
+
+		aKey, err := manager.HMAC(GlobalContext, stat.DefaultKey, []byte("root access key"))
+		if errors.Is(err, kes.ErrNotAllowed) {
+			return // If we don't have permission to compute the HMAC, don't change the cred.
+		}
+		if err != nil {
+			logger.Fatal(err, "Unable to generate root access key using KMS")
+		}
+
+		sKey, err := manager.HMAC(GlobalContext, stat.DefaultKey, []byte("root secret key"))
+		if err != nil {
+			// Here, we must have permission. Otherwise, we would have failed earlier.
+			logger.Fatal(err, "Unable to generate root secret key using KMS")
+		}
+
+		accessKey, err := auth.GenerateAccessKey(20, bytes.NewReader(aKey))
+		if err != nil {
+			logger.Fatal(err, "Unable to generate root access key")
+		}
+		secretKey, err := auth.GenerateSecretKey(32, bytes.NewReader(sKey))
+		if err != nil {
+			logger.Fatal(err, "Unable to generate root secret key")
+		}
+
+		logger.Info("Automatically generated root access key and secret key with the KMS")
+		globalActiveCred = auth.Credentials{
+			AccessKey: accessKey,
+			SecretKey: secretKey,
+		}
+	}
+}
+
 // applyDynamicConfig will apply dynamic config values.
 // Dynamic systems should be in config.SubSystemsDynamic as well.
 func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {

cmd/crossdomain-xml-handler.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,10 +32,14 @@ const crossDomainXMLEntity = "/crossdomain.xml"
 // policy file that grants access to the source domain, allowing the client to continue the transaction.
 func setCrossDomainPolicyMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		cxml := crossDomainXML
+		if globalServerCtxt.CrossDomainXML != "" {
+			cxml = globalServerCtxt.CrossDomainXML
+		}
 		// Look for 'crossdomain.xml' in the incoming request.
 		if r.URL.Path == crossDomainXMLEntity {
 			// Write the standard cross domain policy xml.
-			w.Write([]byte(crossDomainXML))
+			w.Write([]byte(cxml))
 			// Request completed, no need to serve to other handlers.
 			return
 		}

cmd/data-scanner.go

@@ -982,45 +982,9 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 	return lcEvt.Action, size
 }
 
-// applyTierObjSweep removes remote object pending deletion and the free-version
-// tracking this information.
-func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
-	traceFn := globalLifecycleSys.trace(oi)
-	if !oi.TransitionedObject.FreeVersion {
-		// nothing to be done
-		return
-	}
-
-	ignoreNotFoundErr := func(err error) error {
-		switch {
-		case isErrVersionNotFound(err), isErrObjectNotFound(err):
-			return nil
-		}
-		return err
-	}
-	// Remove the remote object
-	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
-	if ignoreNotFoundErr(err) != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
-
-	// Remove this free version
-	_, err = o.DeleteObject(ctx, oi.Bucket, oi.Name, ObjectOptions{
-		VersionID:        oi.VersionID,
-		InclFreeVersions: true,
-	})
-	if err == nil {
-		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, nil, traceFn)
-	}
-	if ignoreNotFoundErr(err) != nil {
-		logger.LogIf(ctx, err)
-	}
-}
-
 // applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo, expState *expiryState) ([]ObjectInfo, error) {
 	done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
 	defer done()
 
@@ -1087,14 +1051,16 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		})
 	}
 
-	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
+	if len(toDel) > 0 {
+		expState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
+	}
 	return objectInfos, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
-func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
-	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo, expState *expiryState) ([]ObjectInfo, error) {
+	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs, expState)
 	if err != nil {
 		return nil, err
 	}
@@ -1274,7 +1240,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		}
 	}()
 
-	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
+	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
@@ -1295,7 +1261,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	if obj.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
 	}
-
+	if lcEvent.Action.DeleteAll() {
+		eventName = event.ObjectRemovedDeleteAllVersions
+	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,

cmd/data-scanner_test.go

@@ -39,14 +39,20 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 	globalBucketMetadataSys = NewBucketMetadataSys()
 	globalBucketObjectLockSys = &BucketObjectLockSys{}
 	globalBucketVersioningSys = &BucketVersioningSys{}
-	globalExpiryState = newExpiryState()
+	es := newExpiryState(context.Background(), objAPI, 0)
+	workers := []chan expiryOp{make(chan expiryOp)}
+	es.workers.Store(&workers)
+	globalExpiryState = es
 	var wg sync.WaitGroup
 	wg.Add(1)
 	expired := make([]ObjectToDelete, 0, 5)
 	go func() {
 		defer wg.Done()
-		for t := range globalExpiryState.byNewerNoncurrentCh {
-			expired = append(expired, t.versions...)
+		workers := globalExpiryState.workers.Load()
+		for t := range (*workers)[0] {
+			if t, ok := t.(newerNoncurrentTask); ok {
+				expired = append(expired, t.versions...)
+			}
 		}
 	}()
 	lc := lifecycle.Lifecycle{
@@ -116,7 +122,7 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 	for i, fi := range fivs[:2] {
 		wants[i] = fi.ToObjectInfo(bucket, obj, versioned)
 	}
-	gots, err := item.applyNewerNoncurrentVersionLimit(context.TODO(), objAPI, fivs)
+	gots, err := item.applyNewerNoncurrentVersionLimit(context.TODO(), objAPI, fivs, es)
 	if err != nil {
 		t.Fatalf("Failed with err: %v", err)
 	}
@@ -125,7 +131,7 @@ func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
 	}
 
 	// Close expiry state's channel to inspect object versions enqueued for expiration
-	close(globalExpiryState.byNewerNoncurrentCh)
+	close(workers[0])
 	wg.Wait()
 	for _, obj := range expired {
 		switch obj.ObjectV.VersionID {

cmd/data-usage-cache_gen.go

@@ -1762,7 +1762,7 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(10)
 	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
@@ -1866,7 +1866,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not omitted
 		// write "rs"
 		err = en.Append(0xa2, 0x72, 0x73)
 		if err != nil {
@@ -1885,7 +1885,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x100) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not omitted
 		// write "ats"
 		err = en.Append(0xa3, 0x61, 0x74, 0x73)
 		if err != nil {
@@ -1920,7 +1920,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(10)
 	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
@@ -1968,7 +1968,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0002 := range z.ObjVersions {
 		o = msgp.AppendUint64(o, z.ObjVersions[za0002])
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not omitted
 		// string "rs"
 		o = append(o, 0xa2, 0x72, 0x73)
 		if z.ReplicationStats == nil {
@@ -1981,7 +1981,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x100) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not omitted
 		// string "ats"
 		o = append(o, 0xa3, 0x61, 0x74, 0x73)
 		if z.AllTierStats == nil {
@@ -3243,7 +3243,7 @@ func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(3)
 	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
@@ -3267,7 +3267,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if zb0001Len == 0 {
 		return
 	}
-	if (zb0001Mask & 0x1) == 0 { // if not empty
+	if (zb0001Mask & 0x1) == 0 { // if not omitted
 		// write "t"
 		err = en.Append(0xa1, 0x74)
 		if err != nil {
@@ -3291,7 +3291,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x2) == 0 { // if not empty
+	if (zb0001Mask & 0x2) == 0 { // if not omitted
 		// write "r"
 		err = en.Append(0xa1, 0x72)
 		if err != nil {
@@ -3303,7 +3303,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x4) == 0 { // if not empty
+	if (zb0001Mask & 0x4) == 0 { // if not omitted
 		// write "rc"
 		err = en.Append(0xa2, 0x72, 0x63)
 		if err != nil {
@@ -3321,7 +3321,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(3)
 	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
@@ -3342,7 +3342,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	if zb0001Len == 0 {
 		return
 	}
-	if (zb0001Mask & 0x1) == 0 { // if not empty
+	if (zb0001Mask & 0x1) == 0 { // if not omitted
 		// string "t"
 		o = append(o, 0xa1, 0x74)
 		o = msgp.AppendMapHeader(o, uint32(len(z.Targets)))
@@ -3355,12 +3355,12 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x2) == 0 { // if not empty
+	if (zb0001Mask & 0x2) == 0 { // if not omitted
 		// string "r"
 		o = append(o, 0xa1, 0x72)
 		o = msgp.AppendUint64(o, z.ReplicaSize)
 	}
-	if (zb0001Mask & 0x4) == 0 { // if not empty
+	if (zb0001Mask & 0x4) == 0 { // if not omitted
 		// string "rc"
 		o = append(o, 0xa2, 0x72, 0x63)
 		o = msgp.AppendUint64(o, z.ReplicaCount)
@@ -3478,6 +3478,8 @@ func (z *replicationAllStatsV1) DecodeMsg(dc *msgp.Reader) (err error) {
 			delete(z.Targets, key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0002 > 0 {
 		zb0002--
 		var za0001 string
@@ -3588,6 +3590,8 @@ func (z *replicationAllStatsV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			delete(z.Targets, key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0002 > 0 {
 		var za0001 string
 		var za0002 replicationStats

cmd/data-usage-utils.go

@@ -140,7 +140,7 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 	return infos
 }
 
-func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
+func (dui DataUsageInfo) tierMetrics() (metrics []MetricV2) {
 	if dui.TierStats == nil {
 		return nil
 	}
@@ -148,17 +148,17 @@ func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
 	//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
 	//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
 	for tier, st := range dui.TierStats.Tiers {
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedBytesMD(),
 			Value:          float64(st.TotalSize),
 			VariableLabels: map[string]string{"tier": tier},
 		})
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedObjectsMD(),
 			Value:          float64(st.NumObjects),
 			VariableLabels: map[string]string{"tier": tier},
 		})
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedVersionsMD(),
 			Value:          float64(st.NumVersions),
 			VariableLabels: map[string]string{"tier": tier},

cmd/data-usage.go

@@ -77,12 +77,10 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 
 	cache := dataUsageCache{}
 
-	prefixUsageCache.Once.Do(func() {
-		prefixUsageCache.TTL = 30 * time.Second
-
+	prefixUsageCache.InitOnce(30*time.Second,
 		// No need to fail upon Update() error, fallback to old value.
-		prefixUsageCache.Relax = true
-		prefixUsageCache.Update = func() (map[string]uint64, error) {
+		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
+		func() (map[string]uint64, error) {
 			m := make(map[string]uint64)
 			for _, pool := range z.serverPools {
 				for _, er := range pool.sets {
@@ -107,8 +105,8 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 				}
 			}
 			return m, nil
-		}
-	})
+		},
+	)
 
 	return prefixUsageCache.Get()
 }

cmd/encryption-v1.go

@@ -34,7 +34,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"

cmd/endpoint.go

@@ -75,6 +75,16 @@ type Endpoint struct {
 	PoolIdx, SetIdx, DiskIdx int
 }
 
+// Equal returns true if endpoint == ep
+func (endpoint Endpoint) Equal(ep Endpoint) bool {
+	if endpoint.IsLocal == ep.IsLocal && endpoint.PoolIdx == ep.PoolIdx && endpoint.SetIdx == ep.SetIdx && endpoint.DiskIdx == ep.DiskIdx {
+		if endpoint.Path == ep.Path && endpoint.Host == ep.Host {
+			return true
+		}
+	}
+	return false
+}
+
 func (endpoint Endpoint) String() string {
 	if endpoint.Host == "" {
 		return endpoint.Path
@@ -580,8 +590,6 @@ func hostResolveToLocalhost(endpoint Endpoint) bool {
 
 // UpdateIsLocal - resolves the host and discovers the local host.
 func (endpoints Endpoints) UpdateIsLocal() error {
-	orchestrated := IsDocker() || IsKubernetes()
-
 	var epsResolved int
 	var foundLocal bool
 	resolvedList := make([]bool, len(endpoints))
@@ -765,8 +773,6 @@ type PoolEndpointList []Endpoints
 
 // UpdateIsLocal - resolves all hosts and discovers which are local
 func (p PoolEndpointList) UpdateIsLocal() error {
-	orchestrated := IsDocker() || IsKubernetes()
-
 	var epsResolved int
 	var epCount int
 
@@ -1024,7 +1030,6 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 			}
 		}
 
-		orchestrated := IsKubernetes() || IsDocker()
 		reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
 		// If not orchestrated
 		// and not setup in reverse proxy

cmd/erasure-healing.go

@@ -144,7 +144,7 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
-func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo, doinline bool) bool {
+func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool {
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
@@ -157,10 +157,6 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 			// always check first.
 			return true
 		}
-		if doinline {
-			// convert small files to 'inline'
-			return true
-		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
@@ -308,7 +304,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		errs, latestMeta, bucket, object, scanMode)
 
 	var erasure Erasure
-	var recreate bool
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
 		// Initialize erasure coding
 		erasure, err = NewErasure(ctx, latestMeta.Erasure.DataBlocks,
@@ -316,15 +311,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		if err != nil {
 			return result, err
 		}
-
-		// Is only 'true' if the opts.Recreate is true and
-		// the object shardSize < smallFileThreshold do not
-		// set this to 'true' arbitrarily and must be only
-		// 'true' with caller ask.
-		recreate = (opts.Recreate &&
-			!latestMeta.InlineData() &&
-			len(latestMeta.Parts) == 1 &&
-			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
 	}
 
 	result.ObjectSize, err = latestMeta.ToObjectInfo(bucket, object, true).GetActualSize()
@@ -353,7 +339,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta, recreate) {
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
@@ -401,7 +387,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		return result, nil
 	}
 
-	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// Allow for dangling deletes, on versions that have DataDir missing etc.
 		// this would end up restoring the correct readable versions.
 		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrs, ObjectOptions{
@@ -499,7 +485,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 
 	var inlineBuffers []*bytes.Buffer
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		if latestMeta.InlineData() || recreate {
+		if latestMeta.InlineData() {
 			inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 		}
 
@@ -603,9 +589,8 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			return result, err
 		}
 
-		// - Remove any parts from healed disks after its been inlined.
 		// - Remove any remaining parts from outdated disks from before transition.
-		if recreate || partsMetadata[i].IsRemote() {
+		if partsMetadata[i].IsRemote() {
 			rmDataDir := partsMetadata[i].DataDir
 			disk.Delete(ctx, bucket, pathJoin(encodeDirObject(object), rmDataDir), DeleteOptions{
 				Immediate: true,
@@ -1047,10 +1032,9 @@ func healTrace(funcName healingMetric, startTime time.Time, bucket, object strin
 	}
 	if opts != nil {
 		tr.Custom = map[string]string{
-			"dry":      fmt.Sprint(opts.DryRun),
-			"remove":   fmt.Sprint(opts.Remove),
-			"recreate": fmt.Sprint(opts.Recreate),
-			"mode":     fmt.Sprint(opts.ScanMode),
+			"dry":    fmt.Sprint(opts.DryRun),
+			"remove": fmt.Sprint(opts.Remove),
+			"mode":   fmt.Sprint(opts.ScanMode),
 		}
 		if result != nil {
 			tr.Custom["version-id"] = result.VersionID

cmd/erasure-metadata.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -300,17 +301,23 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 				fmt.Fprintf(h, "%v", meta.Erasure.Distribution)
 			}
 
-			// ILM transition fields
-			fmt.Fprint(h, meta.TransitionStatus)
-			fmt.Fprint(h, meta.TransitionTier)
-			fmt.Fprint(h, meta.TransitionedObjName)
-			fmt.Fprint(h, meta.TransitionVersionID)
+			if meta.IsRemote() {
+				// ILM transition fields
+				fmt.Fprint(h, meta.TransitionStatus)
+				fmt.Fprint(h, meta.TransitionTier)
+				fmt.Fprint(h, meta.TransitionedObjName)
+				fmt.Fprint(h, meta.TransitionVersionID)
+			}
 
-			// Server-side replication fields
-			fmt.Fprintf(h, "%v", meta.MarkDeleted)
-			fmt.Fprint(h, meta.Metadata[string(meta.ReplicationState.ReplicaStatus)])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicationStatusInternal])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.VersionPurgeStatusInternal])
+			// If metadata says encrypted, ask for it in quorum.
+			if etyp, ok := crypto.IsEncrypted(meta.Metadata); ok {
+				fmt.Fprint(h, etyp)
+			}
+
+			// If compressed, look for compressed FileInfo only
+			if meta.IsCompressed() {
+				fmt.Fprint(h, meta.Metadata[ReservedMetadataPrefix+"compression"])
+			}
 
 			metaHashes[i] = hex.EncodeToString(h.Sum(nil))
 			h.Reset()
@@ -495,14 +502,6 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 		return -1, -1, errErasureReadQuorum
 	}
 
-	if parityBlocks == 0 {
-		// For delete markers do not use 'defaultParityCount' as it is not expected to be the case.
-		// Use maximum allowed read quorum instead, writeQuorum+1 is returned for compatibility sake
-		// but there are no callers that shall be using this.
-		readQuorum := len(partsMetaData) / 2
-		return readQuorum, readQuorum + 1, nil
-	}
-
 	dataBlocks := len(partsMetaData) - parityBlocks
 
 	writeQuorum := dataBlocks
@@ -518,6 +517,7 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 const (
 	tierFVID     = "tier-free-versionID"
 	tierFVMarker = "tier-free-marker"
+	tierSkipFVID = "tier-skip-fvid"
 )
 
 // SetTierFreeVersionID sets free-version's versionID. This method is used by
@@ -544,6 +544,23 @@ func (fi *FileInfo) SetTierFreeVersion() {
 	fi.Metadata[ReservedMetadataPrefixLower+tierFVMarker] = ""
 }
 
+// SetSkipTierFreeVersion indicates to skip adding a tier free version id.
+// Note: Used only when expiring tiered objects and the remote content has
+// already been scheduled for deletion
+func (fi *FileInfo) SetSkipTierFreeVersion() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	fi.Metadata[ReservedMetadataPrefixLower+tierSkipFVID] = ""
+}
+
+// SkipTierFreeVersion returns true if set, false otherwise.
+// See SetSkipTierVersion for its purpose.
+func (fi *FileInfo) SkipTierFreeVersion() bool {
+	_, ok := fi.Metadata[ReservedMetadataPrefixLower+tierSkipFVID]
+	return ok
+}
+
 // TierFreeVersion returns true if version is a free-version.
 func (fi *FileInfo) TierFreeVersion() bool {
 	_, ok := fi.Metadata[ReservedMetadataPrefixLower+tierFVMarker]

cmd/erasure-metadata_test.go

@@ -314,3 +314,11 @@ func TestTransitionInfoEquals(t *testing.T) {
 		t.Fatalf("Expected to be inequal: fi %v ofi %v", fi, ofi)
 	}
 }
+
+func TestSkipTierFreeVersion(t *testing.T) {
+	fi := newFileInfo("object", 8, 8)
+	fi.SetSkipTierFreeVersion()
+	if ok := fi.SkipTierFreeVersion(); !ok {
+		t.Fatal("Expected SkipTierFreeVersion to be set on FileInfo but wasn't")
+	}
+}

cmd/erasure-multipart.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
@@ -380,7 +381,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		rctx := lkctx.Context()
 		obj, err := er.getObjectInfo(rctx, bucket, object, opts)
 		lk.RUnlock(lkctx)
-		if err != nil && !isErrVersionNotFound(err) {
+		if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
 			return nil, err
 		}
 		if opts.CheckPrecondFn(obj) {
@@ -400,36 +401,33 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		parityDrives = er.defaultParityCount
 	}
 
-	// If we have offline disks upgrade the number of erasure codes for this object.
-	parityOrig := parityDrives
+	if globalStorageClass.AvailabilityOptimized() {
+		// If we have offline disks upgrade the number of erasure codes for this object.
+		parityOrig := parityDrives
 
-	var offlineDrives int
-	for _, disk := range onlineDisks {
-		if disk == nil {
-			parityDrives++
-			offlineDrives++
-			continue
+		var offlineDrives int
+		for _, disk := range onlineDisks {
+			if disk == nil || !disk.IsOnline() {
+				parityDrives++
+				offlineDrives++
+				continue
+			}
 		}
-		if !disk.IsOnline() {
-			parityDrives++
-			offlineDrives++
-			continue
+
+		if offlineDrives >= (len(onlineDisks)+1)/2 {
+			// if offline drives are more than 50% of the drives
+			// we have no quorum, we shouldn't proceed just
+			// fail at that point.
+			return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
-	}
 
-	if offlineDrives >= (len(onlineDisks)+1)/2 {
-		// if offline drives are more than 50% of the drives
-		// we have no quorum, we shouldn't proceed just
-		// fail at that point.
-		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
-	}
+		if parityDrives >= len(onlineDisks)/2 {
+			parityDrives = len(onlineDisks) / 2
+		}
 
-	if parityDrives >= len(onlineDisks)/2 {
-		parityDrives = len(onlineDisks) / 2
-	}
-
-	if parityOrig != parityDrives {
-		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+		if parityOrig != parityDrives {
+			userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+		}
 	}
 
 	dataDrives := len(onlineDisks) - parityDrives
@@ -461,6 +459,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
+	// if storageClass is standard no need to save it as part of metadata.
+	if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD {
+		delete(userDefined, xhttp.AmzStorageClass)
+	}
+
 	if opts.WantChecksum != nil && opts.WantChecksum.Type.IsSet() {
 		userDefined[hash.MinIOMultipartChecksum] = opts.WantChecksum.Type.String()
 	}
@@ -499,7 +502,9 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 //
 // Implements S3 compatible initiate multipart API.
 func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	return er.newMultipartUpload(ctx, bucket, object, opts)
 }
@@ -578,7 +583,9 @@ func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry
 //
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	data := r.Reader
 	// Validate input data size and it can never be less than zero.
@@ -777,7 +784,9 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 // - compressed
 // Does not contain currently uploaded parts by design.
 func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	result := MultipartInfo{
 		Bucket:   bucket,
@@ -813,7 +822,9 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
 	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
@@ -969,7 +980,9 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 //
 // Implements S3 compatible Complete multipart API.
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	// Hold write locks to verify uploaded parts, also disallows any
 	// parallel PutObjectPart() requests.
@@ -1336,7 +1349,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // would be removed from the system, rollback is not possible on this
 // operation.
 func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)

cmd/erasure-object.go

@@ -31,12 +31,14 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/klauspost/readahead"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
@@ -66,7 +68,9 @@ func countOnlineDisks(onlineDisks []StorageAPI) (online int) {
 // if source object and destination object are same we only
 // update metadata.
 func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, dstObject, &er)
+	if !dstOpts.NoAuditLog {
+		auditObjectErasureSet(ctx, dstObject, &er)
+	}
 
 	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
 	if !srcInfo.metadataOnly {
@@ -188,7 +192,9 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // GetObjectNInfo - returns object info and an object
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	var unlockOnDefer bool
 	nsUnlocker := func() {}
@@ -419,7 +425,9 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	if !opts.NoLock {
 		// Lock the object before reading.
@@ -1253,7 +1261,9 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry, scanMode m
 
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	data := r.Reader
 
@@ -1288,25 +1298,21 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	storageDisks := er.getDisks()
 
-	parityDrives := len(storageDisks) / 2
-	if !opts.MaxParity {
-		// Get parity and data drive count based on storage class metadata
-		parityDrives = globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
-		if parityDrives < 0 {
-			parityDrives = er.defaultParityCount
-		}
-
+	// Get parity and data drive count based on storage class metadata
+	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
+	if parityDrives < 0 {
+		parityDrives = er.defaultParityCount
+	}
+	if opts.MaxParity {
+		parityDrives = len(storageDisks) / 2
+	}
+	if !opts.MaxParity && globalStorageClass.AvailabilityOptimized() {
 		// If we have offline disks upgrade the number of erasure codes for this object.
 		parityOrig := parityDrives
 
 		var offlineDrives int
 		for _, disk := range storageDisks {
-			if disk == nil {
-				parityDrives++
-				offlineDrives++
-				continue
-			}
-			if !disk.IsOnline() {
+			if disk == nil || !disk.IsOnline() {
 				parityDrives++
 				offlineDrives++
 				continue
@@ -1391,20 +1397,25 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	defer er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
 
 	shardFileSize := erasure.ShardFileSize(data.Size())
+	inlineBlock := globalStorageClass.InlineBlock()
+	if inlineBlock <= 0 {
+		inlineBlock = 128 * humanize.KiByte
+	}
+
 	writers := make([]io.Writer, len(onlineDisks))
 	var inlineBuffers []*bytes.Buffer
 	if shardFileSize >= 0 {
-		if !opts.Versioned && shardFileSize < smallFileThreshold {
+		if !opts.Versioned && shardFileSize < inlineBlock {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-		} else if shardFileSize < smallFileThreshold/8 {
+		} else if shardFileSize < inlineBlock/8 {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 		}
 	} else {
 		// If compressed, use actual size to determine.
 		if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 {
-			if !opts.Versioned && sz < smallFileThreshold {
+			if !opts.Versioned && sz < inlineBlock {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-			} else if sz < smallFileThreshold/8 {
+			} else if sz < inlineBlock/8 {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 			}
 		}
@@ -1509,6 +1520,11 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
+	// if storageClass is standard no need to save it as part of metadata.
+	if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD {
+		delete(userDefined, xhttp.AmzStorageClass)
+	}
+
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
@@ -1605,8 +1621,10 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
 func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
-	for _, obj := range objects {
-		auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
+	if !opts.NoAuditLog {
+		for _, obj := range objects {
+			auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
+		}
 	}
 
 	errs := make([]error, len(objects))
@@ -1775,20 +1793,12 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string) error {
 	disks := er.getDisks()
 	g := errgroup.WithNErrs(len(disks))
-	dirPrefix := encodeDirObject(prefix)
 	for index := range disks {
 		index := index
 		g.Go(func() error {
 			if disks[index] == nil {
 				return nil
 			}
-			// Deletes
-			// - The prefix and its children
-			// - The prefix__XLDIR__
-			defer disks[index].Delete(ctx, bucket, dirPrefix, DeleteOptions{
-				Recursive: true,
-				Immediate: true,
-			})
 			return disks[index].Delete(ctx, bucket, prefix, DeleteOptions{
 				Recursive: true,
 				Immediate: true,
@@ -1807,12 +1817,11 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	if !opts.NoAuditLog {
+		auditObjectErasureSet(ctx, object, &er)
+	}
 
 	if opts.DeletePrefix {
-		if globalCacheConfig.Enabled() {
-			return ObjectInfo{}, toObjectErr(errMethodNotAllowed, bucket, object)
-		}
 		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
 	}
 
@@ -1969,6 +1978,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			ExpireRestored:   opts.Transition.ExpireRestored,
 		}
 		fi.SetTierFreeVersionID(fvID)
+		if opts.SkipFreeVersion {
+			fi.SetSkipTierFreeVersion()
+		}
 		if opts.VersionID != "" {
 			fi.VersionID = opts.VersionID
 		} else if opts.Versioned {
@@ -1998,6 +2010,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		ExpireRestored:   opts.Transition.ExpireRestored,
 	}
 	dfi.SetTierFreeVersionID(fvID)
+	if opts.SkipFreeVersion {
+		dfi.SetSkipTierFreeVersion()
+	}
 	if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}

cmd/erasure-server-pool-decom.go

@@ -26,7 +26,6 @@ import (
 	"math/rand"
 	"net/http"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
@@ -73,9 +72,6 @@ func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
 	if pd == nil {
 		return nil
 	}
-	if pd.StartTime.IsZero() {
-		return nil
-	}
 	return &PoolDecommissionInfo{
 		StartTime:               pd.StartTime,
 		StartSize:               pd.StartSize,
@@ -98,7 +94,7 @@ func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
 
 // bucketPop should be called when a bucket is done decommissioning.
 // Adds the bucket to the list of decommissioned buckets and updates resume numbers.
-func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
+func (pd *PoolDecommissionInfo) bucketPop(bucket string) bool {
 	pd.DecommissionedBuckets = append(pd.DecommissionedBuckets, bucket)
 	for i, b := range pd.QueuedBuckets {
 		if b == bucket {
@@ -110,9 +106,10 @@ func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
 				pd.Prefix = "" // empty this out for the next bucket
 				pd.Object = "" // empty this out for next object
 			}
-			return
+			return true
 		}
 	}
+	return false
 }
 
 func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
@@ -223,12 +220,12 @@ func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool {
 	return p.Pools[idx].Decommission.isBucketDecommissioned(bucket)
 }
 
-func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) {
+func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) bool {
 	if p.Pools[idx].Decommission == nil {
 		// Decommission not in progress.
-		return
+		return false
 	}
-	p.Pools[idx].Decommission.bucketPop(bucket.String())
+	return p.Pools[idx].Decommission.bucketPop(bucket.String())
 }
 
 func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) {
@@ -620,11 +617,12 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:   objInfo.VersionID,
 			UserDefined: objInfo.UserDefined,
+			NoAuditLog:  true,
 		})
 		if err != nil {
 			return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err)
 		}
-		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{})
+		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{NoAuditLog: true})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
 			hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
@@ -639,6 +637,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 					IndexCB: func() []byte {
 						return part.Index // Preserve part Index to ensure decompression works.
 					},
+					NoAuditLog: true,
 				})
 			if err != nil {
 				return fmt.Errorf("decommissionObject: PutObjectPart() %w", err)
@@ -655,6 +654,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
 			DataMovement: true,
 			MTime:        objInfo.ModTime,
+			NoAuditLog:   true,
 		})
 		if err != nil {
 			err = fmt.Errorf("decommissionObject: CompleteMultipartUpload() %w", err)
@@ -680,6 +680,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			IndexCB: func() []byte {
 				return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works.
 			},
+			NoAuditLog: true,
 		})
 	if err != nil {
 		err = fmt.Errorf("decommissionObject: PutObject() %w", err)
@@ -699,17 +700,18 @@ func (v versionsSorter) reverse() {
 }
 
 func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi decomBucketInfo, fn func(entry metaCacheEntry)) error {
-	disks := set.getOnlineDisks()
+	disks, _ := set.getOnlineDisksWithHealing(false)
 	if len(disks) == 0 {
 		return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints())
 	}
 
-	listQuorum := (len(disks) + 1) / 2
+	// However many we ask, versions must exist on ~50%
+	listingQuorum := (set.setDriveCount + 1) / 2
 
 	// How to resolve partial results.
 	resolver := metadataResolutionParams{
-		dirQuorum: listQuorum, // make sure to capture all quorum ratios
-		objQuorum: listQuorum, // make sure to capture all quorum ratios
+		dirQuorum: listingQuorum, // make sure to capture all quorum ratios
+		objQuorum: listingQuorum, // make sure to capture all quorum ratios
 		bucket:    bi.Name,
 	}
 
@@ -719,7 +721,7 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
 		path:           bi.Prefix,
 		recursive:      true,
 		forwardTo:      "",
-		minDisks:       listQuorum,
+		minDisks:       listingQuorum,
 		reportNotFound: false,
 		agreed:         fn,
 		partial: func(entries metaCacheEntries, _ []error) {
@@ -736,14 +738,15 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
 
-	wStr := env.Get("_MINIO_DECOMMISSION_WORKERS", strconv.Itoa(len(pool.sets)))
-	workerSize, err := strconv.Atoi(wStr)
+	const envDecomWorkers = "_MINIO_DECOMMISSION_WORKERS"
+	workerSize, err := env.GetInt(envDecomWorkers, len(pool.sets))
 	if err != nil {
-		return err
+		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
+		workerSize = len(pool.sets)
 	}
 
-	// each set get its own thread separate from the concurrent
-	// objects/versions being decommissioned.
+	// Each decom worker needs one List() goroutine/worker
+	// add that many extra workers.
 	workerSize += len(pool.sets)
 
 	wk, err := workers.New(workerSize)
@@ -839,6 +842,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							DeleteReplication:  version.ReplicationState,
 							DeleteMarker:       true, // make sure we create a delete marker
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
+							NoAuditLog:         true,
 						})
 					var failure bool
 					if err != nil {
@@ -858,6 +862,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						// Success keep a count.
 						decommissioned++
 					}
+					auditLogDecom(ctx, "DecomCopyDeleteMarker", bi.Name, version.Name, versionID, err)
 					continue
 				}
 
@@ -888,6 +893,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							VersionID:    versionID,
 							NoDecryption: true,
 							NoLock:       true,
+							NoAuditLog:   true,
 						})
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
@@ -941,6 +947,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					ObjectOptions{
 						DeletePrefix:       true, // use prefix delete to delete all versions at once.
 						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
+						NoAuditLog:         true,
 					},
 				)
 				stopFn(err)
@@ -965,6 +972,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// We will perpetually retry listing if it fails, since we cannot
 			// possibly give up in this matter
 			for {
+				if contextCanceled(ctx) {
+					break
+				}
+
 				err := set.listObjectsToDecommission(ctx, bi,
 					func(entry metaCacheEntry) {
 						wk.Take()
@@ -1042,8 +1053,10 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 			}
 
 			z.poolMetaMutex.Lock()
-			z.poolMeta.BucketDone(idx, bucket) // remove from pendingBuckets and persist.
-			z.poolMeta.save(ctx, z.serverPools)
+			if z.poolMeta.BucketDone(idx, bucket) {
+				// remove from pendingBuckets and persist.
+				logger.LogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+			}
 			z.poolMetaMutex.Unlock()
 			continue
 		}
@@ -1058,8 +1071,9 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		stopFn(nil)
 
 		z.poolMetaMutex.Lock()
-		z.poolMeta.BucketDone(idx, bucket)
-		z.poolMeta.save(ctx, z.serverPools)
+		if z.poolMeta.BucketDone(idx, bucket) {
+			logger.LogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+		}
 		z.poolMetaMutex.Unlock()
 	}
 	return nil

cmd/erasure-server-pool-rebalance.go

@@ -26,17 +26,17 @@ import (
 	"math"
 	"math/rand"
 	"net/http"
-	"strconv"
 	"strings"
-	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/hash"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
+	"github.com/minio/pkg/v2/workers"
 )
 
 //go:generate msgp -file $GOFILE -unexported
@@ -485,6 +485,41 @@ func (z *erasureServerPools) checkIfRebalanceDone(poolIdx int) bool {
 	return false
 }
 
+func (set *erasureObjects) listObjectsToRebalance(ctx context.Context, bucketName string, fn func(entry metaCacheEntry)) error {
+	disks, _ := set.getOnlineDisksWithHealing(false)
+	if len(disks) == 0 {
+		return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints())
+	}
+
+	// However many we ask, versions must exist on ~50%
+	listingQuorum := (set.setDriveCount + 1) / 2
+
+	// How to resolve partial results.
+	resolver := metadataResolutionParams{
+		dirQuorum: listingQuorum, // make sure to capture all quorum ratios
+		objQuorum: listingQuorum, // make sure to capture all quorum ratios
+		bucket:    bucketName,
+	}
+
+	err := listPathRaw(ctx, listPathRawOptions{
+		disks:          disks,
+		bucket:         bucketName,
+		recursive:      true,
+		forwardTo:      "",
+		minDisks:       listingQuorum,
+		reportNotFound: false,
+		agreed:         fn,
+		partial: func(entries metaCacheEntries, _ []error) {
+			entry, ok := entries.resolve(&resolver)
+			if ok {
+				fn(*entry)
+			}
+		},
+		finished: nil,
+	})
+	return err
+}
+
 // rebalanceBucket rebalances objects under bucket in poolIdx pool
 func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
@@ -496,23 +531,25 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 	rcfg, _ := getReplicationConfig(ctx, bucket)
 
 	pool := z.serverPools[poolIdx]
+
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
-	wStr := env.Get(envRebalanceWorkers, strconv.Itoa(len(pool.sets)))
-	workerSize, err := strconv.Atoi(wStr)
+	workerSize, err := env.GetInt(envRebalanceWorkers, len(pool.sets))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("invalid %s value: %s err: %v, defaulting to %d", envRebalanceWorkers, wStr, err, len(pool.sets)))
+		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
 		workerSize = len(pool.sets)
 	}
-	workers := make(chan struct{}, workerSize)
-	var wg sync.WaitGroup
-	for _, set := range pool.sets {
+
+	// Each decom worker needs one List() goroutine/worker
+	// add that many extra workers.
+	workerSize += len(pool.sets)
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		return err
+	}
+
+	for setIdx, set := range pool.sets {
 		set := set
-		disks := set.getOnlineDisks()
-		if len(disks) == 0 {
-			logger.LogIf(ctx, fmt.Errorf("no online disks found for set with endpoints %s",
-				set.getEndpoints()))
-			continue
-		}
 
 		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 			if lc == nil {
@@ -531,10 +568,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 		}
 
 		rebalanceEntry := func(entry metaCacheEntry) {
-			defer func() {
-				<-workers
-				wg.Done()
-			}()
+			defer wk.Give()
 
 			if entry.isDir() {
 				return
@@ -592,6 +626,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 							DeleteReplication: version.ReplicationState,
 							DeleteMarker:      true, // make sure we create a delete marker
 							SkipRebalancing:   true, // make sure we skip the decommissioned pool
+							NoAuditLog:        true,
 						})
 					var failure bool
 					if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
@@ -603,6 +638,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						z.updatePoolStats(poolIdx, bucket, version)
 						rebalanced++
 					}
+					auditLogRebalance(ctx, "Rebalance:DeleteMarker", bucket, version.Name, versionID, err)
 					continue
 				}
 
@@ -619,6 +655,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 							VersionID:    versionID,
 							NoDecryption: true,
 							NoLock:       true,
+							NoAuditLog:   true,
 						})
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
@@ -663,6 +700,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					ObjectOptions{
 						DeletePrefix:       true, // use prefix delete to delete all versions at once.
 						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
+						NoAuditLog:         true,
 					},
 				)
 				stopFn(err)
@@ -673,44 +711,24 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			}
 		}
 
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			listQuorum := (len(disks) + 1) / 2
-
-			// How to resolve partial results.
-			resolver := metadataResolutionParams{
-				dirQuorum: listQuorum, // make sure to capture all quorum ratios
-				objQuorum: listQuorum, // make sure to capture all quorum ratios
-				bucket:    bucket,
-			}
-			err := listPathRaw(ctx, listPathRawOptions{
-				disks:          disks,
-				bucket:         bucket,
-				recursive:      true,
-				forwardTo:      "",
-				minDisks:       listQuorum,
-				reportNotFound: false,
-				agreed: func(entry metaCacheEntry) {
-					workers <- struct{}{}
-					wg.Add(1)
+		wk.Take()
+		go func(setIdx int) {
+			defer wk.Give()
+			err := set.listObjectsToRebalance(ctx, bucket,
+				func(entry metaCacheEntry) {
+					wk.Take()
 					go rebalanceEntry(entry)
 				},
-				partial: func(entries metaCacheEntries, _ []error) {
-					entry, ok := entries.resolve(&resolver)
-					if ok {
-						workers <- struct{}{}
-						wg.Add(1)
-						go rebalanceEntry(*entry)
-					}
-				},
-				finished: nil,
-			})
-			logger.LogIf(ctx, err)
-		}()
+			)
+			if err == nil || errors.Is(err, context.Canceled) {
+				return
+			}
+			setN := humanize.Ordinal(setIdx + 1)
+			logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN)
+		}(setIdx)
 	}
-	wg.Wait()
+
+	wk.Wait()
 	return nil
 }
 
@@ -783,11 +801,12 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 		res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
 			VersionID:   oi.VersionID,
 			UserDefined: oi.UserDefined,
+			NoAuditLog:  true,
 		})
 		if err != nil {
 			return fmt.Errorf("rebalanceObject: NewMultipartUpload() %w", err)
 		}
-		defer z.AbortMultipartUpload(ctx, bucket, oi.Name, res.UploadID, ObjectOptions{})
+		defer z.AbortMultipartUpload(ctx, bucket, oi.Name, res.UploadID, ObjectOptions{NoAuditLog: true})
 
 		parts := make([]CompletePart, len(oi.Parts))
 		for i, part := range oi.Parts {
@@ -803,6 +822,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 					IndexCB: func() []byte {
 						return part.Index // Preserve part Index to ensure decompression works.
 					},
+					NoAuditLog: true,
 				})
 			if err != nil {
 				return fmt.Errorf("rebalanceObject: PutObjectPart() %w", err)
@@ -815,6 +835,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 		_, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{
 			DataMovement: true,
 			MTime:        oi.ModTime,
+			NoAuditLog:   true,
 		})
 		if err != nil {
 			err = fmt.Errorf("rebalanceObject: CompleteMultipartUpload() %w", err)
@@ -840,6 +861,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 			IndexCB: func() []byte {
 				return oi.Parts[0].Index // Preserve part Index to ensure decompression works.
 			},
+			NoAuditLog: true,
 		})
 	if err != nil {
 		err = fmt.Errorf("rebalanceObject: PutObject() %w", err)

cmd/erasure-server-pool-rebalance_gen.go

@@ -614,6 +614,7 @@ func (z *rebalanceMetrics) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z rebalanceMetrics) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -625,6 +626,7 @@ func (z rebalanceMetrics) EncodeMsg(en *msgp.Writer) (err error) {
 func (z rebalanceMetrics) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }

cmd/erasure-server-pool.go

@@ -92,15 +92,15 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		n = 2048
 	}
 
-	if globalIsCICD {
-		n = 256 // 256MiB for CI/CD environments is sufficient
-	}
-
 	// Avoid allocating more than half of the available memory
 	if maxN := availableMemory() / (blockSizeV2 * 2); n > maxN {
 		n = maxN
 	}
 
+	if globalIsCICD || strconv.IntSize == 32 {
+		n = 256 // 256MiB for CI/CD environments is sufficient or on 32bit platforms.
+	}
+
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV2.
 	globalBytePoolCap = bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
@@ -175,8 +175,25 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	z.poolMeta = newPoolMeta(z, poolMeta{})
 	z.poolMeta.dontSave = true
 
+	bootstrapTrace("newSharedLock", func() {
+		globalLeaderLock = newSharedLock(GlobalContext, z, "leader.lock")
+	})
+
+	// Enable background operations on
+	//
+	// - Disk auto healing
+	// - MRF (most recently failed) healing
+	// - Background expiration routine for lifecycle policies
+	bootstrapTrace("initAutoHeal", func() {
+		initAutoHeal(GlobalContext, z)
+	})
+
+	bootstrapTrace("initHealMRF", func() {
+		go globalMRFState.healRoutine(z)
+	})
+
 	// initialize the object layer.
-	setObjectLayer(z)
+	defer setObjectLayer(z)
 
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	attempt := 1
@@ -1249,13 +1266,17 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 	return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
 }
 
+func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	return z.listObjectsGeneric(ctx, bucket, prefix, marker, delimiter, maxKeys, true)
+}
+
 func (z *erasureServerPools) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
 	marker := continuationToken
 	if marker == "" {
 		marker = startAfter
 	}
 
-	loi, err := z.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
+	loi, err := z.listObjectsGeneric(ctx, bucket, prefix, marker, delimiter, maxKeys, false)
 	if err != nil {
 		return ListObjectsV2Info{}, err
 	}
@@ -1364,9 +1385,10 @@ func maxKeysPlusOne(maxKeys int, addOne bool) int {
 	return maxKeys
 }
 
-func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, v1 bool) (ListObjectsInfo, error) {
 	var loi ListObjectsInfo
 	opts := listPathOptions{
+		V1:          v1,
 		Bucket:      bucket,
 		Prefix:      prefix,
 		Separator:   delimiter,
@@ -1537,8 +1559,23 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	if loi.IsTruncated {
 		last := objects[len(objects)-1]
-		loi.NextMarker = opts.encodeMarker(last.Name)
+		loi.NextMarker = last.Name
 	}
+
+	if merged.lastSkippedEntry != "" {
+		if merged.lastSkippedEntry > loi.NextMarker {
+			// An object hidden by ILM was found during listing. Since the number of entries
+			// fetched from drives is limited, set IsTruncated to true to ask the s3 client
+			// to continue listing if it wishes in order to find if there is more objects.
+			loi.IsTruncated = true
+			loi.NextMarker = merged.lastSkippedEntry
+		}
+	}
+
+	if loi.NextMarker != "" {
+		loi.NextMarker = opts.encodeMarker(loi.NextMarker)
+	}
+
 	return loi, nil
 }
 
@@ -1848,14 +1885,13 @@ var listBucketsCache = cachevalue.New[[]BucketInfo]()
 // that all buckets are present on all serverPools.
 func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	if opts.Cached {
-		listBucketsCache.Once.Do(func() {
-			listBucketsCache.TTL = time.Second
-
-			listBucketsCache.Relax = true
-			listBucketsCache.Update = func() ([]BucketInfo, error) {
+		listBucketsCache.InitOnce(time.Second,
+			cachevalue.Opts{ReturnLastGood: true, NoWait: true},
+			func() ([]BucketInfo, error) {
 				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+				defer cancel()
+
 				buckets, err = z.s3Peer.ListBuckets(ctx, opts)
-				cancel()
 				if err != nil {
 					return nil, err
 				}
@@ -1866,8 +1902,8 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions
 					}
 				}
 				return buckets, nil
-			}
-		})
+			},
+		)
 
 		return listBucketsCache.Get()
 	}
@@ -2302,6 +2338,21 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	storageInfo := z.StorageInfo(ctx, false)
 
 	for _, disk := range storageInfo.Disks {
+		if opts.Maintenance {
+			var skip bool
+			globalLocalDrivesMu.RLock()
+			for _, drive := range globalLocalDrives {
+				if drive != nil && drive.Endpoint().String() == disk.Endpoint {
+					skip = true
+					break
+				}
+			}
+			globalLocalDrivesMu.RUnlock()
+			if skip {
+				continue
+			}
+		}
+
 		if disk.PoolIndex > -1 && disk.SetIndex > -1 {
 			if disk.State == madmin.DriveStateOk {
 				si := erasureSetUpCount[disk.PoolIndex][disk.SetIndex]
@@ -2360,6 +2411,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 
 	result := HealthResult{
 		Healthy:       true,
+		HealthyRead:   true,
 		WriteQuorum:   maximumWriteQuorum,
 		ReadQuorum:    maximumReadQuorum,
 		UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
@@ -2386,23 +2438,27 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 				WriteQuorum:   poolWriteQuorums[poolIdx],
 			})
 
-			result.Healthy = erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
-			if !result.Healthy {
+			healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
+			if !healthy {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
 						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
 			}
-			result.HealthyRead = erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
-			if !result.HealthyRead {
+			result.Healthy = result.Healthy && healthy
+
+			healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
+			if !healthyRead {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Read quorum may be lost on pool: %d, set: %d, expected read quorum: %d",
 						poolIdx, setIdx, poolReadQuorums[poolIdx]))
 			}
+			result.HealthyRead = result.HealthyRead && healthyRead
 		}
 	}
 
 	if opts.Maintenance {
 		result.Healthy = result.Healthy && drivesHealing == 0
+		result.HealthyRead = result.HealthyRead && drivesHealing == 0
 		result.HealingDrives = drivesHealing
 	}
 

cmd/erasure.go

@@ -176,6 +176,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) (disks
 				PoolIndex: endpoints[index].PoolIdx,
 				SetIndex:  endpoints[index].SetIdx,
 				DiskIndex: endpoints[index].DiskIdx,
+				Local:     endpoints[index].IsLocal,
 			}
 			if disks[index] == OfflineDisk {
 				di.State = diskErrToDriveState(errDiskNotFound)

cmd/ftp-server.go

@@ -136,7 +136,7 @@ func startFTPServer(args []string) {
 
 	ftpServer, err := ftp.NewServer(&ftp.Options{
 		Name:           name,
-		WelcomeMessage: fmt.Sprintf("Welcome to MinIO FTP Server Version='%s' License='GNU AGPLv3'", Version),
+		WelcomeMessage: fmt.Sprintf("Welcome to '%s' FTP Server Version='%s' License='%s'", MinioStoreName, MinioLicense, Version),
 		Driver:         NewFTPDriver(),
 		Port:           port,
 		Perm:           ftp.NewSimplePerm("nobody", "nobody"),

cmd/generic-handlers.go

@@ -233,7 +233,8 @@ func guessIsMetricsReq(req *http.Request) bool {
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath ||
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath ||
+		strings.HasPrefix(req.URL.Path, minioReservedBucketPath+metricsV3Path)
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.

cmd/global-heal.go

@@ -224,7 +224,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 		disks, _ := er.getOnlineDisksWithHealing(false)
 		if len(disks) == 0 {
-			logger.LogIf(ctx, fmt.Errorf("no online disks found to heal the bucket `%s`", bucket))
+			// No object healing necessary
+			tracker.bucketDone(bucket)
+			logger.LogIf(ctx, tracker.update(ctx))
 			continue
 		}
 
@@ -236,6 +238,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		type healEntryResult struct {
 			bytes     uint64
 			success   bool
+			skipped   bool
 			entryDone bool
 			name      string
 		}
@@ -256,6 +259,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				bytes: sz,
 			}
 		}
+		healEntrySkipped := func(sz uint64) healEntryResult {
+			return healEntryResult{
+				bytes:   sz,
+				skipped: true,
+			}
+		}
 
 		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 			if lc == nil {
@@ -289,13 +298,16 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					continue
 				}
 
-				tracker.updateProgress(res.success, res.bytes)
+				tracker.updateProgress(res.success, res.skipped, res.bytes)
 			}
 		}()
 
 		send := func(result healEntryResult) bool {
 			select {
 			case <-ctx.Done():
+				if !contextCanceled(ctx) {
+					logger.LogIf(ctx, ctx.Err())
+				}
 				return false
 			case results <- result:
 				return true
@@ -367,6 +379,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bucket, version.Name, version) {
 					versionNotFound++
+					if !send(healEntrySkipped(uint64(version.Size))) {
+						return
+					}
 					continue
 				}
 
@@ -472,9 +487,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 func healBucket(bucket string, scan madmin.HealScanMode) error {
 	// Get background heal sequence to send elements to heal
-	globalHealStateLK.Lock()
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-	globalHealStateLK.Unlock()
 	if ok {
 		return bgSeq.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket)
 	}
@@ -484,9 +497,7 @@ func healBucket(bucket string, scan madmin.HealScanMode) error {
 // healObject sends the given object/version to the background healing workers
 func healObject(bucket, object, versionID string, scan madmin.HealScanMode) error {
 	// Get background heal sequence to send elements to heal
-	globalHealStateLK.Lock()
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-	globalHealStateLK.Unlock()
 	if ok {
 		return bgSeq.queueHealTask(healSource{
 			bucket:    bucket,

cmd/globals.go

@@ -160,16 +160,18 @@ type serverCtxt struct {
 	FTP  []string
 	SFTP []string
 
-	UserTimeout            time.Duration
-	ConnReadDeadline       time.Duration
-	ConnWriteDeadline      time.Duration
-	ConnClientReadDeadline time.Duration
+	UserTimeout             time.Duration
+	ConnReadDeadline        time.Duration
+	ConnWriteDeadline       time.Duration
+	ConnClientReadDeadline  time.Duration
+	ConnClientWriteDeadline time.Duration
 
 	ShutdownTimeout     time.Duration
 	IdleTimeout         time.Duration
 	ReadHeaderTimeout   time.Duration
 	MaxIdleConnsPerHost int
 
+	CrossDomainXML string
 	// The layout of disks as interpreted
 	Layout disksLayout
 }
@@ -311,7 +313,8 @@ var (
 	// Time when the server is started
 	globalBootTime = UTCNow()
 
-	globalActiveCred auth.Credentials
+	globalActiveCred         auth.Credentials
+	globalSiteReplicatorCred siteReplicatorCred
 
 	// Captures if root credentials are set via ENV.
 	globalCredViaEnv bool
@@ -378,13 +381,15 @@ var (
 		return *ptr
 	}
 
-	globalAllHealState *allHealState
+	globalAllHealState = newHealState(GlobalContext, true)
 
 	// The always present healing routine ready to heal objects
-	globalBackgroundHealRoutine *healRoutine
-	globalBackgroundHealState   *allHealState
+	globalBackgroundHealRoutine = newHealRoutine()
+	globalBackgroundHealState   = newHealState(GlobalContext, false)
 
-	globalMRFState mrfState
+	globalMRFState = mrfState{
+		opCh: make(chan partialOperation, mrfOpsQueueSize),
+	}
 
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool
@@ -407,8 +412,6 @@ var (
 
 	globalTierConfigMgr *TierConfigMgr
 
-	globalTierJournal *TierJournal
-
 	globalConsoleSrv *consoleapi.Server
 
 	// handles service freeze or un-freeze S3 API calls.
@@ -468,6 +471,7 @@ var (
 
 	// Indicates if server was started as `--address ":0"`
 	globalDynamicAPIPort bool
+
 	// Add new variable global values here.
 )
 

cmd/grid.go

@@ -35,13 +35,9 @@ var globalGrid atomic.Pointer[grid.Manager]
 var globalGridStart = make(chan struct{})
 
 func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
-	lookupHost := globalDNSCache.LookupHost
-	if IsKubernetes() || IsDocker() {
-		lookupHost = nil
-	}
 	hosts, local := eps.GridHosts()
 	g, err := grid.NewManager(ctx, grid.ManagerOptions{
-		Dialer:       grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))),
+		Dialer:       grid.ContextDialer(xhttp.DialContextWithLookupHost(globalDNSCache.LookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))),
 		Local:        local,
 		Hosts:        hosts,
 		AddAuth:      newCachedAuthToken(),

cmd/handler-api.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -54,6 +55,7 @@ type apiConfig struct {
 	gzipObjects                 bool
 	rootAccess                  bool
 	syncEvents                  bool
+	objectMaxVersions           int
 }
 
 const (
@@ -70,7 +72,7 @@ func cgroupMemLimit() (limit uint64) {
 	if err != nil {
 		return 0
 	}
-	limit, err = strconv.ParseUint(string(buf), 10, 64)
+	limit, err = strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64)
 	if err != nil {
 		// The kernel can return valid but non integer values
 		// but still, no need to interpret more
@@ -86,14 +88,14 @@ func cgroupMemLimit() (limit uint64) {
 }
 
 func availableMemory() (available uint64) {
-	available = 8 << 30 // Default to 8 GiB when we can't find the limits.
+	available = 2048 * blockSizeV2 * 2 // Default to 4 GiB when we can't find the limits.
 
 	if runtime.GOOS == "linux" {
 		// Useful in container mode
 		limit := cgroupMemLimit()
 		if limit > 0 {
-			// A valid value is found
-			available = limit
+			// A valid value is found, return its 75%
+			available = (limit * 3) / 4
 			return
 		}
 	} // for all other platforms limits are based on virtual memory.
@@ -102,7 +104,8 @@ func availableMemory() (available uint64) {
 	if err != nil {
 		return
 	}
-	available = memStats.Available / 2
+	// A valid value is available return its 75%
+	available = (memStats.Available * 3) / 4
 	return
 }
 
@@ -131,6 +134,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 
 	var apiRequestsMaxPerNode int
 	if cfg.RequestsMax <= 0 {
+		// Returns 75% of max memory allowed
 		maxMem := availableMemory()
 
 		// max requests per node is calculated as
@@ -169,7 +173,9 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	}
 	t.replicationPriority = cfg.ReplicationPriority
 	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
-	if globalTransitionState != nil && cfg.TransitionWorkers != t.transitionWorkers {
+
+	// N B api.transition_workers will be deprecated
+	if globalTransitionState != nil {
 		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
 	}
 	t.transitionWorkers = cfg.TransitionWorkers
@@ -181,6 +187,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.gzipObjects = cfg.GzipObjects
 	t.rootAccess = cfg.RootAccess
 	t.syncEvents = cfg.SyncEvents
+	t.objectMaxVersions = cfg.ObjectMaxVersions
 }
 
 func (t *apiConfig) odirectEnabled() bool {
@@ -284,7 +291,11 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 	defer t.mu.RUnlock()
 
 	if t.requestsPool == nil {
-		return nil, time.Duration(0)
+		return nil, 10 * time.Second
+	}
+
+	if t.requestsDeadline <= 0 {
+		return t.requestsPool, 10 * time.Second
 	}
 
 	return t.requestsPool, t.requestsDeadline
@@ -381,3 +392,15 @@ func (t *apiConfig) isSyncEventsEnabled() bool {
 
 	return t.syncEvents
 }
+
+func (t *apiConfig) getObjectMaxVersions() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.objectMaxVersions <= 0 {
+		// defaults to 'maxObjectVersions' when unset.
+		return maxObjectVersions
+	}
+
+	return t.objectMaxVersions
+}

cmd/handler-utils.go

@@ -296,27 +296,23 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 
 		bucket, _ := path2BucketObject(resource)
 
-		globalHTTPStats.currentS3Requests.Inc(api)
-		defer globalHTTPStats.currentS3Requests.Dec(api)
-
-		_, err = globalBucketMetadataSys.Get(bucket) // check if this bucket exists.
-		if bucket != "" && bucket != minioReservedBucket && err == nil {
+		meta, err := globalBucketMetadataSys.Get(bucket) // check if this bucket exists.
+		countBktStat := bucket != "" && bucket != minioReservedBucket && err == nil && !meta.Created.IsZero()
+		if countBktStat {
 			globalBucketHTTPStats.updateHTTPStats(bucket, api, nil)
 		}
 
+		globalHTTPStats.currentS3Requests.Inc(api)
 		f.ServeHTTP(w, r)
+		globalHTTPStats.currentS3Requests.Dec(api)
 
-		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
-		if !ok {
-			return
-		}
-
+		tc, _ := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 		if tc != nil {
 			globalHTTPStats.updateStats(api, tc.ResponseRecorder)
 			globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size()))
 			globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size()))
 
-			if bucket != "" && bucket != minioReservedBucket && err == nil {
+			if countBktStat {
 				globalBucketConnStats.incS3InputBytes(bucket, int64(tc.RequestRecorder.Size()))
 				globalBucketConnStats.incS3OutputBytes(bucket, int64(tc.ResponseRecorder.Size()))
 				globalBucketHTTPStats.updateHTTPStats(bucket, api, tc.ResponseRecorder)
@@ -370,12 +366,6 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	desc := "Do not upgrade one server at a time - please follow the recommended guidelines mentioned here https://github.com/minio/minio#upgrading-minio for your environment"
 	switch {
-	case strings.HasPrefix(r.URL.Path, peerS3Prefix):
-		writeErrorResponseString(r.Context(), w, APIError{
-			Code:           "XMinioPeerS3VersionMismatch",
-			Description:    desc,
-			HTTPStatusCode: http.StatusUpgradeRequired,
-		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",

cmd/http-stats.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"net/http"
+	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -268,6 +269,28 @@ func (s *bucketConnStats) getS3InOutBytes() map[string]inOutBytes {
 	return bucketStats
 }
 
+// Return S3 total input/output bytes for each
+func (s *bucketConnStats) getBucketS3InOutBytes(buckets []string) map[string]inOutBytes {
+	s.RLock()
+	defer s.RUnlock()
+
+	if len(s.stats) == 0 || len(buckets) == 0 {
+		return nil
+	}
+
+	bucketStats := make(map[string]inOutBytes, len(buckets))
+	for _, bucket := range buckets {
+		if stats, ok := s.stats[bucket]; ok {
+			bucketStats[bucket] = inOutBytes{
+				In:  stats.s3InputBytes,
+				Out: stats.s3OutputBytes,
+			}
+		}
+	}
+
+	return bucketStats
+}
+
 // delete metrics once bucket is deleted.
 func (s *bucketConnStats) delete(bucket string) {
 	s.Lock()
@@ -326,7 +349,7 @@ func (stats *HTTPAPIStats) Get(api string) int {
 }
 
 // Load returns the recorded stats.
-func (stats *HTTPAPIStats) Load() map[string]int {
+func (stats *HTTPAPIStats) Load(toLower bool) map[string]int {
 	if stats == nil {
 		return map[string]int{}
 	}
@@ -336,6 +359,9 @@ func (stats *HTTPAPIStats) Load() map[string]int {
 
 	apiStats := make(map[string]int, len(stats.apiStats))
 	for k, v := range stats.apiStats {
+		if toLower {
+			k = strings.ToLower(k)
+		}
 		apiStats[k] = v
 	}
 	return apiStats
@@ -373,7 +399,7 @@ func (st *HTTPStats) incS3RequestsIncoming() {
 }
 
 // Converts http stats into struct to be sent back to the client.
-func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
+func (st *HTTPStats) toServerHTTPStats(toLowerKeys bool) ServerHTTPStats {
 	serverStats := ServerHTTPStats{}
 	serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0)
 	serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue)
@@ -382,22 +408,22 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	serverStats.TotalS3RejectedHeader = atomic.LoadUint64(&st.rejectedRequestsHeader)
 	serverStats.TotalS3RejectedInvalid = atomic.LoadUint64(&st.rejectedRequestsInvalid)
 	serverStats.CurrentS3Requests = ServerHTTPAPIStats{
-		APIStats: st.currentS3Requests.Load(),
+		APIStats: st.currentS3Requests.Load(toLowerKeys),
 	}
 	serverStats.TotalS3Requests = ServerHTTPAPIStats{
-		APIStats: st.totalS3Requests.Load(),
+		APIStats: st.totalS3Requests.Load(toLowerKeys),
 	}
 	serverStats.TotalS3Errors = ServerHTTPAPIStats{
-		APIStats: st.totalS3Errors.Load(),
+		APIStats: st.totalS3Errors.Load(toLowerKeys),
 	}
 	serverStats.TotalS34xxErrors = ServerHTTPAPIStats{
-		APIStats: st.totalS34xxErrors.Load(),
+		APIStats: st.totalS34xxErrors.Load(toLowerKeys),
 	}
 	serverStats.TotalS35xxErrors = ServerHTTPAPIStats{
-		APIStats: st.totalS35xxErrors.Load(),
+		APIStats: st.totalS35xxErrors.Load(toLowerKeys),
 	}
 	serverStats.TotalS3Canceled = ServerHTTPAPIStats{
-		APIStats: st.totalS3Canceled.Load(),
+		APIStats: st.totalS3Canceled.Load(toLowerKeys),
 	}
 	return serverStats
 }

cmd/iam-etcd-store.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/puzpuzpuz/xsync/v3"
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	etcd "go.etcd.io/etcd/client/v3"
 )
@@ -143,6 +144,10 @@ func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error
 	return deleteKeyEtcd(ctx, ies.client, path)
 }
 
+func (ies *IAMEtcdStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, _ int) error {
+	return ies.loadPolicyDoc(ctx, policy, m)
+}
+
 func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
 	data, err := ies.loadIAMConfigBytes(ctx, getPolicyDocPath(policy))
 	if err != nil {
@@ -321,7 +326,11 @@ func (ies *IAMEtcdStore) loadGroups(ctx context.Context, m map[string]GroupInfo)
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error {
+	return ies.loadMappedPolicy(ctx, name, userType, isGroup, m)
+}
+
+func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	var p MappedPolicy
 	err := ies.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
@@ -330,11 +339,11 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
 		}
 		return err
 	}
-	m[name] = p
+	m.Store(name, p)
 	return nil
 }
 
-func getMappedPolicy(ctx context.Context, kv *mvccpb.KeyValue, userType IAMUserType, isGroup bool, m map[string]MappedPolicy, basePrefix string) error {
+func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
 	var p MappedPolicy
 	err := getIAMConfig(&p, kv.Value, string(kv.Key))
 	if err != nil {
@@ -344,11 +353,11 @@ func getMappedPolicy(ctx context.Context, kv *mvccpb.KeyValue, userType IAMUserT
 		return err
 	}
 	name := extractPathPrefixAndSuffix(string(kv.Key), basePrefix, ".json")
-	m[name] = p
+	m.Store(name, p)
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	cctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
 	var basePrefix string
@@ -373,7 +382,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 	// Parse all policies mapping to create the proper data model
 	for _, kv := range r.Kvs {
-		if err = getMappedPolicy(ctx, kv, userType, isGroup, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
+		if err = getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
 			return err
 		}
 	}

cmd/iam-object-store.go

@@ -25,6 +25,7 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"time"
 	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
@@ -34,6 +35,7 @@ import (
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/puzpuzpuz/xsync/v3"
 )
 
 // IAMObjectStore implements IAMStorageAPI
@@ -146,6 +148,41 @@ func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) e
 	return deleteConfig(ctx, iamOS.objAPI, path)
 }
 
+func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error {
+	for {
+	retry:
+		data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy))
+		if err != nil {
+			if err == errConfigNotFound {
+				return errNoSuchPolicy
+			}
+			retries--
+			if retries <= 0 {
+				return err
+			}
+			time.Sleep(500 * time.Millisecond)
+			goto retry
+		}
+
+		var p PolicyDoc
+		err = p.parseJSON(data)
+		if err != nil {
+			return err
+		}
+
+		if p.Version == 0 {
+			// This means that policy was in the old version (without any
+			// timestamp info). We fetch the mod time of the file and save
+			// that as create and update date.
+			p.CreateDate = objInfo.ModTime
+			p.UpdateDate = objInfo.ModTime
+		}
+
+		m[policy] = p
+		return nil
+	}
+}
+
 func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
 	data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy))
 	if err != nil {
@@ -289,9 +326,29 @@ func (iamOS *IAMObjectStore) loadGroups(ctx context.Context, m map[string]GroupI
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool,
-	m map[string]MappedPolicy,
-) error {
+func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error {
+	for {
+	retry:
+		var p MappedPolicy
+		err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
+		if err != nil {
+			if err == errConfigNotFound {
+				return errNoSuchPolicy
+			}
+			retries--
+			if retries <= 0 {
+				return err
+			}
+			time.Sleep(500 * time.Millisecond)
+			goto retry
+		}
+
+		m.Store(name, p)
+		return nil
+	}
+}
+
+func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	var p MappedPolicy
 	err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
@@ -300,11 +357,12 @@ func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string,
 		}
 		return err
 	}
-	m[name] = p
+
+	m.Store(name, p)
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	var basePath string
 	if isGroup {
 		basePath = iamConfigPolicyDBGroupsPrefix

cmd/iam-store.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/policy"
+	"github.com/puzpuzpuz/xsync/v3"
 )
 
 const (
@@ -179,6 +180,16 @@ type MappedPolicy struct {
 	UpdatedAt time.Time `json:"updatedAt,omitempty"`
 }
 
+// mappedPoliciesToMap copies the map of mapped policies to a regular map.
+func mappedPoliciesToMap(m *xsync.MapOf[string, MappedPolicy]) map[string]MappedPolicy {
+	policies := make(map[string]MappedPolicy, m.Size())
+	m.Range(func(k string, v MappedPolicy) bool {
+		policies[k] = v
+		return true
+	})
+	return policies
+}
+
 // converts a mapped policy into a slice of distinct policies
 func (mp MappedPolicy) toSlice() []string {
 	var policies []string
@@ -277,32 +288,32 @@ type iamCache struct {
 	// map of regular username to credentials
 	iamUsersMap map[string]UserIdentity
 	// map of regular username to policy names
-	iamUserPolicyMap map[string]MappedPolicy
+	iamUserPolicyMap *xsync.MapOf[string, MappedPolicy]
 
 	// STS accounts are loaded on demand and not via the periodic IAM reload.
 	// map of STS access key to credentials
 	iamSTSAccountsMap map[string]UserIdentity
 	// map of STS access key to policy names
-	iamSTSPolicyMap map[string]MappedPolicy
+	iamSTSPolicyMap *xsync.MapOf[string, MappedPolicy]
 
 	// map of group names to group info
 	iamGroupsMap map[string]GroupInfo
 	// map of user names to groups they are a member of
 	iamUserGroupMemberships map[string]set.StringSet
 	// map of group names to policy names
-	iamGroupPolicyMap map[string]MappedPolicy
+	iamGroupPolicyMap *xsync.MapOf[string, MappedPolicy]
 }
 
 func newIamCache() *iamCache {
 	return &iamCache{
 		iamPolicyDocsMap:        map[string]PolicyDoc{},
 		iamUsersMap:             map[string]UserIdentity{},
-		iamUserPolicyMap:        map[string]MappedPolicy{},
+		iamUserPolicyMap:        xsync.NewMapOf[string, MappedPolicy](),
 		iamSTSAccountsMap:       map[string]UserIdentity{},
-		iamSTSPolicyMap:         map[string]MappedPolicy{},
+		iamSTSPolicyMap:         xsync.NewMapOf[string, MappedPolicy](),
 		iamGroupsMap:            map[string]GroupInfo{},
 		iamUserGroupMemberships: map[string]set.StringSet{},
-		iamGroupPolicyMap:       map[string]MappedPolicy{},
+		iamGroupPolicyMap:       xsync.NewMapOf[string, MappedPolicy](),
 	}
 }
 
@@ -359,7 +370,13 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[name]
 			if !ok {
-				return nil, time.Time{}, errNoSuchGroup
+				if err := store.loadGroup(context.Background(), name, c.iamGroupsMap); err != nil {
+					return nil, time.Time{}, err
+				}
+				g, ok = c.iamGroupsMap[name]
+				if !ok {
+					return nil, time.Time{}, errNoSuchGroup
+				}
 			}
 
 			// Group is disabled, so we return no policy - this
@@ -369,7 +386,15 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 			}
 		}
 
-		return c.iamGroupPolicyMap[name].toSlice(), c.iamGroupPolicyMap[name].UpdatedAt, nil
+		policy, ok := c.iamGroupPolicyMap.Load(name)
+		if ok {
+			return policy.toSlice(), policy.UpdatedAt, nil
+		}
+		if err := store.loadMappedPolicyWithRetry(context.TODO(), name, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			return nil, time.Time{}, err
+		}
+		policy, _ = c.iamGroupPolicyMap.Load(name)
+		return policy.toSlice(), policy.UpdatedAt, nil
 	}
 
 	// When looking for a user's policies, we also check if the user
@@ -384,15 +409,24 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 	// For internal IDP regular/service account user accounts, the policy
 	// mapping is iamUserPolicyMap. For STS accounts, the parent user would be
 	// passed here and we lookup the mapping in iamSTSPolicyMap.
-	mp, ok := c.iamUserPolicyMap[name]
+	mp, ok := c.iamUserPolicyMap.Load(name)
 	if !ok {
-		// Since user "name" could be a parent user of an STS account, we lookup
-		// mappings for those too.
-		mp, ok = c.iamSTSPolicyMap[name]
+		if err := store.loadMappedPolicyWithRetry(context.TODO(), name, regUser, false, c.iamUserPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			return nil, time.Time{}, err
+		}
+
+		mp, ok = c.iamUserPolicyMap.Load(name)
 		if !ok {
-			// Attempt to load parent user mapping for STS accounts
-			store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap)
-			mp = c.iamSTSPolicyMap[name]
+			// Since user "name" could be a parent user of an STS account, we look up
+			// mappings for those too.
+			mp, ok = c.iamSTSPolicyMap.Load(name)
+			if !ok {
+				// Attempt to load parent user mapping for STS accounts
+				if err := store.loadMappedPolicyWithRetry(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+					return nil, time.Time{}, err
+				}
+				mp, _ = c.iamSTSPolicyMap.Load(name)
+			}
 		}
 	}
 
@@ -400,13 +434,34 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 	policies := mp.toSlice()
 
 	for _, group := range c.iamUserGroupMemberships[name].ToSlice() {
-		// Skip missing or disabled groups
-		gi, ok := c.iamGroupsMap[group]
-		if !ok || gi.Status == statusDisabled {
-			continue
+		if store.getUsersSysType() == MinIOUsersSysType {
+			g, ok := c.iamGroupsMap[group]
+			if !ok {
+				if err := store.loadGroup(context.Background(), group, c.iamGroupsMap); err != nil {
+					return nil, time.Time{}, err
+				}
+				g, ok = c.iamGroupsMap[group]
+				if !ok {
+					return nil, time.Time{}, errNoSuchGroup
+				}
+			}
+
+			// Group is disabled, so we return no policy - this
+			// ensures the request is denied.
+			if g.Status == statusDisabled {
+				return nil, time.Time{}, nil
+			}
 		}
 
-		policies = append(policies, c.iamGroupPolicyMap[group].toSlice()...)
+		policy, ok := c.iamGroupPolicyMap.Load(group)
+		if !ok {
+			if err := store.loadMappedPolicyWithRetry(context.TODO(), group, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return nil, time.Time{}, err
+			}
+			policy, _ = c.iamGroupPolicyMap.Load(group)
+		}
+
+		policies = append(policies, policy.toSlice()...)
 	}
 
 	return policies, mp.UpdatedAt, nil
@@ -441,13 +496,15 @@ type IAMStorageAPI interface {
 	runlock()
 	getUsersSysType() UsersSysType
 	loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error
+	loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error
 	loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error
 	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error
 	loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error
 	loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error
 	loadGroups(ctx context.Context, m map[string]GroupInfo) error
-	loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
-	loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
+	loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error
+	loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error
+	loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error
 	saveIAMConfig(ctx context.Context, item interface{}, path string, opts ...options) error
 	loadIAMConfig(ctx context.Context, item interface{}, path string) error
 	deleteIAMConfig(ctx context.Context, path string) error
@@ -572,9 +629,10 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
 		// here is to account for STS policy mapping changes that should apply
 		// for service accounts derived from such STS accounts (i.e. LDAP STS
 		// accounts).
-		for k, v := range newCache.iamSTSPolicyMap {
-			cache.iamSTSPolicyMap[k] = v
-		}
+		newCache.iamSTSPolicyMap.Range(func(k string, v MappedPolicy) bool {
+			cache.iamSTSPolicyMap.Store(k, v)
+			return true
+		})
 
 		cache.updatedAt = time.Now()
 	}
@@ -613,12 +671,10 @@ func (store *IAMStoreSys) GetMappedPolicy(name string, isGroup bool) (MappedPoli
 	defer store.runlock()
 
 	if isGroup {
-		v, ok := cache.iamGroupPolicyMap[name]
+		v, ok := cache.iamGroupPolicyMap.Load(name)
 		return v, ok
 	}
-
-	v, ok := cache.iamUserPolicyMap[name]
-	return v, ok
+	return cache.iamUserPolicyMap.Load(name)
 }
 
 // GroupNotificationHandler - updates in-memory cache on notification of
@@ -637,7 +693,7 @@ func (store *IAMStoreSys) GroupNotificationHandler(ctx context.Context, group st
 		// group does not exist - so remove from memory.
 		cache.removeGroupFromMembershipsMap(group)
 		delete(cache.iamGroupsMap, group)
-		delete(cache.iamGroupPolicyMap, group)
+		cache.iamGroupPolicyMap.Delete(group)
 
 		cache.updatedAt = time.Now()
 		return nil
@@ -816,7 +872,7 @@ func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string
 
 		// Delete from server memory
 		delete(cache.iamGroupsMap, group)
-		delete(cache.iamGroupPolicyMap, group)
+		cache.iamGroupPolicyMap.Delete(group)
 		cache.updatedAt = time.Now()
 		return cache.updatedAt, nil
 	}
@@ -908,16 +964,17 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 	}
 
 	if store.getUsersSysType() == LDAPUsersSysType {
-		m := map[string]MappedPolicy{}
+		m := xsync.NewMapOf[string, MappedPolicy]()
 		err = store.loadMappedPolicies(ctx, stsUser, true, m)
 		if err != nil {
 			return
 		}
 		cache.iamGroupPolicyMap = m
 		cache.updatedAt = time.Now()
-		for k := range cache.iamGroupPolicyMap {
+		cache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool {
 			res = append(res, k)
-		}
+			return true
+		})
 	}
 
 	return
@@ -935,9 +992,10 @@ func (store *IAMStoreSys) listGroups(ctx context.Context) (res []string, err err
 	}
 
 	if store.getUsersSysType() == LDAPUsersSysType {
-		for k := range cache.iamGroupPolicyMap {
+		cache.iamGroupPolicyMap.Range(func(k string, _ MappedPolicy) bool {
 			res = append(res, k)
-		}
+			return true
+		})
 	}
 	return
 }
@@ -960,14 +1018,14 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 	var mp MappedPolicy
 	if !isGroup {
 		if userType == stsUser {
-			stsMap := map[string]MappedPolicy{}
+			stsMap := xsync.NewMapOf[string, MappedPolicy]()
 
 			// Attempt to load parent user mapping for STS accounts
 			store.loadMappedPolicy(context.TODO(), name, stsUser, false, stsMap)
 
-			mp = stsMap[name]
+			mp, _ = stsMap.Load(name)
 		} else {
-			mp = cache.iamUserPolicyMap[name]
+			mp, _ = cache.iamUserPolicyMap.Load(name)
 		}
 	} else {
 		if store.getUsersSysType() == MinIOUsersSysType {
@@ -982,7 +1040,7 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 				return
 			}
 		}
-		mp = cache.iamGroupPolicyMap[name]
+		mp, _ = cache.iamGroupPolicyMap.Load(name)
 	}
 
 	// Compute net policy change effect and updated policy mapping
@@ -1025,12 +1083,12 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 		}
 		if !isGroup {
 			if userType == stsUser {
-				delete(cache.iamSTSPolicyMap, name)
+				cache.iamSTSPolicyMap.Delete(name)
 			} else {
-				delete(cache.iamUserPolicyMap, name)
+				cache.iamUserPolicyMap.Delete(name)
 			}
 		} else {
-			delete(cache.iamGroupPolicyMap, name)
+			cache.iamGroupPolicyMap.Delete(name)
 		}
 	} else {
 
@@ -1039,12 +1097,12 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 		}
 		if !isGroup {
 			if userType == stsUser {
-				cache.iamSTSPolicyMap[name] = newPolicyMapping
+				cache.iamSTSPolicyMap.Store(name, newPolicyMapping)
 			} else {
-				cache.iamUserPolicyMap[name] = newPolicyMapping
+				cache.iamUserPolicyMap.Store(name, newPolicyMapping)
 			}
 		} else {
-			cache.iamGroupPolicyMap[name] = newPolicyMapping
+			cache.iamGroupPolicyMap.Store(name, newPolicyMapping)
 		}
 	}
 
@@ -1079,12 +1137,12 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 		}
 		if !isGroup {
 			if userType == stsUser {
-				delete(cache.iamSTSPolicyMap, name)
+				cache.iamSTSPolicyMap.Delete(name)
 			} else {
-				delete(cache.iamUserPolicyMap, name)
+				cache.iamUserPolicyMap.Delete(name)
 			}
 		} else {
-			delete(cache.iamGroupPolicyMap, name)
+			cache.iamGroupPolicyMap.Delete(name)
 		}
 		cache.updatedAt = time.Now()
 		return cache.updatedAt, nil
@@ -1103,12 +1161,12 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 	}
 	if !isGroup {
 		if userType == stsUser {
-			cache.iamSTSPolicyMap[name] = mp
+			cache.iamSTSPolicyMap.Store(name, mp)
 		} else {
-			cache.iamUserPolicyMap[name] = mp
+			cache.iamUserPolicyMap.Store(name, mp)
 		}
 	} else {
-		cache.iamGroupPolicyMap[name] = mp
+		cache.iamGroupPolicyMap.Store(name, mp)
 	}
 	cache.updatedAt = time.Now()
 	return mp.UpdatedAt, nil
@@ -1132,33 +1190,35 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
 		delete(cache.iamPolicyDocsMap, policy)
 
 		// update user policy map
-		for u, mp := range cache.iamUserPolicyMap {
+		cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
 			pset := mp.policySet()
 			if !pset.Contains(policy) {
-				continue
+				return true
 			}
 			if store.getUsersSysType() == MinIOUsersSysType {
 				_, ok := cache.iamUsersMap[u]
 				if !ok {
 					// happens when account is deleted or
 					// expired.
-					delete(cache.iamUserPolicyMap, u)
-					continue
+					cache.iamUserPolicyMap.Delete(u)
+					return true
 				}
 			}
 			pset.Remove(policy)
-			cache.iamUserPolicyMap[u] = newMappedPolicy(strings.Join(pset.ToSlice(), ","))
-		}
+			cache.iamUserPolicyMap.Store(u, newMappedPolicy(strings.Join(pset.ToSlice(), ",")))
+			return true
+		})
 
 		// update group policy map
-		for g, mp := range cache.iamGroupPolicyMap {
+		cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool {
 			pset := mp.policySet()
 			if !pset.Contains(policy) {
-				continue
+				return true
 			}
 			pset.Remove(policy)
-			cache.iamGroupPolicyMap[g] = newMappedPolicy(strings.Join(pset.ToSlice(), ","))
-		}
+			cache.iamGroupPolicyMap.Store(g, newMappedPolicy(strings.Join(pset.ToSlice(), ",")))
+			return true
+		})
 
 		cache.updatedAt = time.Now()
 		return nil
@@ -1184,26 +1244,28 @@ func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string, isFro
 		// we do allow deletion.
 		users := []string{}
 		groups := []string{}
-		for u, mp := range cache.iamUserPolicyMap {
+		cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
 			pset := mp.policySet()
 			if store.getUsersSysType() == MinIOUsersSysType {
 				if _, ok := cache.iamUsersMap[u]; !ok {
 					// This case can happen when a temporary account is
 					// deleted or expired - remove it from userPolicyMap.
-					delete(cache.iamUserPolicyMap, u)
-					continue
+					cache.iamUserPolicyMap.Delete(u)
+					return true
 				}
 			}
 			if pset.Contains(policy) {
 				users = append(users, u)
 			}
-		}
-		for g, mp := range cache.iamGroupPolicyMap {
+			return true
+		})
+		cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool {
 			pset := mp.policySet()
 			if pset.Contains(policy) {
 				groups = append(groups, g)
 			}
-		}
+			return true
+		})
 		if len(users) != 0 || len(groups) != 0 {
 			return errPolicyInUse
 		}
@@ -1420,11 +1482,11 @@ func (store *IAMStoreSys) GetBucketUsers(bucket string) (map[string]madmin.UserI
 			continue
 		}
 		var policies []string
-		mp, ok := cache.iamUserPolicyMap[k]
+		mp, ok := cache.iamUserPolicyMap.Load(k)
 		if ok {
 			policies = append(policies, mp.Policies)
 			for _, group := range cache.iamUserGroupMemberships[k].ToSlice() {
-				if nmp, ok := cache.iamGroupPolicyMap[group]; ok {
+				if nmp, ok := cache.iamGroupPolicyMap.Load(group); ok {
 					policies = append(policies, nmp.Policies)
 				}
 			}
@@ -1459,8 +1521,9 @@ func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo {
 		if v.IsTemp() || v.IsServiceAccount() {
 			continue
 		}
+		pl, _ := cache.iamUserPolicyMap.Load(k)
 		result[k] = madmin.UserInfo{
-			PolicyName: cache.iamUserPolicyMap[k].Policies,
+			PolicyName: pl.Policies,
 			Status: func() madmin.AccountStatus {
 				if v.IsValid() {
 					return madmin.AccountEnabled
@@ -1468,7 +1531,7 @@ func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo {
 				return madmin.AccountDisabled
 			}(),
 			MemberOf:  cache.iamUserGroupMemberships[k].ToSlice(),
-			UpdatedAt: cache.iamUserPolicyMap[k].UpdatedAt,
+			UpdatedAt: pl.UpdatedAt,
 		}
 	}
 
@@ -1481,12 +1544,14 @@ func (store *IAMStoreSys) GetUsersWithMappedPolicies() map[string]string {
 	defer store.runlock()
 
 	result := make(map[string]string)
-	for k, v := range cache.iamUserPolicyMap {
+	cache.iamUserPolicyMap.Range(func(k string, v MappedPolicy) bool {
 		result[k] = v.Policies
-	}
-	for k, v := range cache.iamSTSPolicyMap {
+		return true
+	})
+	cache.iamSTSPolicyMap.Range(func(k string, v MappedPolicy) bool {
 		result[k] = v.Policies
-	}
+		return true
+	})
 	return result
 }
 
@@ -1515,14 +1580,14 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 				break
 			}
 		}
-		mappedPolicy, ok := cache.iamUserPolicyMap[name]
+		mappedPolicy, ok := cache.iamUserPolicyMap.Load(name)
 		if !ok {
-			mappedPolicy, ok = cache.iamSTSPolicyMap[name]
+			mappedPolicy, ok = cache.iamSTSPolicyMap.Load(name)
 		}
 		if !ok {
 			// Attempt to load parent user mapping for STS accounts
 			store.loadMappedPolicy(context.TODO(), name, stsUser, false, cache.iamSTSPolicyMap)
-			mappedPolicy, ok = cache.iamSTSPolicyMap[name]
+			mappedPolicy, ok = cache.iamSTSPolicyMap.Load(name)
 			if !ok {
 				return u, errNoSuchUser
 			}
@@ -1543,9 +1608,9 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 	if cred.IsTemp() || cred.IsServiceAccount() {
 		return u, errIAMActionNotAllowed
 	}
-
+	pl, _ := cache.iamUserPolicyMap.Load(name)
 	return madmin.UserInfo{
-		PolicyName: cache.iamUserPolicyMap[name].Policies,
+		PolicyName: pl.Policies,
 		Status: func() madmin.AccountStatus {
 			if cred.IsValid() {
 				return madmin.AccountEnabled
@@ -1553,7 +1618,7 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 			return madmin.AccountDisabled
 		}(),
 		MemberOf:  cache.iamUserGroupMemberships[name].ToSlice(),
-		UpdatedAt: cache.iamUserPolicyMap[name].UpdatedAt,
+		UpdatedAt: pl.UpdatedAt,
 	}, nil
 }
 
@@ -1566,7 +1631,7 @@ func (store *IAMStoreSys) PolicyMappingNotificationHandler(ctx context.Context,
 	cache := store.lock()
 	defer store.unlock()
 
-	var m map[string]MappedPolicy
+	var m *xsync.MapOf[string, MappedPolicy]
 	switch {
 	case isGroup:
 		m = cache.iamGroupPolicyMap
@@ -1577,7 +1642,7 @@ func (store *IAMStoreSys) PolicyMappingNotificationHandler(ctx context.Context,
 	if errors.Is(err, errNoSuchPolicy) {
 		// This means that the policy mapping was deleted, so we update
 		// the cache.
-		delete(m, userOrGroup)
+		m.Delete(userOrGroup)
 		cache.updatedAt = time.Now()
 
 		err = nil
@@ -1642,7 +1707,7 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		}
 
 		// 3. Delete any mapped policy
-		delete(cache.iamUserPolicyMap, accessKey)
+		cache.iamUserPolicyMap.Delete(accessKey)
 
 		return nil
 	}
@@ -1736,7 +1801,7 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 
 	// It is ok to ignore deletion error on the mapped policy
 	store.deleteMappedPolicy(ctx, accessKey, userType, false)
-	delete(cache.iamUserPolicyMap, accessKey)
+	cache.iamUserPolicyMap.Delete(accessKey)
 
 	err := store.deleteUserIdentity(ctx, accessKey, userType)
 	if err == errNoSuchUser {
@@ -1776,7 +1841,7 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 			return time.Time{}, err
 		}
 
-		cache.iamSTSPolicyMap[cred.ParentUser] = mp
+		cache.iamSTSPolicyMap.Store(cred.ParentUser, mp)
 	}
 
 	u := newUserIdentity(cred)
@@ -1813,7 +1878,7 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 		if usersToDelete.Contains(user) || usersToDelete.Contains(cred.ParentUser) {
 			// Delete this user account and its policy mapping
 			store.deleteMappedPolicy(ctx, user, userType, false)
-			delete(cache.iamUserPolicyMap, user)
+			cache.iamUserPolicyMap.Delete(user)
 
 			// we are only logging errors, not handling them.
 			err := store.deleteUserIdentity(ctx, user, userType)
@@ -1863,7 +1928,12 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 		if cred.IsServiceAccount() {
 			claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, cred.SecretKey)
 		} else if cred.IsTemp() {
-			claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
+			var secretKey string
+			secretKey, err = getTokenSigningKey()
+			if err != nil {
+				continue
+			}
+			claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, secretKey)
 		}
 
 		if err != nil {
@@ -1910,13 +1980,13 @@ func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string
 ) []madmin.UserPolicyEntities {
 	var r []madmin.UserPolicyEntities
 	usersSet := set.CreateStringSet(users...)
-	for user, mappedPolicy := range cache.iamUserPolicyMap {
+	cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
-			continue
+			return true
 		}
 
 		if !usersSet.IsEmpty() && !usersSet.Contains(user) {
-			continue
+			return true
 		}
 
 		ps := mappedPolicy.toSlice()
@@ -1925,17 +1995,18 @@ func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string
 			User:     user,
 			Policies: ps,
 		})
-	}
+		return true
+	})
 
-	stsMap := map[string]MappedPolicy{}
+	stsMap := xsync.NewMapOf[string, MappedPolicy]()
 	for _, user := range users {
 		// Attempt to load parent user mapping for STS accounts
 		store.loadMappedPolicy(context.TODO(), user, stsUser, false, stsMap)
 	}
 
-	for user, mappedPolicy := range stsMap {
+	stsMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
-			continue
+			return true
 		}
 
 		ps := mappedPolicy.toSlice()
@@ -1944,7 +2015,8 @@ func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string
 			User:     user,
 			Policies: ps,
 		})
-	}
+		return true
+	})
 
 	sort.Slice(r, func(i, j int) bool {
 		return r[i].User < r[j].User
@@ -1959,13 +2031,13 @@ func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []stri
 ) []madmin.GroupPolicyEntities {
 	var r []madmin.GroupPolicyEntities
 	groupsSet := set.CreateStringSet(groups...)
-	for group, mappedPolicy := range cache.iamGroupPolicyMap {
+	cache.iamGroupPolicyMap.Range(func(group string, mappedPolicy MappedPolicy) bool {
 		if groupPredicate != nil && !groupPredicate(group) {
-			continue
+			return true
 		}
 
 		if !groupsSet.IsEmpty() && !groupsSet.Contains(group) {
-			continue
+			return true
 		}
 
 		ps := mappedPolicy.toSlice()
@@ -1974,7 +2046,8 @@ func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []stri
 			Group:    group,
 			Policies: ps,
 		})
-	}
+		return true
+	})
 
 	sort.Slice(r, func(i, j int) bool {
 		return r[i].Group < r[j].Group
@@ -1990,9 +2063,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 	queryPolSet := set.CreateStringSet(policies...)
 
 	policyToUsersMap := make(map[string]set.StringSet)
-	for user, mappedPolicy := range cache.iamUserPolicyMap {
+	cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
-			continue
+			return true
 		}
 
 		commonPolicySet := mappedPolicy.policySet()
@@ -2008,7 +2081,8 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 				policyToUsersMap[policy] = s
 			}
 		}
-	}
+		return true
+	})
 
 	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
 		for item := range listIAMConfigItems(context.Background(), iamOS.objAPI, iamConfigPrefix+SlashSeparator+policyDBSTSUsersListKey) {
@@ -2037,9 +2111,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 	}
 
 	policyToGroupsMap := make(map[string]set.StringSet)
-	for group, mappedPolicy := range cache.iamGroupPolicyMap {
+	cache.iamGroupPolicyMap.Range(func(group string, mappedPolicy MappedPolicy) bool {
 		if groupPredicate != nil && !groupPredicate(group) {
-			continue
+			return true
 		}
 
 		commonPolicySet := mappedPolicy.policySet()
@@ -2055,7 +2129,8 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 				policyToGroupsMap[policy] = s
 			}
 		}
-	}
+		return true
+	})
 
 	m := make(map[string]madmin.PolicyEntities, len(policyToGroupsMap))
 	for policy, groups := range policyToGroupsMap {
@@ -2477,22 +2552,24 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 		store.loadUser(ctx, accessKey, regUser, cache.iamUsersMap)
 		if _, found = cache.iamUsersMap[accessKey]; found {
 			// load mapped policies
-			store.loadMappedPolicy(ctx, accessKey, regUser, false, cache.iamUserPolicyMap)
+			store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, cache.iamUserPolicyMap, 3)
 		}
 	}
 
 	// Check for service account
 	if !found {
 		store.loadUser(ctx, accessKey, svcUser, cache.iamUsersMap)
-		if svc, found := cache.iamUsersMap[accessKey]; found {
+		var svc UserIdentity
+		svc, found = cache.iamUsersMap[accessKey]
+		if found {
 			// Load parent user and mapped policies.
 			if store.getUsersSysType() == MinIOUsersSysType {
 				store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
-				store.loadMappedPolicy(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap)
+				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap, 3)
 			} else {
 				// In case of LDAP the parent user's policy mapping needs to be
 				// loaded into sts map
-				store.loadMappedPolicy(ctx, svc.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap)
+				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
 			}
 		}
 	}
@@ -2504,22 +2581,24 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 		store.loadUser(ctx, accessKey, stsUser, cache.iamSTSAccountsMap)
 		if stsUserCred, found = cache.iamSTSAccountsMap[accessKey]; found {
 			// Load mapped policy
-			store.loadMappedPolicy(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap)
+			store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
 			stsAccountFound = true
 		}
 	}
 
 	// Load any associated policy definitions
 	if !stsAccountFound {
-		for _, policy := range cache.iamUserPolicyMap[accessKey].toSlice() {
+		pols, _ := cache.iamUserPolicyMap.Load(accessKey)
+		for _, policy := range pols.toSlice() {
 			if _, found = cache.iamPolicyDocsMap[policy]; !found {
-				store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap)
+				store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
 			}
 		}
 	} else {
-		for _, policy := range cache.iamSTSPolicyMap[stsUserCred.Credentials.AccessKey].toSlice() {
+		pols, _ := cache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey)
+		for _, policy := range pols.toSlice() {
 			if _, found = cache.iamPolicyDocsMap[policy]; !found {
-				store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap)
+				store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
 			}
 		}
 	}
@@ -2528,8 +2607,12 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {
 	jwtClaims, err := auth.ExtractClaims(u.Credentials.SessionToken, u.Credentials.SecretKey)
 	if err != nil {
-		// Session tokens for STS creds will be generated with root secret
-		jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, globalActiveCred.SecretKey)
+		secretKey, err := getTokenSigningKey()
+		if err != nil {
+			return nil, err
+		}
+		// Session tokens for STS creds will be generated with root secret or site-replicator-0 secret
+		jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, secretKey)
 		if err != nil {
 			return nil, err
 		}

cmd/iam.go

@@ -201,6 +201,13 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 	atomic.StoreUint64(&sys.LastRefreshTimeUnixNano, uint64(loadStartTime.Add(loadDuration).UnixNano()))
 	atomic.AddUint64(&sys.TotalRefreshSuccesses, 1)
 
+	if !globalSiteReplicatorCred.IsValid() {
+		sa, _, err := sys.getServiceAccount(ctx, siteReplicatorSvcAcc)
+		if err == nil {
+			globalSiteReplicatorCred.Set(sa.Credentials)
+		}
+	}
+
 	if firstTime {
 		bootstrapTraceMsg(fmt.Sprintf("globalIAMSys.Load(): (duration: %s)", loadDuration))
 	}
@@ -1394,7 +1401,12 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 					jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
 				}
 			} else {
-				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+				var secretKey string
+				secretKey, err = getTokenSigningKey()
+				if err != nil {
+					continue
+				}
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, secretKey)
 			}
 			if err != nil {
 				// skip this cred - session token seems invalid

cmd/ilm-config.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"sync"
+
+	"github.com/minio/minio/internal/config/ilm"
+)
+
+var globalILMConfig = ilmConfig{
+	cfg: ilm.Config{
+		ExpirationWorkers: 100,
+		TransitionWorkers: 100,
+	},
+}
+
+type ilmConfig struct {
+	mu  sync.RWMutex
+	cfg ilm.Config
+}
+
+func (c *ilmConfig) getExpirationWorkers() int {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.cfg.ExpirationWorkers
+}
+
+func (c *ilmConfig) getTransitionWorkers() int {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.cfg.TransitionWorkers
+}
+
+func (c *ilmConfig) update(cfg ilm.Config) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.cfg = cfg
+}

cmd/is-dir-empty_linux.go

@@ -0,0 +1,46 @@
+//go:build linux && !appengine
+// +build linux,!appengine
+
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"syscall"
+)
+
+// Returns true if no error and there is no object or prefix inside this directory
+func isDirEmpty(dirname string) bool {
+	var stat syscall.Stat_t
+	if err := syscall.Stat(dirname, &stat); err != nil {
+		return false
+	}
+	if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR && stat.Nlink == 2 {
+		return true
+	}
+	// On filesystems such as btrfs, nfs this is not true, so fallback
+	// to performing readdir() instead.
+	if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR && stat.Nlink < 2 {
+		entries, err := readDirN(dirname, 1)
+		if err != nil {
+			return false
+		}
+		return len(entries) == 0
+	}
+	return false
+}

cmd/is-dir-empty_other.go

@@ -1,3 +1,6 @@
+//go:build !linux
+// +build !linux
+
 // Copyright (c) 2015-2021 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
@@ -17,40 +20,11 @@
 
 package cmd
 
-import (
-	"context"
-	"fmt"
-
-	"github.com/minio/minio/internal/logger"
-)
-
-type tierMemJournal struct {
-	entries chan jentry
-}
-
-func newTierMemJournal(nevents int) *tierMemJournal {
-	return &tierMemJournal{
-		entries: make(chan jentry, nevents),
+// isDirEmpty - returns true if there is no error and no object and prefix inside this directory
+func isDirEmpty(dirname string) bool {
+	entries, err := readDirN(dirname, 1)
+	if err != nil {
+		return false
 	}
-}
-
-func (j *tierMemJournal) processEntries(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case entry := <-j.entries:
-			logger.LogIf(ctx, deleteObjectFromRemoteTier(ctx, entry.ObjName, entry.VersionID, entry.TierName))
-		}
-	}
-}
-
-func (j *tierMemJournal) AddEntry(je jentry) error {
-	select {
-	case j.entries <- je:
-	default:
-		return fmt.Errorf("failed to remove tiered content at %s with version %s from tier %s, will be retried later.",
-			je.ObjName, je.VersionID, je.TierName)
-	}
-	return nil
+	return len(entries) == 0
 }

cmd/kms-handlers.go

@@ -25,7 +25,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/kes-go"
+	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -431,10 +431,6 @@ func (a kmsAPIHandlers) KMSDescribePolicyHandler(w http.ResponseWriter, r *http.
 	writeSuccessResponseJSON(w, p)
 }
 
-type assignPolicyRequest struct {
-	Identity string
-}
-
 // KMSAssignPolicyHandler - POST /minio/kms/v1/policy/assign?policy=<policy>
 func (a kmsAPIHandlers) KMSAssignPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSAssignPolicy")
@@ -449,22 +445,8 @@ func (a kmsAPIHandlers) KMSAssignPolicyHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-	manager, ok := GlobalKMS.(kms.PolicyManager)
-	if !ok {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-	var request assignPolicyRequest
-	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	err := manager.AssignPolicy(ctx, r.Form.Get("policy"), request.Identity)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	writeSuccessResponseHeadersOnly(w)
+	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	return
 }
 
 // KMSDeletePolicyHandler - DELETE /minio/kms/v1/policy/delete?policy=<policy>
@@ -481,17 +463,8 @@ func (a kmsAPIHandlers) KMSDeletePolicyHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-
-	manager, ok := GlobalKMS.(kms.PolicyManager)
-	if !ok {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-	if err := manager.DeletePolicy(ctx, r.Form.Get("policy")); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	writeSuccessResponseHeadersOnly(w)
+	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	return
 }
 
 // KMSListPoliciesHandler - GET /minio/kms/v1/policy/list?pattern=<pattern>
@@ -668,17 +641,8 @@ func (a kmsAPIHandlers) KMSDeleteIdentityHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-	manager, ok := GlobalKMS.(kms.IdentityManager)
-	if !ok {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
-	if err := manager.DeleteIdentity(ctx, r.Form.Get("policy")); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	writeSuccessResponseHeadersOnly(w)
+	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	return
 }
 
 // KMSListIdentitiesHandler - GET /minio/kms/v1/identity/list?pattern=<pattern>

cmd/local-locker_gen.go

@@ -21,6 +21,8 @@ func (z *localLockMap) DecodeMsg(dc *msgp.Reader) (err error) {
 			delete((*z), key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0004 > 0 {
 		zb0004--
 		var zb0001 string
@@ -115,6 +117,8 @@ func (z *localLockMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			delete((*z), key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0004 > 0 {
 		var zb0001 string
 		var zb0002 []lockRequesterInfo

cmd/main.go

@@ -25,7 +25,9 @@ import (
 	"runtime"
 	"runtime/debug"
 	"sort"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/minio/cli"
 	"github.com/minio/minio/internal/color"
@@ -169,8 +171,9 @@ func newApp(name string) *cli.App {
 }
 
 func startupBanner(banner io.Writer) {
+	CopyrightYear = strconv.Itoa(time.Now().Year())
 	fmt.Fprintln(banner, color.Blue("Copyright:")+color.Bold(" 2015-%s MinIO, Inc.", CopyrightYear))
-	fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
+	fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" "+MinioLicense))
 	fmt.Fprintln(banner, color.Blue("Version:")+color.Bold(" %s (%s %s/%s)", ReleaseTag, runtime.Version(), runtime.GOOS, runtime.GOARCH))
 }
 

cmd/metacache-entries.go

@@ -473,6 +473,8 @@ type metaCacheEntriesSorted struct {
 	listID string
 	// Reuse buffers
 	reuse bool
+	// Contain the last skipped object after an ILM expiry evaluation
+	lastSkippedEntry string
 }
 
 // shallowClone will create a shallow clone of the array objects,
@@ -745,10 +747,10 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
 
 		// Merge any unmerged
 		if len(toMerge) > 0 {
-			versions := make([]xlMetaV2ShallowVersion, 0, len(toMerge)+1)
+			versions := make([][]xlMetaV2ShallowVersion, 0, len(toMerge)+1)
 			xl, err := best.xlmeta()
 			if err == nil {
-				versions = append(versions, xl.versions...)
+				versions = append(versions, xl.versions)
 			}
 			for _, idx := range toMerge {
 				other := top[idx]
@@ -776,11 +778,12 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
 						return err
 					}
 				}
-				versions = append(versions, xl2.versions...)
+				versions = append(versions, xl2.versions)
 			}
+
 			if xl != nil && len(versions) > 0 {
 				// Merge all versions. 'strict' doesn't matter since we only need one.
-				xl.versions = mergeXLV2Versions(readQuorum, true, 0, versions)
+				xl.versions = mergeXLV2Versions(readQuorum, true, 0, versions...)
 				if meta, err := xl.AppendTo(metaDataPoolGet()); err == nil {
 					if best.reusable {
 						metaDataPoolPut(best.metadata)

cmd/metacache-server-pool.go

@@ -291,14 +291,6 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	}
 	mu.Unlock()
 
-	// Do lifecycle filtering.
-	if o.Lifecycle != nil || o.Replication.Config != nil {
-		filterIn := make(chan metaCacheEntry, 10)
-		go applyBucketActions(ctx, o, filterIn, results)
-		// Replace results.
-		results = filterIn
-	}
-
 	// Gather results to a single channel.
 	// Quorum is one since we are merging across sets.
 	err := mergeEntryChannels(ctx, inputs, results, 1)
@@ -339,84 +331,50 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	return nil
 }
 
-// applyBucketActions applies lifecycle and replication actions on the listing
-// It will filter out objects if the most recent version should be deleted by lifecycle.
-// Entries that failed replication will be queued if no lifecycle rules got applied.
-// out will be closed when there are no more results.
-// When 'in' is closed or the context is canceled the
-// function closes 'out' and exits.
-func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
-	defer xioutil.SafeClose(out)
+// triggerExpiryAndRepl applies lifecycle and replication actions on the listing
+// It returns true if the listing is non-versioned and the given object is expired.
+func triggerExpiryAndRepl(ctx context.Context, o listPathOptions, obj metaCacheEntry) (skip bool) {
+	versioned := o.Versioning != nil && o.Versioning.Versioned(obj.name)
 
-	for {
-		var obj metaCacheEntry
-		var ok bool
-		select {
-		case <-ctx.Done():
-			return
-		case obj, ok = <-in:
-			if !ok {
-				return
-			}
-		}
-
-		var skip bool
-
-		versioned := o.Versioning != nil && o.Versioning.Versioned(obj.name)
-
-		// skip latest object from listing only for regular
-		// listObjects calls, versioned based listing cannot
-		// filter out between versions 'obj' cannot be truncated
-		// in such a manner, so look for skipping an object only
-		// for regular ListObjects() call only.
-		if !o.Versioned {
-			fi, err := obj.fileInfo(o.Bucket)
-			if err != nil {
-				continue
-			}
-
-			objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
-			if o.Lifecycle != nil {
-				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo).Action
-				skip = act.Delete()
-				if act.DeleteRestored() {
-					// do not skip DeleteRestored* actions
-					skip = false
-				}
-			}
-		}
-
-		// Skip entry only if needed via ILM, skipping is never true for versioned listing.
-		if !skip {
-			select {
-			case <-ctx.Done():
-				return
-			case out <- obj:
-			}
-		}
-
-		fiv, err := obj.fileInfoVersions(o.Bucket)
+	// skip latest object from listing only for regular
+	// listObjects calls, versioned based listing cannot
+	// filter out between versions 'obj' cannot be truncated
+	// in such a manner, so look for skipping an object only
+	// for regular ListObjects() call only.
+	if !o.Versioned && !o.V1 {
+		fi, err := obj.fileInfo(o.Bucket)
 		if err != nil {
-			continue
+			return
 		}
-
-		// Expire all versions if needed, if not attempt to queue for replication.
-		for _, version := range fiv.Versions {
-			objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
-
-			if o.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo)
-				if evt.Action.Delete() {
-					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
-					if !evt.Action.DeleteRestored() {
-						continue
-					} // queue version for replication upon expired restored copies if needed.
-				}
-			}
-
-			queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication, 0)
+		objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
+		if o.Lifecycle != nil {
+			act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo).Action
+			skip = act.Delete() && !act.DeleteRestored()
 		}
 	}
+
+	fiv, err := obj.fileInfoVersions(o.Bucket)
+	if err != nil {
+		return
+	}
+
+	// Expire all versions if needed, if not attempt to queue for replication.
+	for _, version := range fiv.Versions {
+		objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
+
+		if o.Lifecycle != nil {
+			evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo)
+			if evt.Action.Delete() {
+				globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
+				if !evt.Action.DeleteRestored() {
+					continue
+				} // queue version for replication upon expired restored copies if needed.
+			}
+		}
+
+		queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication, 0)
+	}
+	return
 }
 
 func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) {

cmd/metacache-set.go

@@ -98,6 +98,8 @@ type listPathOptions struct {
 
 	// Versioned is this a ListObjectVersions call.
 	Versioned bool
+	// V1 listing type
+	V1 bool
 
 	// Versioning config is used for if the path
 	// has versioning enabled.
@@ -172,7 +174,8 @@ func (o *listPathOptions) debugln(data ...interface{}) {
 	}
 }
 
-// gatherResults will collect all results on the input channel and filter results according to the options.
+// gatherResults will collect all results on the input channel and filter results according
+// to the options or to the current bucket ILM expiry rules.
 // Caller should close the channel when done.
 // The returned function will return the results once there is enough or input is closed,
 // or the context is canceled.
@@ -214,6 +217,12 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() {
 				continue
 			}
+			if o.Lifecycle != nil || o.Replication.Config != nil {
+				if skipped := triggerExpiryAndRepl(ctx, *o, entry); skipped == true {
+					results.lastSkippedEntry = entry.name
+					continue
+				}
+			}
 			if o.Limit > 0 && results.len() >= o.Limit {
 				// We have enough and we have more.
 				// Do not return io.EOF

cmd/metacache-set_gen.go

@@ -114,6 +114,12 @@ func (z *listPathOptions) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Versioned")
 				return
 			}
+		case "V1":
+			z.V1, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "V1")
+				return
+			}
 		case "StopDiskAtLimit":
 			z.StopDiskAtLimit, err = dc.ReadBool()
 			if err != nil {
@@ -145,9 +151,9 @@ func (z *listPathOptions) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *listPathOptions) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 18
+	// map header, size 19
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x12, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x13, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -296,6 +302,16 @@ func (z *listPathOptions) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Versioned")
 		return
 	}
+	// write "V1"
+	err = en.Append(0xa2, 0x56, 0x31)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.V1)
+	if err != nil {
+		err = msgp.WrapError(err, "V1")
+		return
+	}
 	// write "StopDiskAtLimit"
 	err = en.Append(0xaf, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x69, 0x73, 0x6b, 0x41, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74)
 	if err != nil {
@@ -332,9 +348,9 @@ func (z *listPathOptions) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *listPathOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 18
+	// map header, size 19
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x12, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x13, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
@@ -378,6 +394,9 @@ func (z *listPathOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Versioned"
 	o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64)
 	o = msgp.AppendBool(o, z.Versioned)
+	// string "V1"
+	o = append(o, 0xa2, 0x56, 0x31)
+	o = msgp.AppendBool(o, z.V1)
 	// string "StopDiskAtLimit"
 	o = append(o, 0xaf, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x69, 0x73, 0x6b, 0x41, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74)
 	o = msgp.AppendBool(o, z.StopDiskAtLimit)
@@ -498,6 +517,12 @@ func (z *listPathOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Versioned")
 				return
 			}
+		case "V1":
+			z.V1, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "V1")
+				return
+			}
 		case "StopDiskAtLimit":
 			z.StopDiskAtLimit, bts, err = msgp.ReadBoolBytes(bts)
 			if err != nil {
@@ -530,6 +555,6 @@ func (z *listPathOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *listPathOptions) Msgsize() (s int) {
-	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 8 + msgp.StringPrefixSize + len(z.BaseDir) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 13 + msgp.StringPrefixSize + len(z.FilterPrefix) + 7 + msgp.StringPrefixSize + len(z.Marker) + 6 + msgp.IntSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 12 + msgp.BoolSize + 10 + msgp.BoolSize + 10 + msgp.StringPrefixSize + len(z.Separator) + 7 + msgp.BoolSize + 19 + msgp.BoolSize + 10 + msgp.BoolSize + 10 + msgp.BoolSize + 16 + msgp.BoolSize + 5 + msgp.IntSize + 4 + msgp.IntSize
+	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 8 + msgp.StringPrefixSize + len(z.BaseDir) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 13 + msgp.StringPrefixSize + len(z.FilterPrefix) + 7 + msgp.StringPrefixSize + len(z.Marker) + 6 + msgp.IntSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 12 + msgp.BoolSize + 10 + msgp.BoolSize + 10 + msgp.StringPrefixSize + len(z.Separator) + 7 + msgp.BoolSize + 19 + msgp.BoolSize + 10 + msgp.BoolSize + 10 + msgp.BoolSize + 3 + msgp.BoolSize + 16 + msgp.BoolSize + 5 + msgp.IntSize + 4 + msgp.IntSize
 	return
 }

cmd/metrics-realtime.go

@@ -130,12 +130,6 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM
 	}
 
 	metrics := make(map[string]madmin.DiskMetric)
-
-	procStats, procErr := disk.GetAllDrivesIOStats()
-	if procErr != nil {
-		return metrics
-	}
-
 	storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
 	for _, d := range storageInfo.Disks {
 		if len(disks) != 0 {
@@ -170,9 +164,8 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM
 			}
 		}
 
-		// get disk
-		if procErr == nil {
-			st := procStats[disk.DevID{Major: d.Major, Minor: d.Minor}]
+		st, err := disk.GetDriveStats(d.Major, d.Minor)
+		if err == nil {
 			dm.IOStats = madmin.DiskIOStats{
 				ReadIOs:        st.ReadIOs,
 				ReadMerges:     st.ReadMerges,

cmd/metrics-resource.go

@@ -27,7 +27,6 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/shirou/gopsutil/v3/host"
 )
 
 const (
@@ -82,12 +81,12 @@ var (
 	resourceMetricsMapMu sync.RWMutex
 	// resourceMetricsHelpMap maps metric name to its help string
 	resourceMetricsHelpMap map[MetricName]string
-	resourceMetricsGroups  []*MetricsGroup
+	resourceMetricsGroups  []*MetricsGroupV2
 	// initial values for drives (at the time  of server startup)
 	// used for calculating avg values for drive metrics
-	initialDriveStats   map[string]madmin.DiskIOStats
-	initialDriveStatsMu sync.RWMutex
-	initialUptime       uint64
+	latestDriveStats      map[string]madmin.DiskIOStats
+	latestDriveStatsMu    sync.RWMutex
+	lastDriveStatsRefresh time.Time
 )
 
 // PeerResourceMetrics represents the resource metrics
@@ -147,7 +146,7 @@ func init() {
 		writesKBPerSec:    "Kilobytes written per second on a drive",
 		readsAwait:        "Average time for read requests to be served on a drive",
 		writesAwait:       "Average time for write requests to be served on a drive",
-		percUtil:          "Percentage of time the disk was busy since uptime",
+		percUtil:          "Percentage of time the disk was busy",
 		usedBytes:         "Used bytes on a drive",
 		totalBytes:        "Total bytes on a drive",
 		usedInodes:        "Total inodes used on a drive",
@@ -165,7 +164,7 @@ func init() {
 		cpuLoad5Perc:      "CPU load average 5min (percentage)",
 		cpuLoad15Perc:     "CPU load average 15min (percentage)",
 	}
-	resourceMetricsGroups = []*MetricsGroup{
+	resourceMetricsGroups = []*MetricsGroupV2{
 		getResourceMetrics(),
 	}
 
@@ -219,35 +218,32 @@ func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64,
 	resourceMetricsMap[subSys] = subsysMetrics
 }
 
-// updateDriveIOStats - Updates the drive IO stats by calculating the difference between the current
-// and initial values. We cannot rely on host.Uptime here as it will not work in k8s environments, where
-// it will return the pod's uptime but the disk metrics are always from the host (/proc/diskstats)
-func updateDriveIOStats(currentStats madmin.DiskIOStats, initialStats madmin.DiskIOStats, labels map[string]string) {
+// updateDriveIOStats - Updates the drive IO stats by calculating the difference between the current and latest updated values.
+func updateDriveIOStats(currentStats madmin.DiskIOStats, latestStats madmin.DiskIOStats, labels map[string]string) {
 	sectorSize := uint64(512)
 	kib := float64(1 << 10)
-	uptime, _ := host.Uptime()
-	uptimeDiff := float64(uptime - initialUptime)
-	if uptimeDiff == 0 {
+	diffInSeconds := time.Now().UTC().Sub(lastDriveStatsRefresh).Seconds()
+	if diffInSeconds == 0 {
 		// too soon to update the stats
 		return
 	}
 	diffStats := madmin.DiskIOStats{
-		ReadIOs:      currentStats.ReadIOs - initialStats.ReadIOs,
-		WriteIOs:     currentStats.WriteIOs - initialStats.WriteIOs,
-		ReadTicks:    currentStats.ReadTicks - initialStats.ReadTicks,
-		WriteTicks:   currentStats.WriteTicks - initialStats.WriteTicks,
-		TotalTicks:   currentStats.TotalTicks - initialStats.TotalTicks,
-		ReadSectors:  currentStats.ReadSectors - initialStats.ReadSectors,
-		WriteSectors: currentStats.WriteSectors - initialStats.WriteSectors,
+		ReadIOs:      currentStats.ReadIOs - latestStats.ReadIOs,
+		WriteIOs:     currentStats.WriteIOs - latestStats.WriteIOs,
+		ReadTicks:    currentStats.ReadTicks - latestStats.ReadTicks,
+		WriteTicks:   currentStats.WriteTicks - latestStats.WriteTicks,
+		TotalTicks:   currentStats.TotalTicks - latestStats.TotalTicks,
+		ReadSectors:  currentStats.ReadSectors - latestStats.ReadSectors,
+		WriteSectors: currentStats.WriteSectors - latestStats.WriteSectors,
 	}
 
-	updateResourceMetrics(driveSubsystem, readsPerSec, float64(diffStats.ReadIOs)/uptimeDiff, labels, false)
+	updateResourceMetrics(driveSubsystem, readsPerSec, float64(diffStats.ReadIOs)/diffInSeconds, labels, false)
 	readKib := float64(diffStats.ReadSectors*sectorSize) / kib
-	updateResourceMetrics(driveSubsystem, readsKBPerSec, readKib/uptimeDiff, labels, false)
+	updateResourceMetrics(driveSubsystem, readsKBPerSec, readKib/diffInSeconds, labels, false)
 
-	updateResourceMetrics(driveSubsystem, writesPerSec, float64(diffStats.WriteIOs)/uptimeDiff, labels, false)
+	updateResourceMetrics(driveSubsystem, writesPerSec, float64(diffStats.WriteIOs)/diffInSeconds, labels, false)
 	writeKib := float64(diffStats.WriteSectors*sectorSize) / kib
-	updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKib/uptimeDiff, labels, false)
+	updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKib/diffInSeconds, labels, false)
 
 	rdAwait := 0.0
 	if diffStats.ReadIOs > 0 {
@@ -260,18 +256,23 @@ func updateDriveIOStats(currentStats madmin.DiskIOStats, initialStats madmin.Dis
 		wrAwait = float64(diffStats.WriteTicks) / float64(diffStats.WriteIOs)
 	}
 	updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false)
-	updateResourceMetrics(driveSubsystem, percUtil, float64(diffStats.TotalTicks)/(uptimeDiff*10), labels, false)
+	updateResourceMetrics(driveSubsystem, percUtil, float64(diffStats.TotalTicks)/(diffInSeconds*10), labels, false)
 }
 
 func collectDriveMetrics(m madmin.RealtimeMetrics) {
+	latestDriveStatsMu.Lock()
 	for d, dm := range m.ByDisk {
 		labels := map[string]string{"drive": d}
-		initialStats, ok := initialDriveStats[d]
+		latestStats, ok := latestDriveStats[d]
 		if !ok {
+			latestDriveStats[d] = dm.IOStats
 			continue
 		}
-		updateDriveIOStats(dm.IOStats, initialStats, labels)
+		updateDriveIOStats(dm.IOStats, latestStats, labels)
+		latestDriveStats[d] = dm.IOStats
 	}
+	lastDriveStatsRefresh = time.Now().UTC()
+	latestDriveStatsMu.Unlock()
 
 	globalLocalDrivesMu.RLock()
 	localDrives := cloneDrives(globalLocalDrives)
@@ -361,29 +362,25 @@ func collectLocalResourceMetrics() {
 	collectDriveMetrics(m)
 }
 
-// populateInitialValues - populates the initial values
-// for drive stats and host uptime
-func populateInitialValues() {
-	initialDriveStatsMu.Lock()
-
+func initLatestValues() {
 	m := collectLocalMetrics(madmin.MetricsDisk, collectMetricsOpts{
 		hosts: map[string]struct{}{
 			globalLocalNodeName: {},
 		},
 	})
 
-	initialDriveStats = map[string]madmin.DiskIOStats{}
+	latestDriveStatsMu.Lock()
+	latestDriveStats = map[string]madmin.DiskIOStats{}
 	for d, dm := range m.ByDisk {
-		initialDriveStats[d] = dm.IOStats
+		latestDriveStats[d] = dm.IOStats
 	}
-
-	initialUptime, _ = host.Uptime()
-	initialDriveStatsMu.Unlock()
+	lastDriveStatsRefresh = time.Now().UTC()
+	latestDriveStatsMu.Unlock()
 }
 
 // startResourceMetricsCollection - starts the job for collecting resource metrics
 func startResourceMetricsCollection() {
-	populateInitialValues()
+	initLatestValues()
 
 	resourceMetricsMapMu.Lock()
 	resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{}
@@ -408,7 +405,7 @@ func startResourceMetricsCollection() {
 
 // minioResourceCollector is the Collector for resource metrics
 type minioResourceCollector struct {
-	metricsGroups []*MetricsGroup
+	metricsGroups []*MetricsGroupV2
 	desc          *prometheus.Desc
 }
 
@@ -420,7 +417,7 @@ func (c *minioResourceCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect is called by the Prometheus registry when collecting metrics.
 func (c *minioResourceCollector) Collect(out chan<- prometheus.Metric) {
 	var wg sync.WaitGroup
-	publish := func(in <-chan Metric) {
+	publish := func(in <-chan MetricV2) {
 		defer wg.Done()
 		for metric := range in {
 			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
@@ -439,18 +436,18 @@ func (c *minioResourceCollector) Collect(out chan<- prometheus.Metric) {
 // and returns reference of minio resource Collector
 // It creates the Prometheus Description which is used
 // to define Metric and  help string
-func newMinioResourceCollector(metricsGroups []*MetricsGroup) *minioResourceCollector {
+func newMinioResourceCollector(metricsGroups []*MetricsGroupV2) *minioResourceCollector {
 	return &minioResourceCollector{
 		metricsGroups: metricsGroups,
 		desc:          prometheus.NewDesc("minio_resource_stats", "Resource statistics exposed by MinIO server", nil, nil),
 	}
 }
 
-func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAvgMax bool) []Metric {
+func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAvgMax bool) []MetricV2 {
 	help := resourceMetricsHelpMap[rm.Name]
 	name := rm.Name
-	metrics := make([]Metric, 0, 3)
-	metrics = append(metrics, Metric{
+	metrics := make([]MetricV2, 0, 3)
+	metrics = append(metrics, MetricV2{
 		Description:    getResourceMetricDescription(subSys, name, help),
 		Value:          rm.Current,
 		VariableLabels: cloneMSS(rm.Labels),
@@ -459,7 +456,7 @@ func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAv
 	if requireAvgMax {
 		avgName := MetricName(fmt.Sprintf("%s_avg", name))
 		avgHelp := fmt.Sprintf("%s (avg)", help)
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getResourceMetricDescription(subSys, avgName, avgHelp),
 			Value:          math.Round(rm.Avg*100) / 100,
 			VariableLabels: cloneMSS(rm.Labels),
@@ -467,7 +464,7 @@ func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAv
 
 		maxName := MetricName(fmt.Sprintf("%s_max", name))
 		maxHelp := fmt.Sprintf("%s (max)", help)
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getResourceMetricDescription(subSys, maxName, maxHelp),
 			Value:          rm.Max,
 			VariableLabels: cloneMSS(rm.Labels),
@@ -487,12 +484,12 @@ func getResourceMetricDescription(subSys MetricSubsystem, name MetricName, help
 	}
 }
 
-func getResourceMetrics() *MetricsGroup {
-	mg := &MetricsGroup{
+func getResourceMetrics() *MetricsGroupV2 {
+	mg := &MetricsGroupV2{
 		cacheInterval: resourceMetricsCacheInterval,
 	}
-	mg.RegisterRead(func(ctx context.Context) []Metric {
-		metrics := []Metric{}
+	mg.RegisterRead(func(ctx context.Context) []MetricV2 {
+		metrics := []MetricV2{}
 
 		subSystems := []MetricSubsystem{interfaceSubsystem, memSubsystem, driveSubsystem, cpuSubsystem}
 		resourceMetricsMapMu.RLock()

cmd/metrics-router.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"net/http"
 	"strings"
 
 	"github.com/minio/mux"
@@ -30,6 +31,9 @@ const (
 	prometheusMetricsV2BucketPath   = "/v2/metrics/bucket"
 	prometheusMetricsV2NodePath     = "/v2/metrics/node"
 	prometheusMetricsV2ResourcePath = "/v2/metrics/resource"
+
+	// Metrics v3 endpoints
+	metricsV3Path = "/metrics/v3"
 )
 
 // Standard env prometheus auth type
@@ -48,10 +52,10 @@ const (
 func registerMetricsRouter(router *mux.Router) {
 	// metrics router
 	metricsRouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	authType := strings.ToLower(env.Get(EnvPrometheusAuthType, string(prometheusJWT)))
+	authType := prometheusAuthType(strings.ToLower(env.Get(EnvPrometheusAuthType, string(prometheusJWT))))
 
 	auth := AuthMiddleware
-	if prometheusAuthType(authType) == prometheusPublic {
+	if authType == prometheusPublic {
 		auth = NoAuthMiddleware
 	}
 	metricsRouter.Handle(prometheusMetricsPathLegacy, auth(metricsHandler()))
@@ -59,4 +63,11 @@ func registerMetricsRouter(router *mux.Router) {
 	metricsRouter.Handle(prometheusMetricsV2BucketPath, auth(metricsBucketHandler()))
 	metricsRouter.Handle(prometheusMetricsV2NodePath, auth(metricsNodeHandler()))
 	metricsRouter.Handle(prometheusMetricsV2ResourcePath, auth(metricsResourceHandler()))
+
+	// Metrics v3!
+	metricsV3Server := newMetricsV3Server(authType)
+
+	// Register metrics v3 handler. It also accepts an optional query
+	// parameter `?list` - see handler for details.
+	metricsRouter.Methods(http.MethodGet).Path(metricsV3Path + "{pathComps:.*}").Handler(metricsV3Server)
 }

cmd/metrics-v2_gen.go

@@ -7,7 +7,223 @@ import (
 )
 
 // MarshalMsg implements msgp.Marshaler
-func (z *Metric) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *MetricDescription) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 5
+	// string "Namespace"
+	o = append(o, 0x85, 0xa9, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65)
+	o = msgp.AppendString(o, string(z.Namespace))
+	// string "Subsystem"
+	o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d)
+	o = msgp.AppendString(o, string(z.Subsystem))
+	// string "Name"
+	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, string(z.Name))
+	// string "Help"
+	o = append(o, 0xa4, 0x48, 0x65, 0x6c, 0x70)
+	o = msgp.AppendString(o, z.Help)
+	// string "Type"
+	o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = msgp.AppendString(o, string(z.Type))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MetricDescription) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Namespace":
+			{
+				var zb0002 string
+				zb0002, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Namespace")
+					return
+				}
+				z.Namespace = MetricNamespace(zb0002)
+			}
+		case "Subsystem":
+			{
+				var zb0003 string
+				zb0003, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Subsystem")
+					return
+				}
+				z.Subsystem = MetricSubsystem(zb0003)
+			}
+		case "Name":
+			{
+				var zb0004 string
+				zb0004, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Name")
+					return
+				}
+				z.Name = MetricName(zb0004)
+			}
+		case "Help":
+			z.Help, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Help")
+				return
+			}
+		case "Type":
+			{
+				var zb0005 string
+				zb0005, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Type")
+					return
+				}
+				z.Type = MetricTypeV2(zb0005)
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *MetricDescription) Msgsize() (s int) {
+	s = 1 + 10 + msgp.StringPrefixSize + len(string(z.Namespace)) + 10 + msgp.StringPrefixSize + len(string(z.Subsystem)) + 5 + msgp.StringPrefixSize + len(string(z.Name)) + 5 + msgp.StringPrefixSize + len(z.Help) + 5 + msgp.StringPrefixSize + len(string(z.Type))
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z MetricName) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendString(o, string(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MetricName) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 string
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = MetricName(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z MetricName) Msgsize() (s int) {
+	s = msgp.StringPrefixSize + len(string(z))
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z MetricNamespace) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendString(o, string(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MetricNamespace) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 string
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = MetricNamespace(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z MetricNamespace) Msgsize() (s int) {
+	s = msgp.StringPrefixSize + len(string(z))
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z MetricSubsystem) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendString(o, string(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MetricSubsystem) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 string
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = MetricSubsystem(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z MetricSubsystem) Msgsize() (s int) {
+	s = msgp.StringPrefixSize + len(string(z))
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z MetricTypeV2) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendString(o, string(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MetricTypeV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 string
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = MetricTypeV2(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z MetricTypeV2) Msgsize() (s int) {
+	s = msgp.StringPrefixSize + len(string(z))
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *MetricV2) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 6
 	// string "Description"
@@ -48,7 +264,7 @@ func (z *Metric) MarshalMsg(b []byte) (o []byte, err error) {
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (z *Metric) UnmarshalMsg(bts []byte) (o []byte, err error) {
+func (z *MetricV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
 	var zb0001 uint32
@@ -186,7 +402,7 @@ func (z *Metric) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *Metric) Msgsize() (s int) {
+func (z *MetricV2) Msgsize() (s int) {
 	s = 1 + 12 + z.Description.Msgsize() + 13 + msgp.MapHeaderSize
 	if z.StaticLabels != nil {
 		for za0001, za0002 := range z.StaticLabels {
@@ -211,287 +427,6 @@ func (z *Metric) Msgsize() (s int) {
 	return
 }
 
-// MarshalMsg implements msgp.Marshaler
-func (z *MetricDescription) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
-	// string "Namespace"
-	o = append(o, 0x85, 0xa9, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65)
-	o = msgp.AppendString(o, string(z.Namespace))
-	// string "Subsystem"
-	o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d)
-	o = msgp.AppendString(o, string(z.Subsystem))
-	// string "Name"
-	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
-	o = msgp.AppendString(o, string(z.Name))
-	// string "Help"
-	o = append(o, 0xa4, 0x48, 0x65, 0x6c, 0x70)
-	o = msgp.AppendString(o, z.Help)
-	// string "Type"
-	o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
-	o = msgp.AppendString(o, string(z.Type))
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *MetricDescription) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Namespace":
-			{
-				var zb0002 string
-				zb0002, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Namespace")
-					return
-				}
-				z.Namespace = MetricNamespace(zb0002)
-			}
-		case "Subsystem":
-			{
-				var zb0003 string
-				zb0003, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Subsystem")
-					return
-				}
-				z.Subsystem = MetricSubsystem(zb0003)
-			}
-		case "Name":
-			{
-				var zb0004 string
-				zb0004, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Name")
-					return
-				}
-				z.Name = MetricName(zb0004)
-			}
-		case "Help":
-			z.Help, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Help")
-				return
-			}
-		case "Type":
-			{
-				var zb0005 string
-				zb0005, bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Type")
-					return
-				}
-				z.Type = MetricType(zb0005)
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *MetricDescription) Msgsize() (s int) {
-	s = 1 + 10 + msgp.StringPrefixSize + len(string(z.Namespace)) + 10 + msgp.StringPrefixSize + len(string(z.Subsystem)) + 5 + msgp.StringPrefixSize + len(string(z.Name)) + 5 + msgp.StringPrefixSize + len(z.Help) + 5 + msgp.StringPrefixSize + len(string(z.Type))
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z MetricName) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendString(o, string(z))
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *MetricName) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	{
-		var zb0001 string
-		zb0001, bts, err = msgp.ReadStringBytes(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		(*z) = MetricName(zb0001)
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z MetricName) Msgsize() (s int) {
-	s = msgp.StringPrefixSize + len(string(z))
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z MetricNamespace) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendString(o, string(z))
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *MetricNamespace) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	{
-		var zb0001 string
-		zb0001, bts, err = msgp.ReadStringBytes(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		(*z) = MetricNamespace(zb0001)
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z MetricNamespace) Msgsize() (s int) {
-	s = msgp.StringPrefixSize + len(string(z))
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z MetricSubsystem) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendString(o, string(z))
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *MetricSubsystem) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	{
-		var zb0001 string
-		zb0001, bts, err = msgp.ReadStringBytes(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		(*z) = MetricSubsystem(zb0001)
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z MetricSubsystem) Msgsize() (s int) {
-	s = msgp.StringPrefixSize + len(string(z))
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z MetricType) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendString(o, string(z))
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *MetricType) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	{
-		var zb0001 string
-		zb0001, bts, err = msgp.ReadStringBytes(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		(*z) = MetricType(zb0001)
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z MetricType) Msgsize() (s int) {
-	s = msgp.StringPrefixSize + len(string(z))
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *MetricsGroup) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
-	// string "cacheInterval"
-	o = append(o, 0x82, 0xad, 0x63, 0x61, 0x63, 0x68, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c)
-	o = msgp.AppendDuration(o, z.cacheInterval)
-	// string "metricsGroupOpts"
-	o = append(o, 0xb0, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4f, 0x70, 0x74, 0x73)
-	o, err = z.metricsGroupOpts.MarshalMsg(o)
-	if err != nil {
-		err = msgp.WrapError(err, "metricsGroupOpts")
-		return
-	}
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *MetricsGroup) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "cacheInterval":
-			z.cacheInterval, bts, err = msgp.ReadDurationBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "cacheInterval")
-				return
-			}
-		case "metricsGroupOpts":
-			bts, err = z.metricsGroupOpts.UnmarshalMsg(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "metricsGroupOpts")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *MetricsGroup) Msgsize() (s int) {
-	s = 1 + 14 + msgp.DurationSize + 17 + z.metricsGroupOpts.Msgsize()
-	return
-}
-
 // MarshalMsg implements msgp.Marshaler
 func (z *MetricsGroupOpts) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
@@ -642,3 +577,68 @@ func (z *MetricsGroupOpts) Msgsize() (s int) {
 	s = 1 + 22 + msgp.BoolSize + 24 + msgp.BoolSize + 31 + msgp.BoolSize + 28 + msgp.BoolSize + 16 + msgp.BoolSize + 11 + msgp.BoolSize + 29 + msgp.BoolSize + 19 + msgp.BoolSize + 23 + msgp.BoolSize + 26 + msgp.BoolSize + 32 + msgp.BoolSize + 22 + msgp.BoolSize
 	return
 }
+
+// MarshalMsg implements msgp.Marshaler
+func (z *MetricsGroupV2) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "cacheInterval"
+	o = append(o, 0x82, 0xad, 0x63, 0x61, 0x63, 0x68, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c)
+	o = msgp.AppendDuration(o, z.cacheInterval)
+	// string "metricsGroupOpts"
+	o = append(o, 0xb0, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4f, 0x70, 0x74, 0x73)
+	o, err = z.metricsGroupOpts.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "metricsGroupOpts")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MetricsGroupV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "cacheInterval":
+			z.cacheInterval, bts, err = msgp.ReadDurationBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "cacheInterval")
+				return
+			}
+		case "metricsGroupOpts":
+			bts, err = z.metricsGroupOpts.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "metricsGroupOpts")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *MetricsGroupV2) Msgsize() (s int) {
+	s = 1 + 14 + msgp.DurationSize + 17 + z.metricsGroupOpts.Msgsize()
+	return
+}

cmd/metrics-v2_gen_test.go

@@ -8,64 +8,6 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
-func TestMarshalUnmarshalMetric(t *testing.T) {
-	v := Metric{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgMetric(b *testing.B) {
-	v := Metric{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgMetric(b *testing.B) {
-	v := Metric{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalMetric(b *testing.B) {
-	v := Metric{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
 func TestMarshalUnmarshalMetricDescription(t *testing.T) {
 	v := MetricDescription{}
 	bts, err := v.MarshalMsg(nil)
@@ -124,8 +66,8 @@ func BenchmarkUnmarshalMetricDescription(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalMetricsGroup(t *testing.T) {
-	v := MetricsGroup{}
+func TestMarshalUnmarshalMetricV2(t *testing.T) {
+	v := MetricV2{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
@@ -147,8 +89,8 @@ func TestMarshalUnmarshalMetricsGroup(t *testing.T) {
 	}
 }
 
-func BenchmarkMarshalMsgMetricsGroup(b *testing.B) {
-	v := MetricsGroup{}
+func BenchmarkMarshalMsgMetricV2(b *testing.B) {
+	v := MetricV2{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -156,8 +98,8 @@ func BenchmarkMarshalMsgMetricsGroup(b *testing.B) {
 	}
 }
 
-func BenchmarkAppendMsgMetricsGroup(b *testing.B) {
-	v := MetricsGroup{}
+func BenchmarkAppendMsgMetricV2(b *testing.B) {
+	v := MetricV2{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
@@ -168,8 +110,8 @@ func BenchmarkAppendMsgMetricsGroup(b *testing.B) {
 	}
 }
 
-func BenchmarkUnmarshalMetricsGroup(b *testing.B) {
-	v := MetricsGroup{}
+func BenchmarkUnmarshalMetricV2(b *testing.B) {
+	v := MetricV2{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
@@ -239,3 +181,61 @@ func BenchmarkUnmarshalMetricsGroupOpts(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalMetricsGroupV2(t *testing.T) {
+	v := MetricsGroupV2{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgMetricsGroupV2(b *testing.B) {
+	v := MetricsGroupV2{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMetricsGroupV2(b *testing.B) {
+	v := MetricsGroupV2{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMetricsGroupV2(b *testing.B) {
+	v := MetricsGroupV2{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/metrics-v2_test.go

@@ -80,7 +80,7 @@ func TestGetHistogramMetrics(t *testing.T) {
 		}
 	}
 
-	metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD())
+	metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false)
 	// additional labels for +Inf for all histogram metrics
 	if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) {
 		t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics))

cmd/metrics-v3-api.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+)
+
+const (
+	apiRejectedAuthTotal      MetricName = "rejected_auth_total"
+	apiRejectedHeaderTotal    MetricName = "rejected_header_total"
+	apiRejectedTimestampTotal MetricName = "rejected_timestamp_total"
+	apiRejectedInvalidTotal   MetricName = "rejected_invalid_total"
+
+	apiRequestsWaitingTotal  MetricName = "waiting_total"
+	apiRequestsIncomingTotal MetricName = "incoming_total"
+
+	apiRequestsInFlightTotal  MetricName = "inflight_total"
+	apiRequestsTotal          MetricName = "total"
+	apiRequestsErrorsTotal    MetricName = "errors_total"
+	apiRequests5xxErrorsTotal MetricName = "5xx_errors_total"
+	apiRequests4xxErrorsTotal MetricName = "4xx_errors_total"
+	apiRequestsCanceledTotal  MetricName = "canceled_total"
+
+	apiRequestsTTFBSecondsDistribution MetricName = "ttfb_seconds_distribution"
+
+	apiTrafficSentBytes MetricName = "traffic_sent_bytes"
+	apiTrafficRecvBytes MetricName = "traffic_received_bytes"
+)
+
+var (
+	apiRejectedAuthTotalMD = NewCounterMD(apiRejectedAuthTotal,
+		"Total number of requests rejected for auth failure", "type")
+	apiRejectedHeaderTotalMD = NewCounterMD(apiRejectedHeaderTotal,
+		"Total number of requests rejected for invalid header", "type")
+	apiRejectedTimestampTotalMD = NewCounterMD(apiRejectedTimestampTotal,
+		"Total number of requests rejected for invalid timestamp", "type")
+	apiRejectedInvalidTotalMD = NewCounterMD(apiRejectedInvalidTotal,
+		"Total number of invalid requests", "type")
+
+	apiRequestsWaitingTotalMD = NewGaugeMD(apiRequestsWaitingTotal,
+		"Total number of requests in the waiting queue", "type")
+	apiRequestsIncomingTotalMD = NewGaugeMD(apiRequestsIncomingTotal,
+		"Total number of incoming requests", "type")
+
+	apiRequestsInFlightTotalMD = NewGaugeMD(apiRequestsInFlightTotal,
+		"Total number of requests currently in flight", "name", "type")
+	apiRequestsTotalMD = NewCounterMD(apiRequestsTotal,
+		"Total number of requests", "name", "type")
+	apiRequestsErrorsTotalMD = NewCounterMD(apiRequestsErrorsTotal,
+		"Total number of requests with (4xx and 5xx) errors", "name", "type")
+	apiRequests5xxErrorsTotalMD = NewCounterMD(apiRequests5xxErrorsTotal,
+		"Total number of requests with 5xx errors", "name", "type")
+	apiRequests4xxErrorsTotalMD = NewCounterMD(apiRequests4xxErrorsTotal,
+		"Total number of requests with 4xx errors", "name", "type")
+	apiRequestsCanceledTotalMD = NewCounterMD(apiRequestsCanceledTotal,
+		"Total number of requests canceled by the client", "name", "type")
+
+	apiRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution,
+		"Distribution of time to first byte across API calls", "name", "type", "le")
+
+	apiTrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes,
+		"Total number of bytes sent", "type")
+	apiTrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes,
+		"Total number of bytes received", "type")
+)
+
+// loadAPIRequestsHTTPMetrics - reads S3 HTTP metrics.
+//
+// This is a `MetricsLoaderFn`.
+//
+// This includes node level S3 HTTP metrics.
+//
+// This function currently ignores `opts`.
+func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
+	// Collect node level S3 HTTP metrics.
+	httpStats := globalHTTPStats.toServerHTTPStats(false)
+
+	// Currently we only collect S3 API related stats, so we set the "type"
+	// label to "s3".
+
+	m.Set(apiRejectedAuthTotal, float64(httpStats.TotalS3RejectedAuth), "type", "s3")
+	m.Set(apiRejectedTimestampTotal, float64(httpStats.TotalS3RejectedTime), "type", "s3")
+	m.Set(apiRejectedHeaderTotal, float64(httpStats.TotalS3RejectedHeader), "type", "s3")
+	m.Set(apiRejectedInvalidTotal, float64(httpStats.TotalS3RejectedInvalid), "type", "s3")
+	m.Set(apiRequestsWaitingTotal, float64(httpStats.S3RequestsInQueue), "type", "s3")
+	m.Set(apiRequestsIncomingTotal, float64(httpStats.S3RequestsIncoming), "type", "s3")
+
+	for name, value := range httpStats.CurrentS3Requests.APIStats {
+		m.Set(apiRequestsInFlightTotal, float64(value), "name", name, "type", "s3")
+	}
+	for name, value := range httpStats.TotalS3Requests.APIStats {
+		m.Set(apiRequestsTotal, float64(value), "name", name, "type", "s3")
+	}
+	for name, value := range httpStats.TotalS3Errors.APIStats {
+		m.Set(apiRequestsErrorsTotal, float64(value), "name", name, "type", "s3")
+	}
+	for name, value := range httpStats.TotalS35xxErrors.APIStats {
+		m.Set(apiRequests5xxErrorsTotal, float64(value), "name", name, "type", "s3")
+	}
+	for name, value := range httpStats.TotalS34xxErrors.APIStats {
+		m.Set(apiRequests4xxErrorsTotal, float64(value), "name", name, "type", "s3")
+	}
+	for name, value := range httpStats.TotalS3Canceled.APIStats {
+		m.Set(apiRequestsCanceledTotal, float64(value), "name", name, "type", "s3")
+	}
+	return nil
+}
+
+// loadAPIRequestsTTFBMetrics - loads S3 TTFB metrics.
+//
+// This is a `MetricsLoaderFn`.
+func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
+	renameLabels := map[string]string{"api": "name"}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, renameLabels, nil,
+		"type", "s3")
+	return nil
+}
+
+// loadAPIRequestsNetworkMetrics - loads S3 network metrics.
+//
+// This is a `MetricsLoaderFn`.
+func loadAPIRequestsNetworkMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
+	connStats := globalConnStats.toServerConnStats()
+	m.Set(apiTrafficSentBytes, float64(connStats.s3OutputBytes), "type", "s3")
+	m.Set(apiTrafficRecvBytes, float64(connStats.s3InputBytes), "type", "s3")
+	return nil
+}
+
+// Metric Descriptions for bucket level S3 metrics.
+var (
+	apiBucketTrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes,
+		"Total number of bytes received for a bucket", "bucket", "type")
+	apiBucketTrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes,
+		"Total number of bytes sent for a bucket", "bucket", "type")
+
+	apiBucketRequestsInFlightMD = NewGaugeMD(apiRequestsInFlightTotal,
+		"Total number of requests currently in flight for a bucket", "bucket", "name", "type")
+	apiBucketRequestsTotalMD = NewCounterMD(apiRequestsTotal,
+		"Total number of requests for a bucket", "bucket", "name", "type")
+	apiBucketRequestsCanceledMD = NewCounterMD(apiRequestsCanceledTotal,
+		"Total number of requests canceled by the client for a bucket", "bucket", "name", "type")
+	apiBucketRequests4xxErrorsMD = NewCounterMD(apiRequests4xxErrorsTotal,
+		"Total number of requests with 4xx errors for a bucket", "bucket", "name", "type")
+	apiBucketRequests5xxErrorsMD = NewCounterMD(apiRequests5xxErrorsTotal,
+		"Total number of requests with 5xx errors for a bucket", "bucket", "name", "type")
+
+	apiBucketRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution,
+		"Distribution of time to first byte across API calls for a bucket",
+		"bucket", "name", "le", "type")
+)
+
+// loadAPIBucketHTTPMetrics - loads bucket level S3 HTTP metrics.
+//
+// This is a `MetricsLoaderFn`.
+//
+// This includes bucket level S3 HTTP metrics and S3 network in/out metrics.
+func loadAPIBucketHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
+	if len(buckets) == 0 {
+		return nil
+	}
+	for bucket, inOut := range globalBucketConnStats.getBucketS3InOutBytes(buckets) {
+		recvBytes := inOut.In
+		if recvBytes > 0 {
+			m.Set(apiTrafficSentBytes, float64(recvBytes), "bucket", bucket, "type", "s3")
+		}
+		sentBytes := inOut.Out
+		if sentBytes > 0 {
+			m.Set(apiTrafficRecvBytes, float64(sentBytes), "bucket", bucket, "type", "s3")
+		}
+
+		httpStats := globalBucketHTTPStats.load(bucket)
+		for k, v := range httpStats.currentS3Requests.Load(false) {
+			m.Set(apiRequestsInFlightTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
+		}
+
+		for k, v := range httpStats.totalS3Requests.Load(false) {
+			m.Set(apiRequestsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
+		}
+
+		for k, v := range httpStats.totalS3Canceled.Load(false) {
+			m.Set(apiRequestsCanceledTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
+		}
+
+		for k, v := range httpStats.totalS34xxErrors.Load(false) {
+			m.Set(apiRequests4xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
+		}
+
+		for k, v := range httpStats.totalS35xxErrors.Load(false) {
+			m.Set(apiRequests5xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3")
+		}
+	}
+
+	return nil
+}
+
+// loadAPIBucketTTFBMetrics - loads bucket S3 TTFB metrics.
+//
+// This is a `MetricsLoaderFn`.
+func loadAPIBucketTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
+	renameLabels := map[string]string{"api": "name"}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, renameLabels,
+		buckets, "type", "s3")
+	return nil
+}

cmd/metrics-v3-cache.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/cachevalue"
+)
+
+// metricsCache - cache for metrics.
+//
+// When serving metrics, this cache is passed to the MetricsLoaderFn.
+//
+// This cache is used for metrics that would result in network/storage calls.
+type metricsCache struct {
+	dataUsageInfo       *cachevalue.Cache[DataUsageInfo]
+	esetHealthResult    *cachevalue.Cache[HealthResult]
+	driveMetrics        *cachevalue.Cache[storageMetrics]
+	clusterDriveMetrics *cachevalue.Cache[storageMetrics]
+	nodesUpDown         *cachevalue.Cache[nodesOnline]
+}
+
+func newMetricsCache() *metricsCache {
+	return &metricsCache{
+		dataUsageInfo:       newDataUsageInfoCache(),
+		esetHealthResult:    newESetHealthResultCache(),
+		driveMetrics:        newDriveMetricsCache(),
+		clusterDriveMetrics: newClusterStorageInfoCache(),
+		nodesUpDown:         newNodesUpDownCache(),
+	}
+}
+
+type nodesOnline struct {
+	Online, Offline int
+}
+
+func newNodesUpDownCache() *cachevalue.Cache[nodesOnline] {
+	loadNodesUpDown := func() (v nodesOnline, err error) {
+		v.Online, v.Offline = globalNotificationSys.GetPeerOnlineCount()
+		return
+	}
+	return cachevalue.NewFromFunc(1*time.Minute,
+		cachevalue.Opts{ReturnLastGood: true},
+		loadNodesUpDown)
+}
+
+type storageMetrics struct {
+	storageInfo                              madmin.StorageInfo
+	onlineDrives, offlineDrives, totalDrives int
+}
+
+func newDataUsageInfoCache() *cachevalue.Cache[DataUsageInfo] {
+	loadDataUsage := func() (u DataUsageInfo, err error) {
+		objLayer := newObjectLayerFn()
+		if objLayer == nil {
+			return
+		}
+
+		// Collect cluster level object metrics.
+		u, err = loadDataUsageFromBackend(GlobalContext, objLayer)
+		return
+	}
+	return cachevalue.NewFromFunc(1*time.Minute,
+		cachevalue.Opts{ReturnLastGood: true},
+		loadDataUsage)
+}
+
+func newESetHealthResultCache() *cachevalue.Cache[HealthResult] {
+	loadHealth := func() (r HealthResult, err error) {
+		objLayer := newObjectLayerFn()
+		if objLayer == nil {
+			return
+		}
+
+		r = objLayer.Health(GlobalContext, HealthOptions{})
+		return
+	}
+	return cachevalue.NewFromFunc(1*time.Minute,
+		cachevalue.Opts{ReturnLastGood: true},
+		loadHealth,
+	)
+}
+
+func newDriveMetricsCache() *cachevalue.Cache[storageMetrics] {
+	loadDriveMetrics := func() (v storageMetrics, err error) {
+		objLayer := newObjectLayerFn()
+		if objLayer == nil {
+			return
+		}
+
+		storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
+		onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks)
+		totalDrives := onlineDrives.Merge(offlineDrives)
+		v = storageMetrics{
+			storageInfo:   storageInfo,
+			onlineDrives:  onlineDrives.Sum(),
+			offlineDrives: offlineDrives.Sum(),
+			totalDrives:   totalDrives.Sum(),
+		}
+		return
+	}
+	return cachevalue.NewFromFunc(1*time.Minute,
+		cachevalue.Opts{ReturnLastGood: true},
+		loadDriveMetrics)
+}
+
+func newClusterStorageInfoCache() *cachevalue.Cache[storageMetrics] {
+	loadStorageInfo := func() (v storageMetrics, err error) {
+		objLayer := newObjectLayerFn()
+		if objLayer == nil {
+			return storageMetrics{}, nil
+		}
+		storageInfo := objLayer.StorageInfo(GlobalContext, true)
+		onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks)
+		totalDrives := onlineDrives.Merge(offlineDrives)
+		v = storageMetrics{
+			storageInfo:   storageInfo,
+			onlineDrives:  onlineDrives.Sum(),
+			offlineDrives: offlineDrives.Sum(),
+			totalDrives:   totalDrives.Sum(),
+		}
+		return
+	}
+	return cachevalue.NewFromFunc(1*time.Minute,
+		cachevalue.Opts{ReturnLastGood: true},
+		loadStorageInfo,
+	)
+}

cmd/metrics-v3-cluster-erasure-set.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"strconv"
+)
+
+const (
+	erasureSetOverallWriteQuorum = "overall_write_quorum"
+	erasureSetOverallHealth      = "overall_health"
+	erasureSetReadQuorum         = "read_quorum"
+	erasureSetWriteQuorum        = "write_quorum"
+	erasureSetOnlineDrivesCount  = "online_drives_count"
+	erasureSetHealingDrivesCount = "healing_drives_count"
+	erasureSetHealth             = "health"
+)
+
+const (
+	poolIDL = "pool_id"
+	setIDL  = "set_id"
+)
+
+var (
+	erasureSetOverallWriteQuorumMD = NewGaugeMD(erasureSetOverallWriteQuorum,
+		"Overall write quorum across pools and sets")
+	erasureSetOverallHealthMD = NewGaugeMD(erasureSetOverallHealth,
+		"Overall health across pools and sets (1=healthy, 0=unhealthy)")
+	erasureSetReadQuorumMD = NewGaugeMD(erasureSetReadQuorum,
+		"Read quorum for the erasure set in a pool", poolIDL, setIDL)
+	erasureSetWriteQuorumMD = NewGaugeMD(erasureSetWriteQuorum,
+		"Write quorum for the erasure set in a pool", poolIDL, setIDL)
+	erasureSetOnlineDrivesCountMD = NewGaugeMD(erasureSetOnlineDrivesCount,
+		"Count of online drives in the erasure set in a pool", poolIDL, setIDL)
+	erasureSetHealingDrivesCountMD = NewGaugeMD(erasureSetHealingDrivesCount,
+		"Count of healing drives in the erasure set in a pool", poolIDL, setIDL)
+	erasureSetHealthMD = NewGaugeMD(erasureSetHealth,
+		"Health of the erasure set in a pool (1=healthy, 0=unhealthy)",
+		poolIDL, setIDL)
+)
+
+func b2f(v bool) float64 {
+	if v {
+		return 1
+	}
+	return 0
+}
+
+// loadClusterErasureSetMetrics - `MetricsLoaderFn` for cluster storage erasure
+// set metrics.
+func loadClusterErasureSetMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
+	result, _ := c.esetHealthResult.Get()
+
+	m.Set(erasureSetOverallWriteQuorum, float64(result.WriteQuorum))
+	m.Set(erasureSetOverallHealth, b2f(result.Healthy))
+
+	for _, h := range result.ESHealth {
+		poolLV := strconv.Itoa(h.PoolID)
+		setLV := strconv.Itoa(h.SetID)
+		m.Set(erasureSetReadQuorum, float64(h.ReadQuorum),
+			poolIDL, poolLV, setIDL, setLV)
+		m.Set(erasureSetWriteQuorum, float64(h.WriteQuorum),
+			poolIDL, poolLV, setIDL, setLV)
+		m.Set(erasureSetOnlineDrivesCount, float64(h.HealthyDrives),
+			poolIDL, poolLV, setIDL, setLV)
+		m.Set(erasureSetHealingDrivesCount, float64(h.HealingDrives),
+			poolIDL, poolLV, setIDL, setLV)
+		m.Set(erasureSetHealth, b2f(h.Healthy),
+			poolIDL, poolLV, setIDL, setLV)
+	}
+
+	return nil
+}

cmd/metrics-v3-cluster-health.go

@@ -0,0 +1,109 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import "context"
+
+const (
+	healthDrivesOfflineCount = "drives_offline_count"
+	healthDrivesOnlineCount  = "drives_online_count"
+	healthDrivesCount        = "drives_count"
+)
+
+var (
+	healthDrivesOfflineCountMD = NewGaugeMD(healthDrivesOfflineCount,
+		"Count of offline drives in the cluster")
+	healthDrivesOnlineCountMD = NewGaugeMD(healthDrivesOnlineCount,
+		"Count of online drives in the cluster")
+	healthDrivesCountMD = NewGaugeMD(healthDrivesCount,
+		"Count of all drives in the cluster")
+)
+
+// loadClusterHealthDriveMetrics - `MetricsLoaderFn` for cluster storage drive metrics
+// such as online, offline and total drives.
+func loadClusterHealthDriveMetrics(ctx context.Context, m MetricValues,
+	c *metricsCache,
+) error {
+	clusterDriveMetrics, _ := c.clusterDriveMetrics.Get()
+
+	m.Set(healthDrivesOfflineCount, float64(clusterDriveMetrics.offlineDrives))
+	m.Set(healthDrivesOnlineCount, float64(clusterDriveMetrics.onlineDrives))
+	m.Set(healthDrivesCount, float64(clusterDriveMetrics.totalDrives))
+
+	return nil
+}
+
+const (
+	healthNodesOfflineCount = "nodes_offline_count"
+	healthNodesOnlineCount  = "nodes_online_count"
+)
+
+var (
+	healthNodesOfflineCountMD = NewGaugeMD(healthNodesOfflineCount,
+		"Count of offline nodes in the cluster")
+	healthNodesOnlineCountMD = NewGaugeMD(healthNodesOnlineCount,
+		"Count of online nodes in the cluster")
+)
+
+// loadClusterHealthNodeMetrics - `MetricsLoaderFn` for cluster health node
+// metrics.
+func loadClusterHealthNodeMetrics(ctx context.Context, m MetricValues,
+	c *metricsCache,
+) error {
+	nodesUpDown, _ := c.nodesUpDown.Get()
+
+	m.Set(healthNodesOfflineCount, float64(nodesUpDown.Offline))
+	m.Set(healthNodesOnlineCount, float64(nodesUpDown.Online))
+
+	return nil
+}
+
+const (
+	healthCapacityRawTotalBytes    = "capacity_raw_total_bytes"
+	healthCapacityRawFreeBytes     = "capacity_raw_free_bytes"
+	healthCapacityUsableTotalBytes = "capacity_usable_total_bytes"
+	healthCapacityUsableFreeBytes  = "capacity_usable_free_bytes"
+)
+
+var (
+	healthCapacityRawTotalBytesMD = NewGaugeMD(healthCapacityRawTotalBytes,
+		"Total cluster raw storage capacity in bytes")
+	healthCapacityRawFreeBytesMD = NewGaugeMD(healthCapacityRawFreeBytes,
+		"Total cluster raw storage free in bytes")
+	healthCapacityUsableTotalBytesMD = NewGaugeMD(healthCapacityUsableTotalBytes,
+		"Total cluster usable storage capacity in bytes")
+	healthCapacityUsableFreeBytesMD = NewGaugeMD(healthCapacityUsableFreeBytes,
+		"Total cluster usable storage free in bytes")
+)
+
+// loadClusterHealthCapacityMetrics - `MetricsLoaderFn` for cluster storage
+// capacity metrics.
+func loadClusterHealthCapacityMetrics(ctx context.Context, m MetricValues,
+	c *metricsCache,
+) error {
+	clusterDriveMetrics, _ := c.clusterDriveMetrics.Get()
+
+	storageInfo := clusterDriveMetrics.storageInfo
+
+	m.Set(healthCapacityRawTotalBytes, float64(GetTotalCapacity(storageInfo.Disks)))
+	m.Set(healthCapacityRawFreeBytes, float64(GetTotalCapacityFree(storageInfo.Disks)))
+	m.Set(healthCapacityUsableTotalBytes, float64(GetTotalUsableCapacity(storageInfo.Disks, storageInfo)))
+	m.Set(healthCapacityUsableFreeBytes, float64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)))
+
+	return nil
+}

cmd/metrics-v3-cluster-usage.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"github.com/minio/minio/internal/logger"
+)
+
+const (
+	usageSinceLastUpdateSeconds   = "since_last_update_seconds"
+	usageTotalBytes               = "total_bytes"
+	usageObjectsCount             = "count"
+	usageVersionsCount            = "versions_count"
+	usageDeleteMarkersCount       = "delete_markers_count"
+	usageBucketsCount             = "buckets_count"
+	usageSizeDistribution         = "size_distribution"
+	usageVersionCountDistribution = "version_count_distribution"
+)
+
+var (
+	usageSinceLastUpdateSecondsMD = NewGaugeMD(usageSinceLastUpdateSeconds,
+		"Time since last update of usage metrics in seconds")
+	usageTotalBytesMD = NewGaugeMD(usageTotalBytes,
+		"Total cluster usage in bytes")
+	usageObjectsCountMD = NewGaugeMD(usageObjectsCount,
+		"Total cluster objects count")
+	usageVersionsCountMD = NewGaugeMD(usageVersionsCount,
+		"Total cluster object versions (including delete markers) count")
+	usageDeleteMarkersCountMD = NewGaugeMD(usageDeleteMarkersCount,
+		"Total cluster delete markers count")
+	usageBucketsCountMD = NewGaugeMD(usageBucketsCount,
+		"Total cluster buckets count")
+	usageObjectsDistributionMD = NewGaugeMD(usageSizeDistribution,
+		"Cluster object size distribution", "range")
+	usageVersionsDistributionMD = NewGaugeMD(usageVersionCountDistribution,
+		"Cluster object version count distribution", "range")
+)
+
+// loadClusterUsageObjectMetrics - reads cluster usage metrics.
+//
+// This is a `MetricsLoaderFn`.
+func loadClusterUsageObjectMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
+	dataUsageInfo, err := c.dataUsageInfo.Get()
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil
+	}
+
+	// data usage has not captured any data yet.
+	if dataUsageInfo.LastUpdate.IsZero() {
+		return nil
+	}
+
+	var (
+		clusterSize               uint64
+		clusterBuckets            uint64
+		clusterObjectsCount       uint64
+		clusterVersionsCount      uint64
+		clusterDeleteMarkersCount uint64
+	)
+
+	clusterObjectSizesHistogram := map[string]uint64{}
+	clusterVersionsHistogram := map[string]uint64{}
+	for _, usage := range dataUsageInfo.BucketsUsage {
+		clusterBuckets++
+		clusterSize += usage.Size
+		clusterObjectsCount += usage.ObjectsCount
+		clusterVersionsCount += usage.VersionsCount
+		clusterDeleteMarkersCount += usage.DeleteMarkersCount
+		for k, v := range usage.ObjectSizesHistogram {
+			clusterObjectSizesHistogram[k] += v
+		}
+		for k, v := range usage.ObjectVersionsHistogram {
+			clusterVersionsHistogram[k] += v
+		}
+	}
+
+	m.Set(usageSinceLastUpdateSeconds, time.Since(dataUsageInfo.LastUpdate).Seconds())
+	m.Set(usageTotalBytes, float64(clusterSize))
+	m.Set(usageObjectsCount, float64(clusterObjectsCount))
+	m.Set(usageVersionsCount, float64(clusterVersionsCount))
+	m.Set(usageDeleteMarkersCount, float64(clusterDeleteMarkersCount))
+	m.Set(usageBucketsCount, float64(clusterBuckets))
+	for k, v := range clusterObjectSizesHistogram {
+		m.Set(usageSizeDistribution, float64(v), "range", k)
+	}
+	for k, v := range clusterVersionsHistogram {
+		m.Set(usageVersionCountDistribution, float64(v), "range", k)
+	}
+
+	return nil
+}
+
+const (
+	usageBucketQuotaTotalBytes = "quota_total_bytes"
+
+	usageBucketTotalBytes                     = "total_bytes"
+	usageBucketObjectsCount                   = "objects_count"
+	usageBucketVersionsCount                  = "versions_count"
+	usageBucketDeleteMarkersCount             = "delete_markers_count"
+	usageBucketObjectSizeDistribution         = "object_size_distribution"
+	usageBucketObjectVersionCountDistribution = "object_version_count_distribution"
+)
+
+var (
+	usageBucketTotalBytesMD = NewGaugeMD(usageBucketTotalBytes,
+		"Total bucket size in bytes", "bucket")
+	usageBucketObjectsTotalMD = NewGaugeMD(usageBucketObjectsCount,
+		"Total objects count in bucket", "bucket")
+	usageBucketVersionsCountMD = NewGaugeMD(usageBucketVersionsCount,
+		"Total object versions (including delete markers) count in bucket", "bucket")
+	usageBucketDeleteMarkersCountMD = NewGaugeMD(usageBucketDeleteMarkersCount,
+		"Total delete markers count in bucket", "bucket")
+
+	usageBucketQuotaTotalBytesMD = NewGaugeMD(usageBucketQuotaTotalBytes,
+		"Total bucket quota in bytes", "bucket")
+
+	usageBucketObjectSizeDistributionMD = NewGaugeMD(usageBucketObjectSizeDistribution,
+		"Bucket object size distribution", "range", "bucket")
+	usageBucketObjectVersionCountDistributionMD = NewGaugeMD(
+		usageBucketObjectVersionCountDistribution,
+		"Bucket object version count distribution", "range", "bucket")
+)
+
+// loadClusterUsageBucketMetrics - `MetricsLoaderFn` to load bucket usage metrics.
+func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metricsCache, buckets []string) error {
+	dataUsageInfo, err := c.dataUsageInfo.Get()
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil
+	}
+
+	// data usage has not been captured yet.
+	if dataUsageInfo.LastUpdate.IsZero() {
+		return nil
+	}
+
+	m.Set(usageSinceLastUpdateSeconds, float64(time.Since(dataUsageInfo.LastUpdate)))
+
+	for _, bucket := range buckets {
+		usage, ok := dataUsageInfo.BucketsUsage[bucket]
+		if !ok {
+			continue
+		}
+		quota, err := globalBucketQuotaSys.Get(ctx, bucket)
+		if err != nil {
+			// Log and continue if we are unable to retrieve metrics for this
+			// bucket.
+			logger.LogIf(ctx, err)
+			continue
+		}
+
+		m.Set(usageBucketTotalBytes, float64(usage.Size), "bucket", bucket)
+		m.Set(usageBucketObjectsCount, float64(usage.ObjectsCount), "bucket", bucket)
+		m.Set(usageBucketVersionsCount, float64(usage.VersionsCount), "bucket", bucket)
+		m.Set(usageBucketDeleteMarkersCount, float64(usage.DeleteMarkersCount), "bucket", bucket)
+
+		if quota != nil && quota.Quota > 0 {
+			m.Set(usageBucketQuotaTotalBytes, float64(quota.Quota), "bucket", bucket)
+		}
+
+		for k, v := range usage.ObjectSizesHistogram {
+			m.Set(usageBucketObjectSizeDistribution, float64(v), "range", k, "bucket", bucket)
+		}
+		for k, v := range usage.ObjectVersionsHistogram {
+			m.Set(usageBucketObjectVersionCountDistribution, float64(v), "range", k, "bucket", bucket)
+		}
+
+	}
+	return nil
+}

cmd/metrics-v3-handler.go

@@ -0,0 +1,254 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/mcontext"
+	"github.com/minio/mux"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+type promLogger struct{}
+
+func (p promLogger) Println(v ...interface{}) {
+	s := make([]string, 0, len(v))
+	for _, val := range v {
+		s = append(s, fmt.Sprintf("%v", val))
+	}
+	err := fmt.Errorf("metrics handler error: %v", strings.Join(s, " "))
+	logger.LogIf(GlobalContext, err)
+}
+
+type metricsV3Server struct {
+	registry *prometheus.Registry
+	opts     promhttp.HandlerOpts
+	authFn   func(http.Handler) http.Handler
+
+	metricsData *metricsV3Collection
+}
+
+func newMetricsV3Server(authType prometheusAuthType) *metricsV3Server {
+	registry := prometheus.NewRegistry()
+	authFn := AuthMiddleware
+	if authType == prometheusPublic {
+		authFn = NoAuthMiddleware
+	}
+
+	metricGroups := newMetricGroups(registry)
+
+	return &metricsV3Server{
+		registry: registry,
+		opts: promhttp.HandlerOpts{
+			ErrorLog:            promLogger{},
+			ErrorHandling:       promhttp.HTTPErrorOnError,
+			Registry:            registry,
+			MaxRequestsInFlight: 2,
+		},
+		authFn: authFn,
+
+		metricsData: metricGroups,
+	}
+}
+
+// metricDisplay - contains info on a metric for display purposes.
+type metricDisplay struct {
+	Name   string   `json:"name"`
+	Help   string   `json:"help"`
+	Type   string   `json:"type"`
+	Labels []string `json:"labels"`
+}
+
+func (md metricDisplay) String() string {
+	return fmt.Sprintf("Name: %s\nType: %s\nHelp: %s\nLabels: {%s}\n", md.Name, md.Type, md.Help, strings.Join(md.Labels, ","))
+}
+
+func (md metricDisplay) TableRow() string {
+	labels := strings.Join(md.Labels, ",")
+	if labels == "" {
+		labels = ""
+	} else {
+		labels = "`" + labels + "`"
+	}
+	return fmt.Sprintf("| `%s` | `%s` | %s | %s |\n", md.Name, md.Type, md.Help, labels)
+}
+
+// listMetrics - returns a handler that lists all the metrics that could be
+// returned for the requested path.
+//
+// FIXME: It currently only lists `minio_` prefixed metrics.
+func (h *metricsV3Server) listMetrics(path string) http.Handler {
+	// First collect all matching MetricsGroup's
+	matchingMG := make(map[collectorPath]*MetricsGroup)
+	for _, collPath := range h.metricsData.collectorPaths {
+		if collPath.isDescendantOf(path) {
+			if v, ok := h.metricsData.mgMap[collPath]; ok {
+				matchingMG[collPath] = v
+			} else {
+				matchingMG[collPath] = h.metricsData.bucketMGMap[collPath]
+			}
+		}
+	}
+
+	if len(matchingMG) == 0 {
+		return nil
+	}
+
+	var metrics []metricDisplay
+	for _, collectorPath := range h.metricsData.collectorPaths {
+		if mg, ok := matchingMG[collectorPath]; ok {
+			var commonLabels []string
+			for k := range mg.ExtraLabels {
+				commonLabels = append(commonLabels, k)
+			}
+			for _, d := range mg.Descriptors {
+				labels := slices.Clone(d.VariableLabels)
+				labels = append(labels, commonLabels...)
+				metric := metricDisplay{
+					Name:   mg.MetricFQN(d.Name),
+					Help:   d.Help,
+					Type:   d.Type.String(),
+					Labels: labels,
+				}
+				metrics = append(metrics, metric)
+			}
+		}
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		contentType := r.Header.Get("Content-Type")
+		if contentType == "application/json" {
+			w.Header().Set("Content-Type", "application/json")
+			jsonEncoder := json.NewEncoder(w)
+			jsonEncoder.Encode(metrics)
+			return
+		}
+
+		// If not JSON, return plain text. We format it as a markdown table for
+		// readability.
+		w.Header().Set("Content-Type", "text/plain")
+		var b strings.Builder
+		b.WriteString("| Name | Type | Help | Labels |\n")
+		b.WriteString("| ---- | ---- | ---- | ------ |\n")
+		for _, metric := range metrics {
+			b.WriteString(metric.TableRow())
+		}
+		w.Write([]byte(b.String()))
+	})
+}
+
+func (h *metricsV3Server) handle(path string, isListingRequest bool, buckets []string) http.Handler {
+	var notFoundHandler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "Metrics Resource Not found", http.StatusNotFound)
+	})
+
+	// Require that metrics path has at least component.
+	if path == "/" {
+		return notFoundHandler
+	}
+
+	if isListingRequest {
+		handler := h.listMetrics(path)
+		if handler == nil {
+			return notFoundHandler
+		}
+		return handler
+	}
+
+	// In each of the following cases, we check if the collect path is a
+	// descendant of `path`, and if so, we add the corresponding gatherer to
+	// the list of gatherers. This way, /api/a will return all metrics returned
+	// by /api/a/b and /api/a/c (and any other matching descendant collector
+	// paths).
+
+	var gatherers []prometheus.Gatherer
+	for _, collectorPath := range h.metricsData.collectorPaths {
+		if collectorPath.isDescendantOf(path) {
+			gatherer := h.metricsData.mgGatherers[collectorPath]
+
+			// For Bucket metrics we need to set the buckets argument inside the
+			// metric group, so that it will affect collection. If no buckets
+			// are provided, we will not return bucket metrics.
+			if bmg, ok := h.metricsData.bucketMGMap[collectorPath]; ok {
+				if len(buckets) == 0 {
+					continue
+				}
+				unLocker := bmg.LockAndSetBuckets(buckets)
+				defer unLocker()
+			}
+			gatherers = append(gatherers, gatherer)
+		}
+	}
+
+	if len(gatherers) == 0 {
+		return notFoundHandler
+	}
+
+	return promhttp.HandlerFor(prometheus.Gatherers(gatherers), h.opts)
+}
+
+// ServeHTTP - implements http.Handler interface.
+//
+// When the `list` query parameter is provided (its value is ignored), the
+// server lists all metrics that could be returned for the requested path.
+//
+// The (repeatable) `buckets` query parameter is a list of bucket names (or it
+// could be a comma separated value) to return metrics with a bucket label.
+// Bucket metrics will be returned only for the provided buckets. If no buckets
+// parameter is provided, no bucket metrics are returned.
+func (h *metricsV3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	pathComponents := mux.Vars(r)["pathComps"]
+	isListingRequest := r.Form.Has("list")
+
+	// Parse optional buckets query parameter.
+	bucketsParam := r.Form["buckets"]
+	buckets := make([]string, 0, len(bucketsParam))
+	for _, bp := range bucketsParam {
+		bp = strings.TrimSpace(bp)
+		if bp == "" {
+			continue
+		}
+		splits := strings.Split(bp, ",")
+		for _, split := range splits {
+			buckets = append(buckets, strings.TrimSpace(split))
+		}
+	}
+
+	innerHandler := h.handle(pathComponents, isListingRequest, buckets)
+
+	// Add tracing to the prom. handler
+	tracedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
+		if ok {
+			tc.FuncName = "handler.MetricsV3"
+			tc.ResponseRecorder.LogErrBody = true
+		}
+
+		innerHandler.ServeHTTP(w, r)
+	})
+
+	// Add authentication
+	h.authFn(tracedHandler).ServeHTTP(w, r)
+}

cmd/metrics-v3-system-drive.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/minio/minio/internal/logger"
+)
+
+// label constants
+const (
+	driveL      = "drive"
+	poolIndexL  = "pool_index"
+	setIndexL   = "set_index"
+	driveIndexL = "drive_index"
+
+	apiL = "api"
+)
+
+var allDriveLabels = []string{driveL, poolIndexL, setIndexL, driveIndexL}
+
+const (
+	driveUsedBytes               = "used_bytes"
+	driveFreeBytes               = "free_bytes"
+	driveTotalBytes              = "total_bytes"
+	driveFreeInodes              = "free_inodes"
+	driveTimeoutErrorsTotal      = "timeout_errors_total"
+	driveAvailabilityErrorsTotal = "availability_errors_total"
+	driveWaitingIO               = "waiting_io"
+	driveAPILatencyMicros        = "api_latency_micros"
+
+	driveOfflineCount = "offline_count"
+	driveOnlineCount  = "online_count"
+	driveCount        = "count"
+)
+
+var (
+	driveUsedBytesMD = NewGaugeMD(driveUsedBytes,
+		"Total storage used on a drive in bytes", allDriveLabels...)
+	driveFreeBytesMD = NewGaugeMD(driveFreeBytes,
+		"Total storage free on a drive in bytes", allDriveLabels...)
+	driveTotalBytesMD = NewGaugeMD(driveTotalBytes,
+		"Total storage available on a drive in bytes", allDriveLabels...)
+	driveFreeInodesMD = NewGaugeMD(driveFreeInodes,
+		"Total free inodes on a drive", allDriveLabels...)
+	driveTimeoutErrorsMD = NewCounterMD(driveTimeoutErrorsTotal,
+		"Total timeout errors on a drive", allDriveLabels...)
+	driveAvailabilityErrorsMD = NewCounterMD(driveAvailabilityErrorsTotal,
+		"Total availability errors (I/O errors, permission denied and timeouts) on a drive",
+		allDriveLabels...)
+	driveWaitingIOMD = NewGaugeMD(driveWaitingIO,
+		"Total waiting I/O operations on a drive", allDriveLabels...)
+	driveAPILatencyMD = NewGaugeMD(driveAPILatencyMicros,
+		"Average last minute latency in µs for drive API storage operations",
+		append(allDriveLabels, apiL)...)
+
+	driveOfflineCountMD = NewGaugeMD(driveOfflineCount,
+		"Count of offline drives")
+	driveOnlineCountMD = NewGaugeMD(driveOnlineCount,
+		"Count of online drives")
+	driveCountMD = NewGaugeMD(driveCount,
+		"Count of all drives")
+)
+
+// loadDriveMetrics - `MetricsLoaderFn` for node drive metrics.
+func loadDriveMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
+	driveMetrics, err := c.driveMetrics.Get()
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil
+	}
+
+	storageInfo := driveMetrics.storageInfo
+
+	for _, disk := range storageInfo.Disks {
+		labels := []string{
+			driveL, disk.DrivePath,
+			poolIndexL, strconv.Itoa(disk.PoolIndex),
+			setIndexL, strconv.Itoa(disk.SetIndex),
+			driveIndexL, strconv.Itoa(disk.DiskIndex),
+		}
+
+		m.Set(driveUsedBytes, float64(disk.UsedSpace), labels...)
+		m.Set(driveFreeBytes, float64(disk.AvailableSpace), labels...)
+		m.Set(driveTotalBytes, float64(disk.TotalSpace), labels...)
+		m.Set(driveFreeInodes, float64(disk.FreeInodes), labels...)
+
+		if disk.Metrics != nil {
+			m.Set(driveTimeoutErrorsTotal, float64(disk.Metrics.TotalErrorsTimeout), labels...)
+			m.Set(driveAvailabilityErrorsTotal, float64(disk.Metrics.TotalErrorsAvailability), labels...)
+			m.Set(driveWaitingIO, float64(disk.Metrics.TotalWaiting), labels...)
+
+			// Append the api label for the drive API latencies.
+			labels = append(labels, "api", "")
+			lastIdx := len(labels) - 1
+			for apiName, latency := range disk.Metrics.LastMinute {
+				labels[lastIdx] = "storage." + apiName
+				m.Set(driveAPILatencyMicros, float64(latency.Avg().Microseconds()),
+					labels...)
+			}
+		}
+	}
+
+	m.Set(driveOfflineCount, float64(driveMetrics.offlineDrives))
+	m.Set(driveOnlineCount, float64(driveMetrics.onlineDrives))
+	m.Set(driveCount, float64(driveMetrics.totalDrives))
+
+	return nil
+}

cmd/metrics-v3-system-network.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+
+	"github.com/minio/minio/internal/rest"
+)
+
+const (
+	internodeErrorsTotal      MetricName = "errors_total"
+	internodeDialErrorsTotal  MetricName = "dial_errors_total"
+	internodeDialAvgTimeNanos MetricName = "dial_avg_time_nanos"
+	internodeSentBytesTotal   MetricName = "sent_bytes_total"
+	internodeRecvBytesTotal   MetricName = "recv_bytes_total"
+)
+
+var (
+	internodeErrorsTotalMD = NewCounterMD(internodeErrorsTotal,
+		"Total number of failed internode calls")
+	internodeDialedErrorsTotalMD = NewCounterMD(internodeDialErrorsTotal,
+		"Total number of internode TCP dial timeouts and errors")
+	internodeDialAvgTimeNanosMD = NewGaugeMD(internodeDialAvgTimeNanos,
+		"Average dial time of internode TCP calls in nanoseconds")
+	internodeSentBytesTotalMD = NewCounterMD(internodeSentBytesTotal,
+		"Total number of bytes sent to other peer nodes")
+	internodeRecvBytesTotalMD = NewCounterMD(internodeRecvBytesTotal,
+		"Total number of bytes received from other peer nodes")
+)
+
+// loadNetworkInternodeMetrics - reads internode network metrics.
+//
+// This is a `MetricsLoaderFn`.
+func loadNetworkInternodeMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
+	connStats := globalConnStats.toServerConnStats()
+	rpcStats := rest.GetRPCStats()
+	if globalIsDistErasure {
+		m.Set(internodeErrorsTotal, float64(rpcStats.Errs))
+		m.Set(internodeDialErrorsTotal, float64(rpcStats.DialErrs))
+		m.Set(internodeDialAvgTimeNanos, float64(rpcStats.DialAvgDuration))
+		m.Set(internodeSentBytesTotal, float64(connStats.internodeOutputBytes))
+		m.Set(internodeRecvBytesTotal, float64(connStats.internodeInputBytes))
+	}
+	return nil
+}

cmd/metrics-v3-types.go

@@ -0,0 +1,487 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/minio/minio/internal/logger"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/exp/slices"
+)
+
+type collectorPath string
+
+// metricPrefix converts a collector path to a metric name prefix. The path is
+// converted to snake-case (by replaced '/' and '-' with '_') and prefixed with
+// `minio_`.
+func (cp collectorPath) metricPrefix() string {
+	s := strings.TrimPrefix(string(cp), "/")
+	s = strings.ReplaceAll(s, "/", "_")
+	s = strings.ReplaceAll(s, "-", "_")
+	return "minio_" + s
+}
+
+// isDescendantOf returns true if it is a descendant of (or the same as)
+// `ancestor`.
+//
+// For example:
+//
+//	 	/a, /a/b, /a/b/c are all descendants of /a.
+//		/abc or /abd/a are not descendants of /ab.
+func (cp collectorPath) isDescendantOf(arg string) bool {
+	descendant := string(cp)
+	if descendant == arg {
+		return true
+	}
+	if len(arg) >= len(descendant) {
+		return false
+	}
+	if !strings.HasSuffix(arg, "/") {
+		arg += "/"
+	}
+	return strings.HasPrefix(descendant, arg)
+}
+
+// MetricType - represents the type of a metric.
+type MetricType int
+
+const (
+	// CounterMT - represents a counter metric.
+	CounterMT MetricType = iota
+	// GaugeMT - represents a gauge metric.
+	GaugeMT
+	// HistogramMT - represents a histogram metric.
+	HistogramMT
+)
+
+func (mt MetricType) String() string {
+	switch mt {
+	case CounterMT:
+		return "counter"
+	case GaugeMT:
+		return "gauge"
+	case HistogramMT:
+		return "histogram"
+	default:
+		return "*unknown*"
+	}
+}
+
+func (mt MetricType) toProm() prometheus.ValueType {
+	switch mt {
+	case CounterMT:
+		return prometheus.CounterValue
+	case GaugeMT:
+		return prometheus.GaugeValue
+	case HistogramMT:
+		return prometheus.CounterValue
+	default:
+		panic(fmt.Sprintf("unknown metric type: %d", mt))
+	}
+}
+
+// MetricDescriptor - represents a metric descriptor.
+type MetricDescriptor struct {
+	Name           MetricName
+	Type           MetricType
+	Help           string
+	VariableLabels []string
+
+	// managed values follow:
+	labelSet map[string]struct{}
+}
+
+func (md *MetricDescriptor) getLabelSet() map[string]struct{} {
+	if md.labelSet != nil {
+		return md.labelSet
+	}
+	md.labelSet = make(map[string]struct{}, len(md.VariableLabels))
+	for _, label := range md.VariableLabels {
+		md.labelSet[label] = struct{}{}
+	}
+	return md.labelSet
+}
+
+func (md *MetricDescriptor) toPromName(namePrefix string) string {
+	return prometheus.BuildFQName(namePrefix, "", string(md.Name))
+}
+
+func (md *MetricDescriptor) toPromDesc(namePrefix string, extraLabels map[string]string) *prometheus.Desc {
+	return prometheus.NewDesc(
+		md.toPromName(namePrefix),
+		md.Help,
+		md.VariableLabels, extraLabels,
+	)
+}
+
+// NewCounterMD - creates a new counter metric descriptor.
+func NewCounterMD(name MetricName, help string, labels ...string) MetricDescriptor {
+	return MetricDescriptor{
+		Name:           name,
+		Type:           CounterMT,
+		Help:           help,
+		VariableLabels: labels,
+	}
+}
+
+// NewGaugeMD - creates a new gauge metric descriptor.
+func NewGaugeMD(name MetricName, help string, labels ...string) MetricDescriptor {
+	return MetricDescriptor{
+		Name:           name,
+		Type:           GaugeMT,
+		Help:           help,
+		VariableLabels: labels,
+	}
+}
+
+type metricValue struct {
+	Labels map[string]string
+	Value  float64
+}
+
+// MetricValues - type to set metric values retrieved while loading metrics. A
+// value of this type is passed to the `MetricsLoaderFn`.
+type MetricValues struct {
+	values      map[MetricName][]metricValue
+	descriptors map[MetricName]MetricDescriptor
+}
+
+func newMetricValues(d map[MetricName]MetricDescriptor) MetricValues {
+	return MetricValues{
+		values:      make(map[MetricName][]metricValue, len(d)),
+		descriptors: d,
+	}
+}
+
+// ToPromMetrics - converts the internal metric values to Prometheus
+// adding the given name prefix. The extraLabels are added to each metric as
+// constant labels.
+func (m *MetricValues) ToPromMetrics(namePrefix string, extraLabels map[string]string,
+) []prometheus.Metric {
+	metrics := make([]prometheus.Metric, 0, len(m.values))
+	for metricName, mv := range m.values {
+		desc := m.descriptors[metricName]
+		promDesc := desc.toPromDesc(namePrefix, extraLabels)
+		for _, v := range mv {
+			// labelValues is in the same order as the variable labels in the
+			// descriptor.
+			labelValues := make([]string, 0, len(v.Labels))
+			for _, k := range desc.VariableLabels {
+				labelValues = append(labelValues, v.Labels[k])
+			}
+			metrics = append(metrics,
+				prometheus.MustNewConstMetric(promDesc, desc.Type.toProm(), v.Value,
+					labelValues...))
+		}
+	}
+	return metrics
+}
+
+// Set - sets a metric value along with any provided labels. It is used only
+// with Gauge and Counter metrics.
+//
+// If the MetricName given here is not present in the `MetricsGroup`'s
+// descriptors, this function panics.
+//
+// Panics if `labels` is not a list of ordered label name and label value pairs
+// or if all labels for the metric are not provided.
+func (m *MetricValues) Set(name MetricName, value float64, labels ...string) {
+	desc, ok := m.descriptors[name]
+	if !ok {
+		panic(fmt.Sprintf("metric has no description: %s", name))
+	}
+
+	if len(labels)%2 != 0 {
+		panic("labels must be a list of ordered key-value pairs")
+	}
+
+	validLabels := desc.getLabelSet()
+	labelMap := make(map[string]string, len(labels)/2)
+	for i := 0; i < len(labels); i += 2 {
+		if _, ok := validLabels[labels[i]]; !ok {
+			panic(fmt.Sprintf("invalid label: %s (metric: %s)", labels[i], name))
+		}
+		labelMap[labels[i]] = labels[i+1]
+	}
+
+	if len(labels)/2 != len(validLabels) {
+		panic(fmt.Sprintf("not all labels were given values"))
+	}
+
+	v, ok := m.values[name]
+	if !ok {
+		v = make([]metricValue, 0, 1)
+	}
+	m.values[name] = append(v, metricValue{
+		Labels: labelMap,
+		Value:  value,
+	})
+}
+
+// SetHistogram - sets values for the given MetricName using the provided
+// histogram.
+//
+// `renameLabels` is a map of label names to rename. The keys are the original
+// label names and the values are the new label names.
+//
+// TODO: bucketFilter doc
+//
+// `extraLabels` are additional labels to add to each metric. They are ordered
+// label name and value pairs.
+func (m *MetricValues) SetHistogram(name MetricName, hist *prometheus.HistogramVec,
+	renameLabels map[string]string, bucketFilter []string, extraLabels ...string,
+) {
+	if _, ok := m.descriptors[name]; !ok {
+		panic(fmt.Sprintf("metric has no description: %s", name))
+	}
+	dummyDesc := MetricDescription{}
+	metricsV2 := getHistogramMetrics(hist, dummyDesc, false)
+	for _, metric := range metricsV2 {
+		// If a bucket filter is provided, only add metrics for the given
+		// buckets.
+		if len(bucketFilter) > 0 {
+			if !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) {
+				continue
+			}
+		}
+
+		labels := make([]string, 0, len(metric.VariableLabels)*2)
+		for k, v := range metric.VariableLabels {
+			if newLabel, ok := renameLabels[k]; ok {
+				labels = append(labels, newLabel, v)
+			} else {
+				labels = append(labels, k, v)
+			}
+		}
+		labels = append(labels, extraLabels...)
+		m.Set(name, metric.Value, labels...)
+	}
+}
+
+// MetricsLoaderFn - represents a function to load metrics from the
+// metricsCache.
+//
+// Note that returning an error here will cause the Metrics handler to return a
+// 500 Internal Server Error.
+type MetricsLoaderFn func(context.Context, MetricValues, *metricsCache) error
+
+// JoinLoaders - joins multiple loaders into a single loader. The returned
+// loader will call each of the given loaders in order. If any of the loaders
+// return an error, the returned loader will return that error.
+func JoinLoaders(loaders ...MetricsLoaderFn) MetricsLoaderFn {
+	return func(ctx context.Context, m MetricValues, c *metricsCache) error {
+		for _, loader := range loaders {
+			if err := loader(ctx, m, c); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+// BucketMetricsLoaderFn - represents a function to load metrics from the
+// metricsCache and the system for a given list of buckets.
+//
+// Note that returning an error here will cause the Metrics handler to return a
+// 500 Internal Server Error.
+type BucketMetricsLoaderFn func(context.Context, MetricValues, *metricsCache, []string) error
+
+// JoinBucketLoaders - joins multiple bucket loaders into a single loader,
+// similar to `JoinLoaders`.
+func JoinBucketLoaders(loaders ...BucketMetricsLoaderFn) BucketMetricsLoaderFn {
+	return func(ctx context.Context, m MetricValues, c *metricsCache, b []string) error {
+		for _, loader := range loaders {
+			if err := loader(ctx, m, c, b); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
+// MetricsGroup - represents a group of metrics. It includes a `MetricsLoaderFn`
+// function that provides a way to load the metrics from the system. The metrics
+// are cached and refreshed after a given timeout.
+//
+// For metrics with a `bucket` dimension, a list of buckets argument is required
+// to collect the metrics.
+//
+// It implements the prometheus.Collector interface for metric groups without a
+// bucket dimension. For metric groups with a bucket dimension, use the
+// `GetBucketCollector` method to get a `BucketCollector` that implements the
+// prometheus.Collector interface.
+type MetricsGroup struct {
+	// Path (relative to the Metrics v3 base endpoint) at which this group of
+	// metrics is served. This value is converted into a metric name prefix
+	// using `.metricPrefix()` and is added to each metric returned.
+	CollectorPath collectorPath
+	// List of all metric descriptors that could be returned by the loader.
+	Descriptors []MetricDescriptor
+	// (Optional) Extra (constant) label KV pairs to be added to each metric in
+	// the group.
+	ExtraLabels map[string]string
+
+	// Loader functions to load metrics. Only one of these will be set. Metrics
+	// returned by these functions must be present in the `Descriptors` list.
+	loader       MetricsLoaderFn
+	bucketLoader BucketMetricsLoaderFn
+
+	// Cache for all metrics groups. Set via `.SetCache` method.
+	cache *metricsCache
+
+	// managed values follow:
+
+	// map of metric descriptors by metric name.
+	descriptorMap map[MetricName]MetricDescriptor
+
+	// For bucket metrics, the list of buckets is stored here. It is used in the
+	// Collect() call. This is protected by the `bucketsLock`.
+	bucketsLock sync.Mutex
+	buckets     []string
+}
+
+// NewMetricsGroup creates a new MetricsGroup. To create a metrics group for
+// metrics with a `bucket` dimension (label), use `NewBucketMetricsGroup`.
+//
+// The `loader` function loads metrics from the cache and the system.
+func NewMetricsGroup(path collectorPath, descriptors []MetricDescriptor,
+	loader MetricsLoaderFn,
+) *MetricsGroup {
+	mg := &MetricsGroup{
+		CollectorPath: path,
+		Descriptors:   descriptors,
+		loader:        loader,
+	}
+	mg.validate()
+	return mg
+}
+
+// NewBucketMetricsGroup creates a new MetricsGroup for metrics with a `bucket`
+// dimension (label).
+//
+// The `loader` function loads metrics from the cache and the system for a given
+// list of buckets.
+func NewBucketMetricsGroup(path collectorPath, descriptors []MetricDescriptor,
+	loader BucketMetricsLoaderFn,
+) *MetricsGroup {
+	mg := &MetricsGroup{
+		CollectorPath: path,
+		Descriptors:   descriptors,
+		bucketLoader:  loader,
+	}
+	mg.validate()
+	return mg
+}
+
+// AddExtraLabels - adds extra (constant) label KV pairs to the metrics group.
+// This is a helper to initialize the `ExtraLabels` field. The argument is a
+// list of ordered label name and value pairs.
+func (mg *MetricsGroup) AddExtraLabels(labels ...string) {
+	if len(labels)%2 != 0 {
+		panic("Labels must be an ordered list of name value pairs")
+	}
+	if mg.ExtraLabels == nil {
+		mg.ExtraLabels = make(map[string]string, len(labels))
+	}
+	for i := 0; i < len(labels); i += 2 {
+		mg.ExtraLabels[labels[i]] = labels[i+1]
+	}
+}
+
+// IsBucketMetricsGroup - returns true if the given MetricsGroup is a bucket
+// metrics group.
+func (mg *MetricsGroup) IsBucketMetricsGroup() bool {
+	return mg.bucketLoader != nil
+}
+
+// Describe - implements prometheus.Collector interface.
+func (mg *MetricsGroup) Describe(ch chan<- *prometheus.Desc) {
+	for _, desc := range mg.Descriptors {
+		ch <- desc.toPromDesc(mg.CollectorPath.metricPrefix(), mg.ExtraLabels)
+	}
+}
+
+// Collect - implements prometheus.Collector interface.
+func (mg *MetricsGroup) Collect(ch chan<- prometheus.Metric) {
+	metricValues := newMetricValues(mg.descriptorMap)
+
+	var err error
+	if mg.IsBucketMetricsGroup() {
+		err = mg.bucketLoader(GlobalContext, metricValues, mg.cache, mg.buckets)
+	} else {
+		err = mg.loader(GlobalContext, metricValues, mg.cache)
+	}
+
+	// There is no way to handle errors here, so we panic the current goroutine
+	// and the Metrics API handler returns a 500 HTTP status code. This should
+	// normally not happen, and usually indicates a bug.
+	logger.CriticalIf(GlobalContext, errors.Wrap(err, "failed to get metrics"))
+
+	promMetrics := metricValues.ToPromMetrics(mg.CollectorPath.metricPrefix(),
+		mg.ExtraLabels)
+	for _, metric := range promMetrics {
+		ch <- metric
+	}
+}
+
+// LockAndSetBuckets - locks the buckets and sets the given buckets. It returns
+// a function to unlock the buckets.
+func (mg *MetricsGroup) LockAndSetBuckets(buckets []string) func() {
+	mg.bucketsLock.Lock()
+	mg.buckets = buckets
+	return func() {
+		mg.bucketsLock.Unlock()
+	}
+}
+
+// MetricFQN - returns the fully qualified name for the given metric name.
+func (mg *MetricsGroup) MetricFQN(name MetricName) string {
+	v, ok := mg.descriptorMap[name]
+	if !ok {
+		// This should never happen.
+		return ""
+	}
+	return v.toPromName(mg.CollectorPath.metricPrefix())
+}
+
+func (mg *MetricsGroup) validate() {
+	if len(mg.Descriptors) == 0 {
+		panic("Descriptors must be set")
+	}
+
+	// For bools A and B, A XOR B <=> A != B.
+	isExactlyOneSet := (mg.loader == nil) != (mg.bucketLoader == nil)
+	if !isExactlyOneSet {
+		panic("Exactly one Loader function must be set")
+	}
+
+	mg.descriptorMap = make(map[MetricName]MetricDescriptor, len(mg.Descriptors))
+	for _, desc := range mg.Descriptors {
+		mg.descriptorMap[desc.Name] = desc
+	}
+}
+
+// SetCache is a helper to initialize MetricsGroup. It sets the cache object.
+func (mg *MetricsGroup) SetCache(c *metricsCache) {
+	mg.cache = c
+}

cmd/metrics-v3.go

@@ -0,0 +1,272 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+)
+
+// Collector paths.
+//
+// These are paths under the top-level /minio/metrics/v3 metrics endpoint. Each
+// of these paths returns a set of V3 metrics.
+const (
+	apiRequestsCollectorPath collectorPath = "/api/requests"
+	apiBucketCollectorPath   collectorPath = "/api/bucket"
+
+	systemNetworkInternodeCollectorPath collectorPath = "/system/network/internode"
+	systemDriveCollectorPath            collectorPath = "/system/drive"
+	systemProcessCollectorPath          collectorPath = "/system/process"
+	systemGoCollectorPath               collectorPath = "/system/go"
+
+	clusterHealthCollectorPath       collectorPath = "/cluster/health"
+	clusterUsageObjectsCollectorPath collectorPath = "/cluster/usage/objects"
+	clusterUsageBucketsCollectorPath collectorPath = "/cluster/usage/buckets"
+	clusterErasureSetCollectorPath   collectorPath = "/cluster/erasure-set"
+)
+
+const (
+	clusterBasePath = "/cluster"
+)
+
+type metricsV3Collection struct {
+	mgMap       map[collectorPath]*MetricsGroup
+	bucketMGMap map[collectorPath]*MetricsGroup
+
+	// Gatherers for non-bucket MetricsGroup's
+	mgGatherers map[collectorPath]prometheus.Gatherer
+
+	collectorPaths []collectorPath
+}
+
+func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
+	// Create all metric groups.
+	apiRequestsMG := NewMetricsGroup(apiRequestsCollectorPath,
+		[]MetricDescriptor{
+			apiRejectedAuthTotalMD,
+			apiRejectedHeaderTotalMD,
+			apiRejectedTimestampTotalMD,
+			apiRejectedInvalidTotalMD,
+
+			apiRequestsWaitingTotalMD,
+			apiRequestsIncomingTotalMD,
+			apiRequestsInFlightTotalMD,
+			apiRequestsTotalMD,
+			apiRequestsErrorsTotalMD,
+			apiRequests5xxErrorsTotalMD,
+			apiRequests4xxErrorsTotalMD,
+			apiRequestsCanceledTotalMD,
+
+			apiRequestsTTFBSecondsDistributionMD,
+
+			apiTrafficSentBytesMD,
+			apiTrafficRecvBytesMD,
+		},
+		JoinLoaders(loadAPIRequestsHTTPMetrics, loadAPIRequestsTTFBMetrics,
+			loadAPIRequestsNetworkMetrics),
+	)
+
+	apiBucketMG := NewBucketMetricsGroup(apiBucketCollectorPath,
+		[]MetricDescriptor{
+			apiBucketTrafficRecvBytesMD,
+			apiBucketTrafficSentBytesMD,
+
+			apiBucketRequestsInFlightMD,
+			apiBucketRequestsTotalMD,
+			apiBucketRequestsCanceledMD,
+			apiBucketRequests4xxErrorsMD,
+			apiBucketRequests5xxErrorsMD,
+
+			apiBucketRequestsTTFBSecondsDistributionMD,
+		},
+		JoinBucketLoaders(loadAPIBucketHTTPMetrics, loadAPIBucketTTFBMetrics),
+	)
+
+	systemNetworkInternodeMG := NewMetricsGroup(systemNetworkInternodeCollectorPath,
+		[]MetricDescriptor{
+			internodeErrorsTotalMD,
+			internodeDialedErrorsTotalMD,
+			internodeDialAvgTimeNanosMD,
+			internodeSentBytesTotalMD,
+			internodeRecvBytesTotalMD,
+		},
+		loadNetworkInternodeMetrics,
+	)
+
+	systemDriveMG := NewMetricsGroup(systemDriveCollectorPath,
+		[]MetricDescriptor{
+			driveUsedBytesMD,
+			driveFreeBytesMD,
+			driveTotalBytesMD,
+			driveFreeInodesMD,
+			driveTimeoutErrorsMD,
+			driveAvailabilityErrorsMD,
+			driveWaitingIOMD,
+			driveAPILatencyMD,
+
+			driveOfflineCountMD,
+			driveOnlineCountMD,
+			driveCountMD,
+		},
+		loadDriveMetrics,
+	)
+
+	clusterHealthMG := NewMetricsGroup(clusterHealthCollectorPath,
+		[]MetricDescriptor{
+			healthDrivesOfflineCountMD,
+			healthDrivesOnlineCountMD,
+			healthDrivesCountMD,
+
+			healthNodesOfflineCountMD,
+			healthNodesOnlineCountMD,
+
+			healthCapacityRawTotalBytesMD,
+			healthCapacityRawFreeBytesMD,
+			healthCapacityUsableTotalBytesMD,
+			healthCapacityUsableFreeBytesMD,
+		},
+		JoinLoaders(loadClusterHealthDriveMetrics,
+			loadClusterHealthNodeMetrics,
+			loadClusterHealthCapacityMetrics),
+	)
+
+	clusterUsageObjectsMG := NewMetricsGroup(clusterUsageObjectsCollectorPath,
+		[]MetricDescriptor{
+			usageSinceLastUpdateSecondsMD,
+			usageTotalBytesMD,
+			usageObjectsCountMD,
+			usageVersionsCountMD,
+			usageDeleteMarkersCountMD,
+			usageBucketsCountMD,
+			usageObjectsDistributionMD,
+			usageVersionsDistributionMD,
+		},
+		loadClusterUsageObjectMetrics,
+	)
+
+	clusterUsageBucketsMG := NewBucketMetricsGroup(clusterUsageBucketsCollectorPath,
+		[]MetricDescriptor{
+			usageSinceLastUpdateSecondsMD,
+			usageBucketTotalBytesMD,
+			usageBucketObjectsTotalMD,
+			usageBucketVersionsCountMD,
+			usageBucketDeleteMarkersCountMD,
+			usageBucketQuotaTotalBytesMD,
+			usageBucketObjectSizeDistributionMD,
+			usageBucketObjectVersionCountDistributionMD,
+		},
+		loadClusterUsageBucketMetrics,
+	)
+
+	clusterErasureSetMG := NewMetricsGroup(clusterErasureSetCollectorPath,
+		[]MetricDescriptor{
+			erasureSetOverallWriteQuorumMD,
+			erasureSetOverallHealthMD,
+			erasureSetReadQuorumMD,
+			erasureSetWriteQuorumMD,
+			erasureSetOnlineDrivesCountMD,
+			erasureSetHealingDrivesCountMD,
+			erasureSetHealthMD,
+		},
+		loadClusterErasureSetMetrics,
+	)
+
+	allMetricGroups := []*MetricsGroup{
+		apiRequestsMG,
+		apiBucketMG,
+
+		systemNetworkInternodeMG,
+		systemDriveMG,
+
+		clusterHealthMG,
+		clusterUsageObjectsMG,
+		clusterUsageBucketsMG,
+		clusterErasureSetMG,
+	}
+
+	// Bucket metrics are special, they always include the bucket label. These
+	// metrics required a list of buckets to be passed to the loader, and the list
+	// of buckets is not known until the request is made. So we keep a separate
+	// map for bucket metrics and handle them specially.
+
+	// Add the serverName and poolIndex labels to all non-cluster metrics.
+	//
+	// Also create metric group maps and set the cache.
+	metricsCache := newMetricsCache()
+	mgMap := make(map[collectorPath]*MetricsGroup)
+	bucketMGMap := make(map[collectorPath]*MetricsGroup)
+	for _, mg := range allMetricGroups {
+		if !strings.HasPrefix(string(mg.CollectorPath), clusterBasePath) {
+			mg.AddExtraLabels(
+				serverName, globalLocalNodeName,
+				// poolIndex, strconv.Itoa(globalLocalPoolIdx),
+			)
+		}
+		mg.SetCache(metricsCache)
+		if mg.IsBucketMetricsGroup() {
+			bucketMGMap[mg.CollectorPath] = mg
+		} else {
+			mgMap[mg.CollectorPath] = mg
+		}
+	}
+
+	// Prepare to register the collectors. Other than `MetricGroup` collectors,
+	// we also have standard collectors like `ProcessCollector` and `GoCollector`.
+
+	// Create all Non-`MetricGroup` collectors here.
+	collectors := map[collectorPath]prometheus.Collector{
+		systemProcessCollectorPath: collectors.NewProcessCollector(collectors.ProcessCollectorOpts{
+			ReportErrors: true,
+		}),
+		systemGoCollectorPath: collectors.NewGoCollector(),
+	}
+
+	// Add all `MetricGroup` collectors to the map.
+	for _, mg := range allMetricGroups {
+		collectors[mg.CollectorPath] = mg
+	}
+
+	// Helper function to register a collector and return a gatherer for it.
+	mustRegister := func(c ...prometheus.Collector) prometheus.Gatherer {
+		subRegistry := prometheus.NewRegistry()
+		for _, col := range c {
+			subRegistry.MustRegister(col)
+		}
+		r.MustRegister(subRegistry)
+		return subRegistry
+	}
+
+	// Register all collectors and create gatherers for them.
+	gatherers := make(map[collectorPath]prometheus.Gatherer, len(collectors))
+	collectorPaths := make([]collectorPath, 0, len(collectors))
+	for path, collector := range collectors {
+		gatherers[path] = mustRegister(collector)
+		collectorPaths = append(collectorPaths, path)
+	}
+	slices.Sort(collectorPaths)
+	return &metricsV3Collection{
+		mgMap:          mgMap,
+		bucketMGMap:    bucketMGMap,
+		mgGatherers:    gatherers,
+		collectorPaths: collectorPaths,
+	}
+}

cmd/metrics.go

@@ -190,7 +190,7 @@ func healingMetricsPrometheus(ch chan<- prometheus.Metric) {
 // collects http metrics for MinIO server in Prometheus specific format
 // and sends to given channel
 func httpMetricsPrometheus(ch chan<- prometheus.Metric) {
-	httpStats := globalHTTPStats.toServerHTTPStats()
+	httpStats := globalHTTPStats.toServerHTTPStats(true)
 
 	for api, value := range httpStats.CurrentS3Requests.APIStats {
 		ch <- prometheus.MustNewConstMetric(

cmd/mrf.go

@@ -19,10 +19,10 @@ package cmd
 
 import (
 	"context"
-	"sync"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/pkg/v2/wildcard"
 )
 
 const (
@@ -44,37 +44,15 @@ type partialOperation struct {
 // mrfState sncapsulates all the information
 // related to the global background MRF.
 type mrfState struct {
-	ctx   context.Context
-	pools *erasureServerPools
-
-	mu   sync.RWMutex
 	opCh chan partialOperation
 }
 
-// Initialize healing MRF subsystem
-func (m *mrfState) init(ctx context.Context, objAPI ObjectLayer) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	m.ctx = ctx
-	m.opCh = make(chan partialOperation, mrfOpsQueueSize)
-
-	var ok bool
-	m.pools, ok = objAPI.(*erasureServerPools)
-	if ok {
-		go m.healRoutine()
-	}
-}
-
 // Add a partial S3 operation (put/delete) when one or more disks are offline.
 func (m *mrfState) addPartialOp(op partialOperation) {
 	if m == nil {
 		return
 	}
 
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
 	select {
 	case m.opCh <- op:
 	default:
@@ -86,16 +64,35 @@ var healSleeper = newDynamicSleeper(5, time.Second, false)
 // healRoutine listens to new disks reconnection events and
 // issues healing requests for queued objects belonging to the
 // corresponding erasure set
-func (m *mrfState) healRoutine() {
+func (m *mrfState) healRoutine(z *erasureServerPools) {
 	for {
 		select {
-		case <-m.ctx.Done():
+		case <-GlobalContext.Done():
 			return
 		case u, ok := <-m.opCh:
 			if !ok {
 				return
 			}
 
+			// We might land at .metacache, .trash, .multipart
+			// no need to heal them skip, only when bucket
+			// is '.minio.sys'
+			if u.bucket == minioMetaBucket {
+				// No MRF needed for temporary objects
+				if wildcard.Match("buckets/*/.metacache/*", u.object) {
+					continue
+				}
+				if wildcard.Match("tmp/*", u.object) {
+					continue
+				}
+				if wildcard.Match("multipart/*", u.object) {
+					continue
+				}
+				if wildcard.Match("tmp-old/*", u.object) {
+					continue
+				}
+			}
+
 			now := time.Now()
 			if now.Sub(u.queued) < time.Second {
 				// let recently failed networks to reconnect
@@ -115,7 +112,7 @@ func (m *mrfState) healRoutine() {
 				healBucket(u.bucket, scan)
 			} else {
 				if u.allVersions {
-					m.pools.serverPools[u.poolIndex].sets[u.setIndex].listAndHeal(u.bucket, u.object, u.scanMode, healObjectVersionsDisparity)
+					z.serverPools[u.poolIndex].sets[u.setIndex].listAndHeal(u.bucket, u.object, u.scanMode, healObjectVersionsDisparity)
 				} else {
 					healObject(u.bucket, u.object, u.versionID, scan)
 				}
@@ -125,8 +122,3 @@ func (m *mrfState) healRoutine() {
 		}
 	}
 }
-
-// Initialize healing MRF
-func initHealMRF(ctx context.Context, obj ObjectLayer) {
-	globalMRFState.init(ctx, obj)
-}

cmd/notification.go

@@ -870,12 +870,12 @@ func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType,
 }
 
 // GetResourceMetrics - gets the resource metrics from all nodes excluding self.
-func (sys *NotificationSys) GetResourceMetrics(ctx context.Context) <-chan Metric {
+func (sys *NotificationSys) GetResourceMetrics(ctx context.Context) <-chan MetricV2 {
 	if sys == nil {
 		return nil
 	}
 	g := errgroup.WithNErrs(len(sys.peerClients))
-	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	peerChannels := make([]<-chan MetricV2, len(sys.peerClients))
 	for index := range sys.peerClients {
 		index := index
 		g.Go(func() error {
@@ -1214,8 +1214,8 @@ func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...
 	return consolidatedReport
 }
 
-func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels []<-chan Metric, g *errgroup.Group) <-chan Metric {
-	ch := make(chan Metric)
+func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels []<-chan MetricV2, g *errgroup.Group) <-chan MetricV2 {
+	ch := make(chan MetricV2)
 	var wg sync.WaitGroup
 	for index, err := range g.Wait() {
 		if err != nil {
@@ -1229,7 +1229,7 @@ func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels
 			continue
 		}
 		wg.Add(1)
-		go func(ctx context.Context, peerChannel <-chan Metric, wg *sync.WaitGroup) {
+		go func(ctx context.Context, peerChannel <-chan MetricV2, wg *sync.WaitGroup) {
 			defer wg.Done()
 			for {
 				select {
@@ -1248,7 +1248,7 @@ func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels
 			}
 		}(ctx, peerChannels[index], &wg)
 	}
-	go func(wg *sync.WaitGroup, ch chan Metric) {
+	go func(wg *sync.WaitGroup, ch chan MetricV2) {
 		wg.Wait()
 		xioutil.SafeClose(ch)
 	}(&wg, ch)
@@ -1256,12 +1256,12 @@ func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels
 }
 
 // GetBucketMetrics - gets the cluster level bucket metrics from all nodes excluding self.
-func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric {
+func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan MetricV2 {
 	if sys == nil {
 		return nil
 	}
 	g := errgroup.WithNErrs(len(sys.peerClients))
-	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	peerChannels := make([]<-chan MetricV2, len(sys.peerClients))
 	for index := range sys.peerClients {
 		index := index
 		g.Go(func() error {
@@ -1277,12 +1277,12 @@ func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric
 }
 
 // GetClusterMetrics - gets the cluster metrics from all nodes excluding self.
-func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric {
+func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan MetricV2 {
 	if sys == nil {
 		return nil
 	}
 	g := errgroup.WithNErrs(len(sys.peerClients))
-	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	peerChannels := make([]<-chan MetricV2, len(sys.peerClients))
 	for index := range sys.peerClients {
 		index := index
 		g.Go(func() error {

cmd/object-api-datatypes.go

@@ -30,6 +30,8 @@ import (
 	"github.com/minio/minio/internal/logger"
 )
 
+//go:generate msgp -file $GOFILE -io=false -tests=false -unexported=false
+
 // BackendType - represents different backend types.
 type BackendType int
 
@@ -187,9 +189,9 @@ type ObjectInfo struct {
 	Parts []ObjectPartInfo `json:"-"`
 
 	// Implements writer and reader used by CopyObject API
-	Writer       io.WriteCloser `json:"-"`
-	Reader       *hash.Reader   `json:"-"`
-	PutObjReader *PutObjReader  `json:"-"`
+	Writer       io.WriteCloser `json:"-" msg:"-"`
+	Reader       *hash.Reader   `json:"-" msg:"-"`
+	PutObjReader *PutObjReader  `json:"-" msg:"-"`
 
 	metadataOnly bool
 	versionOnly  bool // adds a new version, only used by CopyObject

cmd/object-api-interface.go

@@ -32,6 +32,10 @@ import (
 	xioutil "github.com/minio/minio/internal/ioutil"
 )
 
+//go:generate msgp -file $GOFILE -io=false -tests=false -unexported=false
+
+//msgp:ignore ObjectOptions TransitionOptions DeleteBucketOptions
+
 // CheckPreconditionFn returns true if precondition check failed.
 type CheckPreconditionFn func(o ObjectInfo) bool
 
@@ -117,12 +121,19 @@ type ObjectOptions struct {
 	// Object must have been read at this point.
 	IndexCB func() []byte
 
+	// InclFreeVersions indicates that free versions need to be included
+	// when looking up a version by fi.VersionID
 	InclFreeVersions bool
+	// SkipFreeVersion skips adding a free version when a tiered version is
+	// being 'replaced'
+	// Note: Used only when a tiered object is being expired.
+	SkipFreeVersion bool
 
 	MetadataChg           bool                  // is true if it is a metadata update operation.
 	EvalRetentionBypassFn EvalRetentionBypassFn // only set for enforcing retention bypass on DeleteObject.
 
 	FastGetObjInfo bool // Only for S3 Head/Get Object calls for now
+	NoAuditLog     bool // Only set for decom, rebalance, to avoid double audits.
 }
 
 // WalkOptions provides filtering, marker and other Walk() specific options.

cmd/object-api-interface_gen.go

@@ -0,0 +1,319 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// MarshalMsg implements msgp.Marshaler
+func (z BucketOptions) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Deleted"
+	o = append(o, 0x82, 0xa7, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.Deleted)
+	// string "Cached"
+	o = append(o, 0xa6, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.Cached)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Deleted":
+			z.Deleted, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Deleted")
+				return
+			}
+		case "Cached":
+			z.Cached, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Cached")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BucketOptions) Msgsize() (s int) {
+	s = 1 + 8 + msgp.BoolSize + 7 + msgp.BoolSize
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z ExpirationOptions) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "Expire"
+	o = append(o, 0x81, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
+	o = msgp.AppendBool(o, z.Expire)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Expire":
+			z.Expire, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Expire")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z ExpirationOptions) Msgsize() (s int) {
+	s = 1 + 7 + msgp.BoolSize
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *MakeBucketOptions) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 5
+	// string "LockEnabled"
+	o = append(o, 0x85, 0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.LockEnabled)
+	// string "VersioningEnabled"
+	o = append(o, 0xb1, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.VersioningEnabled)
+	// string "ForceCreate"
+	o = append(o, 0xab, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65)
+	o = msgp.AppendBool(o, z.ForceCreate)
+	// string "CreatedAt"
+	o = append(o, 0xa9, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.CreatedAt)
+	// string "NoLock"
+	o = append(o, 0xa6, 0x4e, 0x6f, 0x4c, 0x6f, 0x63, 0x6b)
+	o = msgp.AppendBool(o, z.NoLock)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MakeBucketOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "LockEnabled":
+			z.LockEnabled, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LockEnabled")
+				return
+			}
+		case "VersioningEnabled":
+			z.VersioningEnabled, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "VersioningEnabled")
+				return
+			}
+		case "ForceCreate":
+			z.ForceCreate, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ForceCreate")
+				return
+			}
+		case "CreatedAt":
+			z.CreatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "CreatedAt")
+				return
+			}
+		case "NoLock":
+			z.NoLock, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NoLock")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *MakeBucketOptions) Msgsize() (s int) {
+	s = 1 + 12 + msgp.BoolSize + 18 + msgp.BoolSize + 12 + msgp.BoolSize + 10 + msgp.TimeSize + 7 + msgp.BoolSize
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *WalkOptions) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 4
+	// string "Marker"
+	o = append(o, 0x84, 0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72)
+	o = msgp.AppendString(o, z.Marker)
+	// string "LatestOnly"
+	o = append(o, 0xaa, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x79)
+	o = msgp.AppendBool(o, z.LatestOnly)
+	// string "AskDisks"
+	o = append(o, 0xa8, 0x41, 0x73, 0x6b, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	o = msgp.AppendString(o, z.AskDisks)
+	// string "VersionsSort"
+	o = append(o, 0xac, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x53, 0x6f, 0x72, 0x74)
+	o = msgp.AppendUint8(o, uint8(z.VersionsSort))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *WalkOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Marker":
+			z.Marker, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Marker")
+				return
+			}
+		case "LatestOnly":
+			z.LatestOnly, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LatestOnly")
+				return
+			}
+		case "AskDisks":
+			z.AskDisks, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "AskDisks")
+				return
+			}
+		case "VersionsSort":
+			{
+				var zb0002 uint8
+				zb0002, bts, err = msgp.ReadUint8Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "VersionsSort")
+					return
+				}
+				z.VersionsSort = WalkVersionsSortOrder(zb0002)
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *WalkOptions) Msgsize() (s int) {
+	s = 1 + 7 + msgp.StringPrefixSize + len(z.Marker) + 11 + msgp.BoolSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 13 + msgp.Uint8Size
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z WalkVersionsSortOrder) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint8(o, uint8(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *WalkVersionsSortOrder) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 uint8
+		zb0001, bts, err = msgp.ReadUint8Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = WalkVersionsSortOrder(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z WalkVersionsSortOrder) Msgsize() (s int) {
+	s = msgp.Uint8Size
+	return
+}

cmd/object-api-utils.go

@@ -577,22 +577,35 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi
 	}
 
 	// Filter compression includes.
-	exclude := len(cfg.Extensions) > 0 || len(cfg.MimeTypes) > 0
+	if len(cfg.Extensions) == 0 && len(cfg.MimeTypes) == 0 {
+		// Nothing to filter, include everything.
+		return false
+	}
+
 	if len(cfg.Extensions) > 0 && hasStringSuffixInSlice(objStr, cfg.Extensions) {
-		exclude = false
+		// Matched an extension to compress, do not exclude.
+		return false
 	}
 
 	if len(cfg.MimeTypes) > 0 && hasPattern(cfg.MimeTypes, contentType) {
-		exclude = false
+		// Matched an MIME type to compress, do not exclude.
+		return false
 	}
-	return exclude
+
+	// Did not match any inclusion filters, exclude from compression.
+	return true
 }
 
 // Utility which returns if a string is present in the list.
-// Comparison is case insensitive.
+// Comparison is case insensitive. Explicit short-circuit if
+// the list contains the wildcard "*".
 func hasStringSuffixInSlice(str string, list []string) bool {
 	str = strings.ToLower(str)
 	for _, v := range list {
+		if v == "*" {
+			return true
+		}
+
 		if strings.HasSuffix(str, strings.ToLower(v)) {
 			return true
 		}

cmd/object-handlers-common.go

@@ -138,7 +138,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 //	x-minio-source-etag
 func checkPreconditionsPUT(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, opts ObjectOptions) bool {
 	// Return false for methods other than PUT.
-	if r.Method != http.MethodPut {
+	if r.Method != http.MethodPut && r.Method != http.MethodPost {
 		return false
 	}
 	// If the object doesn't have a modtime (IsZero), or the modtime

cmd/object-handlers.go

@@ -1721,7 +1721,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if dstOpts.ReplicationRequest {
 			srcTimestamp := dstOpts.ReplicationSourceTaggingTimestamp
 			if !srcTimestamp.IsZero() {
-				ondiskTimestamp, err := time.Parse(lastTaggingTimestamp, time.RFC3339Nano)
+				ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastTaggingTimestamp)
 				// update tagging metadata only if replica  timestamp is newer than what's on disk
 				if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) {
 					srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano)
@@ -1748,7 +1748,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if dstOpts.ReplicationRequest {
 			srcTimestamp := dstOpts.ReplicationSourceRetentionTimestamp
 			if !srcTimestamp.IsZero() {
-				ondiskTimestamp, err := time.Parse(lastretentionTimestamp, time.RFC3339Nano)
+				ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastretentionTimestamp)
 				// update retention metadata only if replica  timestamp is newer than what's on disk
 				if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) {
 					srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
@@ -1768,7 +1768,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if dstOpts.ReplicationRequest {
 			srcTimestamp := dstOpts.ReplicationSourceLegalholdTimestamp
 			if !srcTimestamp.IsZero() {
-				ondiskTimestamp, err := time.Parse(lastLegalHoldTimestamp, time.RFC3339Nano)
+				ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastLegalHoldTimestamp)
 				// update legalhold metadata only if replica timestamp is newer than what's on disk
 				if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) {
 					srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)

cmd/object-handlers_test.go

@@ -1851,7 +1851,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 		// expected output.
 		expectedRespStatus int
 	}{
-		// Test case - 1, copy part 1 from from newObject1, ignore request headers.
+		// Test case - 1, copy part 1 from newObject1, ignore request headers.
 		{
 			bucketName:         bucketName,
 			uploadID:           uploadID,
@@ -2011,7 +2011,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusOK,
 		},
-		// Test case - 14, copy part 1 from from newObject1 with null versionId
+		// Test case - 14, copy part 1 from newObject1 with null versionId
 		{
 			bucketName:         bucketName,
 			uploadID:           uploadID,
@@ -2020,7 +2020,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusOK,
 		},
-		// Test case - 15, copy part 1 from from newObject1 with non null versionId
+		// Test case - 15, copy part 1 from newObject1 with non null versionId
 		{
 			bucketName:         bucketName,
 			uploadID:           uploadID,