feat: Allow at most one claim based OpenID IDP (#16145)

Signed-off-by: Andreas Auernhammer <hi@aead.dev>

.github/workflows/go-healing.yml

@@ -28,14 +28,6 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:

.github/workflows/go-lint.yml

@@ -28,30 +28,13 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - uses: actions/cache@v2
-        if: matrix.os == 'ubuntu-latest'
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
-      - uses: actions/cache@v2
-        if: matrix.os == 'windows-latest'
-        with:
-          path: |
-            %LocalAppData%\go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'windows-latest'
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
         run: |
+          netsh int ipv4 set dynamicport tcp start=60000 num=61000
           go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
           go test -v --timeout 50m ./...
       - name: Build on ${{ matrix.os }}
@@ -63,9 +46,6 @@ jobs:
           sudo apt install jq -y
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
-          nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
-          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
-          go list -deps -json ./... | jq -s 'unique_by(.Module.Path)|.[]|select(has("Module"))|.Module' | ./nancy sleuth
           make
           make test
           make test-race

.github/workflows/go.yml

@@ -28,14 +28,6 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:

.github/workflows/iam-integrations.yaml

@@ -80,14 +80,6 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
       - name: Test LDAP/OpenID/Etcd combo
         env:
           LDAP_TEST_SERVER: ${{ matrix.ldap }}

.github/workflows/replication.yaml

@@ -29,14 +29,6 @@ jobs:
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
       - name: Test Decom
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

.gitignore

@@ -25,7 +25,7 @@ mc.*
 s3-check-md5*
 xl-meta*
 healing-*
-inspect*
+inspect*.zip
 200M*
 hash-set
 minio.RELEASE*

.golangci.yml

@@ -1,14 +1,7 @@
 linters-settings:
-  golint:
-    min-confidence: 0
-
   gofumpt:
     lang-version: "1.18"
 
-    # Choose whether or not to use the extra rules that are disabled
-    # by default
-    extra-rules: false
-
   misspell:
     locale: US
 

.nancy-ignore

@@ -1,6 +0,0 @@
-CVE-2020-26160
-CVE-2020-15136
-CVE-2020-15115
-CVE-2020-15114
-CVE-2022-2835
-CVE-2022-2837

CREDITS

@@ -31,6 +31,33 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
+aead.dev/mem
+https://aead.dev/mem
+----------------------------------------------------------------
+MIT License
+
+Copyright (c) 2022 Andreas Auernhammer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 cloud.google.com/go
 https://cloud.google.com/go
 ----------------------------------------------------------------
@@ -2567,6 +2594,33 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ================================================================
 
+github.com/aymanbagabas/go-osc52
+https://github.com/aymanbagabas/go-osc52
+----------------------------------------------------------------
+MIT License
+
+Copyright (c) 2022 Ayman Bagabas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 github.com/bcicen/jstream
 https://github.com/bcicen/jstream
 ----------------------------------------------------------------
@@ -4514,213 +4568,6 @@ https://github.com/docker/go-units
 
 ================================================================
 
-github.com/dswarbrick/smart
-https://github.com/dswarbrick/smart
-----------------------------------------------------------------
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/dustin/go-humanize
 https://github.com/dustin/go-humanize
 ----------------------------------------------------------------
@@ -14610,6 +14457,33 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 ================================================================
 
+github.com/lithammer/shortuuid/v4
+https://github.com/lithammer/shortuuid/v4
+----------------------------------------------------------------
+The MIT License (MIT)
+
+Copyright (c) 2018 Peter Lithammer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 github.com/lucasb-eyer/go-colorful
 https://github.com/lucasb-eyer/go-colorful
 ----------------------------------------------------------------
@@ -24108,6 +23982,37 @@ https://github.com/prometheus/prom2json
 
 ================================================================
 
+github.com/rabbitmq/amqp091-go
+https://github.com/rabbitmq/amqp091-go
+----------------------------------------------------------------
+AMQP 0-9-1 Go Client
+Copyright (c) 2021 VMware, Inc. or its affiliates. All Rights Reserved.
+
+Copyright (c) 2012-2021, Sean Treadway, SoundCloud Ltd.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/rcrowley/go-metrics
 https://github.com/rcrowley/go-metrics
 ----------------------------------------------------------------
@@ -24454,35 +24359,6 @@ THE SOFTWARE.
 
 ================================================================
 
-github.com/streadway/amqp
-https://github.com/streadway/amqp
-----------------------------------------------------------------
-Copyright (c) 2012-2019, Sean Treadway, SoundCloud Ltd.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice, this
-list of conditions and the following disclaimer in the documentation and/or
-other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-================================================================
-
 github.com/stretchr/testify
 https://github.com/stretchr/testify
 ----------------------------------------------------------------
@@ -24606,7 +24482,7 @@ https://github.com/tklauser/go-sysconf
 ----------------------------------------------------------------
 BSD 3-Clause License
 
-Copyright (c) 2018-2021, Tobias Klauser
+Copyright (c) 2018-2022, Tobias Klauser
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
 }
 
 function _build() {

cmd/admin-bucket-handlers.go

@@ -160,11 +160,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	bucket := pathClean(vars["bucket"])
 	update := r.Form.Get("update") == "true"
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
@@ -289,10 +284,6 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	bucket := pathClean(vars["bucket"])
 	arnType := vars["type"]
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
 	if objectAPI == nil {
@@ -328,10 +319,6 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	bucket := pathClean(vars["bucket"])
 	arn := vars["arn"]
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
@@ -373,10 +360,6 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	bucket := pathClean(r.Form.Get("bucket"))
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
 	if objectAPI == nil {
@@ -661,10 +644,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
 	if objectAPI == nil {
@@ -1091,11 +1070,6 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
 	if objectAPI == nil {
 		return

cmd/admin-handler-utils.go

@@ -84,7 +84,13 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			Description:    e.Error(),
 			HTTPStatusCode: http.StatusBadRequest,
 		}
-	case config.Error:
+	case config.ErrConfigNotFound:
+		apiErr = APIError{
+			Code:           "XMinioConfigNotFoundError",
+			Description:    e.Error(),
+			HTTPStatusCode: http.StatusNotFound,
+		}
+	case config.ErrConfigGeneric:
 		apiErr = APIError{
 			Code:           "XMinioConfigError",
 			Description:    e.Error(),
@@ -124,6 +130,18 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case errors.Is(err, errDecommissionRebalanceAlreadyRunning):
+			apiErr = APIError{
+				Code:           "XMinioDecommissionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, errRebalanceDecommissionAlreadyRunning):
+			apiErr = APIError{
+				Code:           "XMinioRebalanceNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errConfigNotFound):
 			apiErr = APIError{
 				Code:           "XMinioConfigError",

cmd/admin-handlers-config-kv.go

@@ -225,7 +225,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	var s strings.Builder
 	for _, subSysConfig := range subSysConfigs {
-		subSysConfig.AddString(&s, false)
+		subSysConfig.WriteTo(&s, false)
 	}
 
 	password := cred.SecretKey
@@ -454,10 +454,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	var s strings.Builder
 	hkvs := config.HelpSubSysMap[""]
-	var count int
-	for _, hkv := range hkvs {
-		count += len(cfg[hkv.Key])
-	}
 	for _, hkv := range hkvs {
 		// We ignore the error below, as we cannot get one.
 		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
@@ -482,7 +478,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			case config.IdentityPluginSubSys:
 				off = !idplugin.Enabled(item.Config)
 			}
-			item.AddString(&s, off)
+			item.WriteTo(&s, off)
 		}
 	}
 

cmd/admin-handlers-idp-config.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,22 +30,14 @@ import (
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
+	cfgldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/ldap"
 )
 
-// List of implemented ID config types.
-var idCfgTypes = set.CreateStringSet("openid")
-
-// SetIdentityProviderCfg:
-//
-// PUT <admin-prefix>/id-cfg?type=openid&name=dex1
-func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetIdentityCfg")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
+func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
@@ -56,6 +49,14 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
 		return
 	}
 
+	// Ensure body content type is opaque to ensure that request body has not
+	// been interpreted as form data.
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/octet-stream" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+		return
+	}
+
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
@@ -64,42 +65,48 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
 		return
 	}
 
-	cfgType := mux.Vars(r)["type"]
-	if !idCfgTypes.Contains(cfgType) {
-		// TODO: change this to invalid type error when implementation
-		// is complete.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	idpCfgType := mux.Vars(r)["type"]
+	if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
 		return
 	}
 
-	var cfgDataBuilder strings.Builder
-	switch cfgType {
-	case "openid":
-		fmt.Fprintf(&cfgDataBuilder, "identity_openid")
+	var subSys string
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
+		subSys = madmin.IdentityOpenIDSubSys
+	case madmin.LDAPIDPCfg:
+		subSys = madmin.IdentityLDAPSubSys
 	}
 
-	// Ensure body content type is opaque.
-	contentType := r.Header.Get("Content-Type")
-	if contentType != "application/octet-stream" {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
-		return
-	}
-
-	// Subsystem configuration name could be empty.
 	cfgName := mux.Vars(r)["name"]
+	cfgTarget := madmin.Default
 	if cfgName != "" {
-		fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
+		cfgTarget = cfgName
+		if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
+			// LDAP does not support multiple configurations. So cfgName must be
+			// empty or `madmin.Default`.
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+			return
+		}
 	}
 
-	fmt.Fprintf(&cfgDataBuilder, "%s%s", config.KvSpaceSeparator, string(reqBytes))
-
-	cfgData := cfgDataBuilder.String()
-	subSys, _, _, err := config.GetSubSys(cfgData)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+	// Check that this is a valid Create vs Update API call.
+	s := globalServerConfig.Clone()
+	if apiErrCode := handleCreateUpdateValidation(s, subSys, cfgTarget, isUpdate); apiErrCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
 		return
 	}
 
+	cfgData := ""
+	{
+		tgtSuffix := ""
+		if cfgTarget != madmin.Default {
+			tgtSuffix = config.SubSystemSeparator + cfgTarget
+		}
+		cfgData = subSys + tgtSuffix + config.KvSpaceSeparator + string(reqBytes)
+	}
+
 	cfg, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -119,6 +126,16 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	}
 
 	if err = validateConfig(cfg, subSys); err != nil {
+
+		var validationErr ldap.Validation
+		if errors.As(err, &validationErr) {
+			// If we got an LDAP validation error, we need to send appropriate
+			// error message back to client (likely mc).
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
+				validationErr.FormatError(), r.URL)
+			return
+		}
+
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -138,86 +155,101 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// GetIdentityProviderCfg:
-//
-// GET <admin-prefix>/id-cfg?type=openid&name=dex_test
-//
-// GetIdentityProviderCfg returns a list of configured IDPs on the server if
-// name is empty. If name is non-empty, returns the configuration details for
-// the IDP of the given type and configuration name. The configuration name for
-// the default ("un-named") configuration target is `_`.
-func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetIdentityProviderCfg")
+func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isUpdate bool) APIErrorCode {
+	if cfgTarget != madmin.Default {
+		// This cannot give an error at this point.
+		subSysTargets, _ := s.GetAvailableTargets(subSys)
+		subSysTargetsSet := set.CreateStringSet(subSysTargets...)
+		if isUpdate && !subSysTargetsSet.Contains(cfgTarget) {
+			return ErrAdminConfigIDPCfgNameDoesNotExist
+		}
+		if !isUpdate && subSysTargetsSet.Contains(cfgTarget) {
+			return ErrAdminConfigIDPCfgNameAlreadyExists
+		}
 
+		return ErrNone
+	}
+
+	// For the default configuration name, since it will always be an available
+	// target, we need to check if a configuration value has been set previously
+	// to figure out if this is a valid create or update API call.
+
+	// This cannot really error (FIXME: improve the type for GetConfigInfo)
+	var cfgInfos []madmin.IDPCfgInfo
+	switch subSys {
+	case madmin.IdentityOpenIDSubSys:
+		cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
+	case madmin.IdentityLDAPSubSys:
+		cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
+	}
+
+	if len(cfgInfos) > 0 && !isUpdate {
+		return ErrAdminConfigIDPCfgNameAlreadyExists
+	}
+	if len(cfgInfos) == 0 && isUpdate {
+		return ErrAdminConfigIDPCfgNameDoesNotExist
+	}
+	return ErrNone
+}
+
+// AddIdentityProviderCfg: adds a new IDP config for openid/ldap.
+//
+// PUT <admin-prefix>/idp-cfg/openid/dex1 -> create named config `dex1`
+//
+// PUT <admin-prefix>/idp-cfg/openid/_ -> create (default) named config `_`
+func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddIdentityProviderCfg")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	a.addOrUpdateIDPHandler(ctx, w, r, false)
+}
+
+// UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
+//
+// PATCH <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
+//
+// PATCH <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
+func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "UpdateIdentityProviderCfg")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	a.addOrUpdateIDPHandler(ctx, w, r, true)
+}
+
+// ListIdentityProviderCfg:
+//
+// GET <admin-prefix>/idp-cfg/openid -> lists openid provider configs.
+func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListIdentityProviderCfg")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
-
-	cfgType := mux.Vars(r)["type"]
-	cfgName := r.Form.Get("name")
 	password := cred.SecretKey
 
-	if !idCfgTypes.Contains(cfgType) {
-		// TODO: change this to invalid type error when implementation
-		// is complete.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	idpCfgType := mux.Vars(r)["type"]
+	if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
 		return
 	}
 
-	// If no cfgName is provided, we list.
-	if cfgName == "" {
-		a.listIdentityProviders(ctx, w, r, cfgType, password)
-		return
-	}
+	var cfgList []madmin.IDPListItem
+	var err error
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
+		cfg := globalServerConfig.Clone()
+		cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
+	case madmin.LDAPIDPCfg:
+		cfg := globalServerConfig.Clone()
+		cfgList, err = globalLDAPConfig.GetConfigList(cfg)
 
-	cfg := globalServerConfig.Clone()
-
-	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
-	if err != nil {
-		if err == openid.ErrProviderConfigNotFound {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
-			return
-		}
-
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	res := madmin.IDPConfig{
-		Type: cfgType,
-		Name: cfgName,
-		Info: cfgInfos,
-	}
-	data, err := json.Marshal(res)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	econfigData, err := madmin.EncryptData(password, data)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, econfigData)
-}
-
-func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
-	// var subSys string
-	switch cfgType {
-	case "openid":
-		// subSys = config.IdentityOpenIDSubSys
 	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
 
-	cfg := globalServerConfig.Clone()
-	cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -238,9 +270,69 @@ func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.Resp
 	writeSuccessResponseJSON(w, econfigData)
 }
 
+// GetIdentityProviderCfg:
+//
+// GET <admin-prefix>/idp-cfg/openid/dex_test
+func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetIdentityProviderCfg")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	idpCfgType := mux.Vars(r)["type"]
+	cfgName := mux.Vars(r)["name"]
+	password := cred.SecretKey
+
+	if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+	var cfgInfos []madmin.IDPCfgInfo
+	var err error
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
+		cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	case madmin.LDAPIDPCfg:
+		cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
+	}
+	if err != nil {
+		if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			return
+		}
+
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	res := madmin.IDPConfig{
+		Type: idpCfgType,
+		Name: cfgName,
+		Info: cfgInfos,
+	}
+	data, err := json.Marshal(res)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
 // DeleteIdentityProviderCfg:
 //
-// DELETE <admin-prefix>/id-cfg?type=openid&name=dex_test
+// DELETE <admin-prefix>/idp-cfg/openid/dex_test
 func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteIdentityProviderCfg")
 
@@ -251,57 +343,82 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		return
 	}
 
-	cfgType := mux.Vars(r)["type"]
+	idpCfgType := mux.Vars(r)["type"]
 	cfgName := mux.Vars(r)["name"]
-	if !idCfgTypes.Contains(cfgType) {
-		// TODO: change this to invalid type error when implementation
-		// is complete.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
 		return
 	}
 
-	cfg := globalServerConfig.Clone()
+	cfgCopy := globalServerConfig.Clone()
+	var subSys string
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
+		subSys = config.IdentityOpenIDSubSys
+		cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
+		if err != nil {
+			if errors.Is(err, openid.ErrProviderConfigNotFound) {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+				return
+			}
 
-	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
-	if err != nil {
-		if err == openid.ErrProviderConfigNotFound {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	hasEnv := false
-	for _, ci := range cfgInfos {
-		if ci.IsCfg && ci.IsEnv {
-			hasEnv = true
-			break
+		hasEnv := false
+		for _, ci := range cfgInfos {
+			if ci.IsCfg && ci.IsEnv {
+				hasEnv = true
+				break
+			}
 		}
-	}
 
-	if hasEnv {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
-		return
-	}
+		if hasEnv {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
+			return
+		}
+	case madmin.LDAPIDPCfg:
+		subSys = config.IdentityLDAPSubSys
+		cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
+		if err != nil {
+			if errors.Is(err, openid.ErrProviderConfigNotFound) {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+				return
+			}
 
-	var subSys string
-	switch cfgType {
-	case "openid":
-		subSys = config.IdentityOpenIDSubSys
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		hasEnv := false
+		for _, ci := range cfgInfos {
+			if ci.IsCfg && ci.IsEnv {
+				hasEnv = true
+				break
+			}
+		}
+
+		if hasEnv {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
+			return
+		}
 	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
 
-	cfg, err = readServerConfig(ctx, objectAPI)
+	cfg, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
+	cfgKey := fmt.Sprintf("%s:%s", subSys, cfgName)
+	if cfgName == madmin.Default {
+		cfgKey = subSys
+	}
+	if err = cfg.DelKVS(cfgKey); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-idp-ldap.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
+//
+// GET <admin-prefix>/idp/ldap/policy-entities?[query-params]
+//
+// Query params:
+//
+//	user=... -> repeatable query parameter, specifying users to query for
+//	policy mapping
+//
+//	group=... -> repeatable query parameter, specifying groups to query for
+//	policy mapping
+//
+//	policy=... -> repeatable query parameter, specifying policy to query for
+//	user/group mapping
+//
+// When all query parameters are omitted, returns mappings for all policies.
+func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListLDAPPolicyMappingEntities")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	// Check authorization.
+
+	objectAPI, cred := validateAdminReq(ctx, w, r,
+		iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Validate API arguments.
+
+	q := madmin.PolicyEntitiesQuery{
+		Users:  r.Form["user"],
+		Groups: r.Form["group"],
+		Policy: r.Form["policy"],
+	}
+
+	// Query IAM
+
+	res, err := globalIAMSys.QueryLDAPPolicyEntities(r.Context(), q)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Encode result and send response.
+
+	data, err := json.Marshal(res)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	password := cred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, econfigData)
+}

cmd/admin-handlers-pools.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 
@@ -27,6 +28,11 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
+var (
+	errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is aleady in progress")
+	errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
+)
+
 func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartDecommission")
 
@@ -49,6 +55,11 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	if pools.IsRebalanceStarted() {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errDecommissionRebalanceAlreadyRunning), r.URL)
+		return
+	}
+
 	vars := mux.Vars(r)
 	v := vars["pool"]
 
@@ -200,3 +211,137 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 
 	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
 }
+
+func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RebalanceStart")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// NB rebalance-start admin API is always coordinated from first pool's
+	// first node. The following is required to serialize (the effects of)
+	// concurrent rebalance-start commands.
+	if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
+		for nodeIdx, proxyEp := range globalProxyEndpoints {
+			if proxyEp.Endpoint.Host == ep.Host {
+				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+					return
+				}
+			}
+		}
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok || len(pools.serverPools) == 1 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	if pools.IsDecommissionRunning() {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errRebalanceDecommissionAlreadyRunning), r.URL)
+		return
+	}
+
+	if pools.IsRebalanceStarted() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL)
+		return
+	}
+
+	bucketInfos, err := objectAPI.ListBuckets(ctx, BucketOptions{})
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	buckets := make([]string, 0, len(bucketInfos))
+	for _, bInfo := range bucketInfos {
+		buckets = append(buckets, bInfo.Name)
+	}
+
+	var id string
+	if id, err = pools.initRebalanceMeta(ctx, buckets); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	// Rebalance routine is run on the first node of any pool participating in rebalance.
+	pools.StartRebalance()
+
+	b, err := json.Marshal(struct {
+		ID string `json:"id"`
+	}{ID: id})
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, b)
+	// Notify peers to load rebalance.bin and start rebalance routine if they happen to be
+	// participating pool's leader node
+	globalNotificationSys.LoadRebalanceMeta(ctx, true)
+}
+
+func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RebalanceStatus")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Proxy rebalance-status to first pool first node, so that users see a
+	// consistent view of rebalance progress even though different rebalancing
+	// pools may temporarily have out of date info on the others.
+	if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal {
+		for nodeIdx, proxyEp := range globalProxyEndpoints {
+			if proxyEp.Endpoint.Host == ep.Host {
+				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+					return
+				}
+			}
+		}
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	rs, err := rebalanceStatus(ctx, pools)
+	if err != nil {
+		if errors.Is(err, errRebalanceNotStarted) || errors.Is(err, errConfigNotFound) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL)
+			return
+		}
+		logger.LogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(rs))
+}
+
+func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RebalanceStop")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// Cancel any ongoing rebalance operation
+	globalNotificationSys.StopRebalance(r.Context())
+	writeSuccessResponseHeadersOnly(w)
+	logger.LogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+}

cmd/admin-handlers-site-replication.go

@@ -515,3 +515,45 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+// SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op
+func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationResyncOp")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationResyncAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var peerSite madmin.PeerInfo
+	if err := parseJSONBody(ctx, r.Body, &peerSite, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	vars := mux.Vars(r)
+	op := madmin.SiteResyncOp(vars["operation"])
+	var (
+		status madmin.SRResyncOpStatus
+		err    error
+	)
+	switch op {
+	case madmin.SiteResyncStart:
+		status, err = globalSiteReplicationSys.startResync(ctx, objectAPI, peerSite)
+	case madmin.SiteResyncCancel:
+		status, err = globalSiteReplicationSys.cancelResync(ctx, objectAPI, peerSite)
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
+	}
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	body, err := json.Marshal(status)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, body)
+}

cmd/admin-handlers-users.go

@@ -657,7 +657,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 		// In case of LDAP we need to resolve the targetUser to a DN and
 		// query their groups:
-		if globalLDAPConfig.Enabled {
+		if globalLDAPConfig.Enabled() {
 			opts.claims[ldapUserN] = targetUser // simple username
 			targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
 			if err != nil {
@@ -1149,15 +1149,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	var dataUsageInfo DataUsageInfo
-	var err error
-	if !globalIsGateway {
-		// Load the latest calculated data usage
-		dataUsageInfo, _ = loadDataUsageFromBackend(ctx, objectAPI)
-	}
+	// Load the latest calculated data usage
+	dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
 
 	// If etcd, dns federation configured list buckets from etcd.
 	var buckets []BucketInfo
+	var err error
 	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && !IsErrIgnored(err,
@@ -1189,31 +1186,46 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		accountName = cred.ParentUser
 	}
 
+	roleArn := iampolicy.Args{Claims: claims}.GetRoleArn()
+	policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(claims, iamPolicyClaimNameOpenID())
+	var effectivePolicy iampolicy.Policy
+
 	var buf []byte
-	if accountName == globalActiveCred.AccessKey {
+	switch {
+	case accountName == globalActiveCred.AccessKey:
 		for _, policy := range iampolicy.DefaultPolicies {
 			if policy.Name == "consoleAdmin" {
-				buf, err = json.MarshalIndent(policy.Definition, "", " ")
-				if err != nil {
-					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-					return
-				}
+				effectivePolicy = policy.Definition
 				break
 			}
 		}
-	} else {
+
+	case roleArn != "":
+		_, policy, err := globalIAMSys.GetRolePolicy(roleArn)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		policySlice := newMappedPolicy(policy).toSlice()
+		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
+
+	case hasPolicyClaim:
+		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
+
+	default:
 		policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+		effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
 
-		buf, err = json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
+	}
+	buf, err = json.MarshalIndent(effectivePolicy, "", " ")
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 
 	acctInfo := madmin.AccountInfo{
@@ -2086,7 +2098,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 
 				// In case of LDAP we need to resolve the targetUser to a DN and
 				// query their groups:
-				if globalLDAPConfig.Enabled {
+				if globalLDAPConfig.Enabled() {
 					opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
 					targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
 					if err != nil {

cmd/admin-handlers-users_test.go

@@ -1231,6 +1231,19 @@ func (c *check) mustGetObject(ctx context.Context, client *minio.Client, bucket,
 	}
 }
 
+func (c *check) mustHeadObject(ctx context.Context, client *minio.Client, bucket, object string, tagCount int) {
+	c.Helper()
+
+	oinfo, err := client.StatObject(ctx, bucket, object, minio.StatObjectOptions{})
+	if err != nil {
+		c.Fatalf("user was unable to download the object: %v", err)
+	}
+
+	if oinfo.UserTagCount != tagCount {
+		c.Fatalf("expected tagCount: %d, got %d", tagCount, oinfo.UserTagCount)
+	}
+}
+
 func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) {
 	c.Helper()
 	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})

cmd/admin-handlers.go

@@ -21,8 +21,12 @@ import (
 	"bytes"
 	"context"
 	crand "crypto/rand"
+	"crypto/rsa"
 	"crypto/subtle"
+	"crypto/x509"
+	"encoding/base64"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"hash/crc32"
@@ -44,13 +48,13 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go"
+	"github.com/minio/madmin-go/estream"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
-	"github.com/minio/minio/internal/pubsub"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
 	"github.com/secure-io/sio-go"
@@ -411,6 +415,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 			}
 		}
 	}
+	jobID := r.Form.Get("by-jobID")
 
 	hosts := strings.Split(r.Form.Get("hosts"), ",")
 	byHost := strings.EqualFold(r.Form.Get("by-host"), "true")
@@ -423,7 +428,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 			}
 		}
 	}
-
+	dID := r.Form.Get("by-depID")
 	done := ctx.Done()
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
@@ -432,12 +437,21 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 	enc := json.NewEncoder(w)
 	for n > 0 {
 		var m madmin.RealtimeMetrics
-		mLocal := collectLocalMetrics(types, hostMap, diskMap)
+		mLocal := collectLocalMetrics(types, collectMetricsOpts{
+			hosts: hostMap,
+			disks: diskMap,
+			jobID: jobID,
+			depID: dID,
+		})
 		m.Merge(&mLocal)
-
 		// Allow half the interval for collecting remote...
 		cctx, cancel := context.WithTimeout(ctx, interval/2)
-		mRemote := collectRemoteMetrics(cctx, types, hostMap, diskMap)
+		mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{
+			hosts: hostMap,
+			disks: diskMap,
+			jobID: jobID,
+			depID: dID,
+		})
 		cancel()
 		m.Merge(&mRemote)
 		if !byHost {
@@ -449,7 +463,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 
 		m.Final = n <= 1
 
-		// Marshal API response
+		// Marshal API reesponse
 		if err := enc.Encode(&m); err != nil {
 			n = 0
 		}
@@ -498,9 +512,10 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
-func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
+func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
+		Elapsed:    now.Sub(l.Timestamp),
 		Resource:   resource,
 		ServerList: []string{server},
 		Source:     l.Source,
@@ -517,6 +532,7 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEn
 }
 
 func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
+	now := time.Now().UTC()
 	entryMap := make(map[string]*madmin.LockEntry)
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
@@ -527,7 +543,7 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 				if val, ok := entryMap[lockReqInfo.Name]; ok {
 					val.ServerList = append(val.ServerList, peerLock.Addr)
 				} else {
-					entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
+					entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
 				}
 			}
 		}
@@ -935,11 +951,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
-		return
-	}
-
 	hip, errCode := extractHealInitParams(mux.Vars(r), r.Form, r.Body)
 	if errCode != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
@@ -1126,12 +1137,6 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	// Check if this setup has an erasure coded backend.
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
-		return
-	}
-
 	aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1207,11 +1212,6 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
@@ -1380,11 +1380,6 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
 
@@ -1516,16 +1511,12 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan pubsub.Maskable, 4000)
+	traceCh := make(chan madmin.TraceInfo, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
-	mask := pubsub.MaskFromMaskable(traceOpts.TraceTypes())
 
-	err = globalTrace.Subscribe(mask, traceCh, ctx.Done(), func(entry pubsub.Maskable) bool {
-		if e, ok := entry.(madmin.TraceInfo); ok {
-			return shouldTrace(e, traceOpts)
-		}
-		return false
+	err = globalTrace.Subscribe(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
+		return shouldTrace(entry, traceOpts)
 	})
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
@@ -1597,7 +1588,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	setEventStreamHeaders(w)
 
-	logCh := make(chan pubsub.Maskable, 4000)
+	logCh := make(chan log.Info, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
@@ -1623,9 +1614,11 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	for {
 		select {
-		case entry := <-logCh:
-			log, ok := entry.(log.Info)
-			if ok && log.SendLog(node, logKind) {
+		case log, ok := <-logCh:
+			if !ok {
+				return
+			}
+			if log.SendLog(node, logKind) {
 				if err := enc.Encode(log); err != nil {
 					return
 				}
@@ -1789,8 +1782,8 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 	kmsStat := fetchKMSStatus()
 
 	ldap := madmin.LDAP{}
-	if globalLDAPConfig.Enabled {
-		ldapConn, err := globalLDAPConfig.Connect()
+	if globalLDAPConfig.Enabled() {
+		ldapConn, err := globalLDAPConfig.LDAP.Connect()
 		//nolint:gocritic
 		if err != nil {
 			ldap.Status = string(madmin.ItemOffline)
@@ -1901,7 +1894,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 	}
 
 	client := &http.Client{
-		Transport: NewGatewayHTTPTransport(),
+		Transport: NewHTTPTransport(),
 		Timeout:   10 * time.Second,
 	}
 
@@ -1918,73 +1911,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 	return ki
 }
 
-// HealthInfoHandler - GET /minio/admin/v3/healthinfo
-// ----------
-// Get server health info
-func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HealthInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
-	if objectAPI == nil {
-		return
-	}
-
-	query := r.Form
-	healthInfo := madmin.HealthInfo{
-		Version: madmin.HealthInfoVersion,
-		Minio: madmin.MinioHealthInfo{
-			Info: madmin.MinioInfo{
-				DeploymentID: globalDeploymentID,
-			},
-		},
-	}
-	healthInfoCh := make(chan madmin.HealthInfo)
-
-	enc := json.NewEncoder(w)
-
-	setCommonHeaders(w)
-
-	setEventStreamHeaders(w)
-
-	w.WriteHeader(http.StatusOK)
-
-	errResp := func(err error) {
-		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
-			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
-		encodedErrorResponse := encodeResponse(errorResponse)
-		healthInfo.Error = string(encodedErrorResponse)
-		logger.LogIf(ctx, enc.Encode(healthInfo))
-	}
-
-	deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
-	if dstr := r.Form.Get("deadline"); dstr != "" {
-		var err error
-		deadline, err = time.ParseDuration(dstr)
-		if err != nil {
-			errResp(err)
-			return
-		}
-	}
-
-	nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
-	lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
-	if err != nil { // returns a locked lock
-		errResp(err)
-		return
-	}
-	defer nsLock.Unlock(lkctx.Cancel)
-
-	healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
-	defer healthCancel()
-
-	// Freeze all incoming S3 API calls before running speedtest.
-	globalNotificationSys.ServiceFreeze(ctx, true)
-
-	// unfreeze all incoming S3 API calls after speedtest.
-	defer globalNotificationSys.ServiceFreeze(ctx, false)
-
+func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) {
 	hostAnonymizer := createHostAnonymizer()
 	// anonAddr - Anonymizes hosts in given input string.
 	anonAddr := func(addr string) string {
@@ -2130,7 +2057,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
 			anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
-			return strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
+			if len(globalMinioConsoleHost) > 0 {
+				anonCmdLine = strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1")
+			}
+			return anonCmdLine
 		}
 
 		// Server start command regex groups:
@@ -2206,7 +2136,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	getAndWriteMinioConfig := func() {
 		if query.Get("minioconfig") == "true" {
-			config, err := readServerConfig(ctx, objectAPI)
+			config, err := readServerConfig(healthCtx, objectAPI)
 			if err != nil {
 				healthInfo.Minio.Config = madmin.MinioConfig{
 					Error: err.Error(),
@@ -2254,8 +2184,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		getAndWriteSysConfig()
 
 		if query.Get("minioinfo") == "true" {
-			infoMessage := getServerInfo(ctx, r)
-			servers := []madmin.ServerInfo{}
+			infoMessage := getServerInfo(healthCtx, nil)
+			servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
 				servers = append(servers, madmin.ServerInfo{
@@ -2274,6 +2204,11 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 						Frees:      server.MemStats.Frees,
 						HeapAlloc:  server.MemStats.HeapAlloc,
 					},
+					GoMaxProcs:     server.GoMaxProcs,
+					NumCPU:         server.NumCPU,
+					RuntimeVersion: server.RuntimeVersion,
+					GCStats:        server.GCStats,
+					MinioEnvVars:   server.MinioEnvVars,
 				})
 			}
 
@@ -2299,6 +2234,68 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			partialWrite(healthInfo)
 		}
 	}()
+}
+
+// HealthInfoHandler - GET /minio/admin/v3/healthinfo
+// ----------
+// Get server health info
+func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HealthInfo")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	query := r.Form
+	healthInfoCh := make(chan madmin.HealthInfo)
+	enc := json.NewEncoder(w)
+
+	healthInfo := madmin.HealthInfo{
+		TimeStamp: time.Now().UTC(),
+		Version:   madmin.HealthInfoVersion,
+		Minio: madmin.MinioHealthInfo{
+			Info: madmin.MinioInfo{
+				DeploymentID: globalDeploymentID,
+			},
+		},
+	}
+
+	errResp := func(err error) {
+		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
+			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		encodedErrorResponse := encodeResponse(errorResponse)
+		healthInfo.Error = string(encodedErrorResponse)
+		logger.LogIf(ctx, enc.Encode(healthInfo))
+	}
+
+	deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
+	if dstr := query.Get("deadline"); dstr != "" {
+		var err error
+		deadline, err = time.ParseDuration(dstr)
+		if err != nil {
+			errResp(err)
+			return
+		}
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
+	lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
+	if err != nil { // returns a locked lock
+		errResp(err)
+		return
+	}
+
+	defer nsLock.Unlock(lkctx.Cancel)
+	healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
+	defer healthCancel()
+
+	go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo)
+
+	setCommonHeaders(w)
+	setEventStreamHeaders(w)
+	w.WriteHeader(http.StatusOK)
 
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
@@ -2522,63 +2519,21 @@ func fetchKMSStatus() madmin.KMS {
 func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
 	var auditloggerInfo []madmin.Audit
-	for _, target := range logger.SystemTargets() {
-		if target.Endpoint() != "" {
-			tgt := target.String()
-			err := checkConnection(target.Endpoint(), 15*time.Second)
-			if err == nil {
-				mapLog := make(map[string]madmin.Status)
-				mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
-				loggerInfo = append(loggerInfo, mapLog)
-			} else {
-				mapLog := make(map[string]madmin.Status)
-				mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
-				loggerInfo = append(loggerInfo, mapLog)
-			}
+	for _, tgt := range logger.SystemTargets() {
+		if tgt.Endpoint() != "" {
+			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(tgt)})
 		}
 	}
 
-	for _, target := range logger.AuditTargets() {
-		if target.Endpoint() != "" {
-			tgt := target.String()
-			err := checkConnection(target.Endpoint(), 15*time.Second)
-			if err == nil {
-				mapAudit := make(map[string]madmin.Status)
-				mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
-				auditloggerInfo = append(auditloggerInfo, mapAudit)
-			} else {
-				mapAudit := make(map[string]madmin.Status)
-				mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
-				auditloggerInfo = append(auditloggerInfo, mapAudit)
-			}
+	for _, tgt := range logger.AuditTargets() {
+		if tgt.Endpoint() != "" {
+			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(tgt)})
 		}
 	}
 
 	return loggerInfo, auditloggerInfo
 }
 
-// checkConnection - ping an endpoint , return err in case of no connection
-func checkConnection(endpointStr string, timeout time.Duration) error {
-	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
-	defer cancel()
-
-	client := &http.Client{
-		Transport: globalProxyTransport,
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
-	if err != nil {
-		return err
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer xhttp.DrainBody(resp.Body)
-	return nil
-}
-
 func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
 	// Send profiling data to zip as file
 	header, zerr := zip.FileInfoHeader(dummyFileInfo{
@@ -2601,13 +2556,14 @@ func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
 	return err
 }
 
-// appendClusterMetaInfoToZip gets information of the current cluster and embedded
-// it in the passed zipwriter, This is not a critical function and it is allowed
-// to fail with a ten seconds timeout.
-func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
+// getClusterMetaInfo gets information of the current cluster and
+// returns it.
+// This is not a critical function, and it is allowed
+// to fail with a ten seconds timeout, returning nil.
+func getClusterMetaInfo(ctx context.Context) []byte {
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
-		return
+		return nil
 	}
 
 	// Add a ten seconds timeout because getting profiling data
@@ -2616,7 +2572,10 @@ func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
 	defer cancel()
 
 	resultCh := make(chan madmin.ClusterRegistrationInfo)
+
 	go func() {
+		defer close(resultCh)
+
 		ci := madmin.ClusterRegistrationInfo{}
 		ci.Info.NoOfServerPools = len(globalEndpoints)
 		ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
@@ -2638,25 +2597,39 @@ func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
 
 		ci.DeploymentID = globalDeploymentID
 		ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
-		resultCh <- ci
+
+		select {
+		case resultCh <- ci:
+		case <-ctx.Done():
+			return
+		}
 	}()
 
 	select {
 	case <-ctx.Done():
-		return
+		return nil
 	case ci := <-resultCh:
-		out, err := json.MarshalIndent(ci, "", "   ")
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return
-		}
-		err = embedFileInZip(zipWriter, "cluster.info", out)
+		out, err := json.MarshalIndent(ci, "", "  ")
 		if err != nil {
 			logger.LogIf(ctx, err)
+			return nil
 		}
+		return out
 	}
 }
 
+func bytesToPublicKey(pub []byte) (*rsa.PublicKey, error) {
+	block, _ := pem.Decode(pub)
+	if block != nil {
+		pub = block.Bytes
+	}
+	key, err := x509.ParsePKCS1PublicKey(pub)
+	if err != nil {
+		return nil, err
+	}
+	return key, nil
+}
+
 // getRawDataer provides an interface for getting raw FS files.
 type getRawDataer interface {
 	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
@@ -2683,12 +2656,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	if err := parseForm(r); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	volume := r.Form.Get("volume")
-	file := r.Form.Get("file")
 	if len(volume) == 0 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
 		return
 	}
+	file := r.Form.Get("file")
 	if len(file) == 0 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
@@ -2701,41 +2679,102 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	var key [32]byte
-	// MUST use crypto/rand
-	n, err := crand.Read(key[:])
-	if err != nil || n != len(key) {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
-		return
-	}
-	stream, err := sio.AES_256_GCM.Stream(key[:])
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
-		return
-	}
-	// Zero nonce, we only use each key once, and 32 bytes is plenty.
-	nonce := make([]byte, stream.NonceSize())
-	encw := stream.EncryptWriter(w, nonce, nil)
+	var publicKey *rsa.PublicKey
 
-	defer encw.Close()
+	publicKeyB64 := r.Form.Get("public-key")
+	if publicKeyB64 != "" {
+		publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyB64)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		publicKey, err = bytesToPublicKey(publicKeyBytes)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
 
 	// Write a version for making *incompatible* changes.
 	// The AdminClient will reject any version it does not know.
-	w.Write([]byte{1})
+	var inspectZipW *zip.Writer
+	if publicKey != nil {
+		w.WriteHeader(200)
+		stream := estream.NewWriter(w)
+		defer stream.Close()
 
-	// Write key first (without encryption)
-	_, err = w.Write(key[:])
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
-		return
+		clusterKey, err := bytesToPublicKey(subnetAdminPublicKey)
+		if err != nil {
+			logger.LogIf(ctx, stream.AddError(err.Error()))
+			return
+		}
+		err = stream.AddKeyEncrypted(clusterKey)
+		if err != nil {
+			logger.LogIf(ctx, stream.AddError(err.Error()))
+			return
+		}
+		if b := getClusterMetaInfo(ctx); len(b) > 0 {
+			w, err := stream.AddEncryptedStream("cluster.info", nil)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return
+			}
+			w.Write(b)
+			w.Close()
+		}
+
+		// Add new key for inspect data.
+		if err := stream.AddKeyEncrypted(publicKey); err != nil {
+			logger.LogIf(ctx, stream.AddError(err.Error()))
+			return
+		}
+		encStream, err := stream.AddEncryptedStream("inspect.zip", nil)
+		if err != nil {
+			logger.LogIf(ctx, stream.AddError(err.Error()))
+			return
+		}
+		defer encStream.Close()
+
+		inspectZipW = zip.NewWriter(encStream)
+		defer inspectZipW.Close()
+	} else {
+		// Legacy: Remove if we stop supporting inspection without public key.
+		var key [32]byte
+		// MUST use crypto/rand
+		n, err := crand.Read(key[:])
+		if err != nil || n != len(key) {
+			logger.LogIf(ctx, err)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		// Write a version for making *incompatible* changes.
+		// The AdminClient will reject any version it does not know.
+		if publicKey == nil {
+			w.Write([]byte{1})
+			w.Write(key[:])
+		}
+
+		stream, err := sio.AES_256_GCM.Stream(key[:])
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return
+		}
+		// Zero nonce, we only use each key once, and 32 bytes is plenty.
+		nonce := make([]byte, stream.NonceSize())
+		encw := stream.EncryptWriter(w, nonce, nil)
+		defer encw.Close()
+
+		// Initialize a zip writer which will provide a zipped content
+		// of profiling data of all nodes
+		inspectZipW = zip.NewWriter(encw)
+		defer inspectZipW.Close()
+
+		if b := getClusterMetaInfo(ctx); len(b) > 0 {
+			logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b))
+		}
 	}
 
-	// Initialize a zip writer which will provide a zipped content
-	// of profiling data of all nodes
-	zipWriter := zip.NewWriter(encw)
-	defer zipWriter.Close()
 	rawDataFn := func(r io.Reader, host, disk, filename string, si StatInfo) error {
 		// Prefix host+disk
 		filename = path.Join(host, disk, filename)
@@ -2764,17 +2803,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			return nil
 		}
 		header.Method = zip.Deflate
-		zwriter, zerr := zipWriter.CreateHeader(header)
+		zwriter, zerr := inspectZipW.CreateHeader(header)
 		if zerr != nil {
 			logger.LogIf(ctx, zerr)
 			return nil
 		}
-		if _, err = io.Copy(zwriter, r); err != nil {
+		if _, err := io.Copy(zwriter, r); err != nil {
 			logger.LogIf(ctx, err)
 		}
 		return nil
 	}
-	err = o.GetRawData(ctx, volume, file, rawDataFn)
+	err := o.GetRawData(ctx, volume, file, rawDataFn)
 	if !errors.Is(err, errFileNotFound) {
 		logger.LogIf(ctx, err)
 	}
@@ -2786,22 +2825,17 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	if !errors.Is(err, errFileNotFound) {
 		logger.LogIf(ctx, err)
 	}
-	// save args passed to inspect command
-	inspectArgs := []string{fmt.Sprintf(" Inspect path: %s%s%s\n", volume, slashSeparator, file)}
-	cmdLine := []string{"Server command line args: "}
-	for _, pool := range globalEndpoints {
-		cmdLine = append(cmdLine, pool.CmdLine)
-	}
-	cmdLine = append(cmdLine, "\n")
-	inspectArgs = append(inspectArgs, cmdLine...)
-	inspectArgsBytes := []byte(strings.Join(inspectArgs, " "))
-	if err = rawDataFn(bytes.NewReader(inspectArgsBytes), "", "", "inspect-input.txt", StatInfo{
-		Size: int64(len(inspectArgsBytes)),
-	}); err != nil {
-		logger.LogIf(ctx, err)
-	}
 
-	appendClusterMetaInfoToZip(ctx, zipWriter)
+	// save args passed to inspect command
+	var sb bytes.Buffer
+	fmt.Fprintf(&sb, "Inspect path: %s%s%s\n", volume, slashSeparator, file)
+	sb.WriteString("Server command line args:")
+	for _, pool := range globalEndpoints {
+		sb.WriteString(" ")
+		sb.WriteString(pool.CmdLine)
+	}
+	sb.WriteString("\n")
+	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
 }
 
 func createHostAnonymizerForFSMode() map[string]string {

cmd/admin-handlers_test.go

@@ -73,7 +73,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 
 	globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
 
-	initAllSubsystems()
+	initAllSubsystems(ctx)
 
 	initConfigSubsystem(ctx, objLayer)
 

cmd/admin-heal-ops.go

@@ -100,14 +100,14 @@ type allHealState struct {
 }
 
 // newHealState - initialize global heal state management
-func newHealState(cleanup bool) *allHealState {
+func newHealState(ctx context.Context, cleanup bool) *allHealState {
 	hstate := &allHealState{
 		healSeqMap:     make(map[string]*healSequence),
 		healLocalDisks: make(map[Endpoint]bool),
 		healStatus:     make(map[string]healingTracker),
 	}
 	if cleanup {
-		go hstate.periodicHealSeqsClean(GlobalContext)
+		go hstate.periodicHealSeqsClean(ctx)
 	}
 	return hstate
 }

cmd/admin-router.go

@@ -60,7 +60,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		// Info operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(gz(httpTraceAll(adminAPI.ServerInfoHandler)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/inspect-data").HandlerFunc(httpTraceHdrs(adminAPI.InspectDataHandler)).Queries("volume", "{volume:.*}", "file", "{file:.*}")
+		adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(httpTraceAll(adminAPI.InspectDataHandler))
 
 		// StorageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
@@ -84,6 +84,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
 			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
+
+			// Rebalance operations
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStart)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStatus)))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStop)))
 		}
 
 		// Profiling operations - deprecated API
@@ -179,9 +184,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
 
 		// IDentity Provider configuration APIs
-		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg))).Queries("type", "{type:.*}")
-		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddIdentityProviderCfg)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateIdentityProviderCfg)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListIdentityProviderCfg)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg)))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg)))
+
+		// LDAP IAM operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListLDAPPolicyMappingEntities)))
 
 		// -- END IAM APIs --
 
@@ -206,6 +216,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
 
+		// Batch job operations
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.StartBatchJob)))
+
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ListBatchJobs)))
+
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
+
 		// Bucket migration operations
 		// ExportBucketMetaHandler
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
@@ -238,6 +258,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationResyncOp))).Queries("operation", "{operation:.*}")
 
 		if globalIsDistErasure {
 			// Top locks
@@ -264,16 +285,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(gz(httpTraceAll(adminAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}")
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSKeyStatusHandler)))
 
-		if !globalIsGateway {
-			// Keep obdinfo for backward compatibility with mc
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
-				HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
-			// -- Health API --
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
-				HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
-				HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
-		}
+		// Keep obdinfo for backward compatibility with mc
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
+			HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
+		// -- Health API --
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
+			HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
+			HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
 	}
 
 	// If none of the routes match add default error handler routes

cmd/admin-server-info.go

@@ -20,10 +20,14 @@ package cmd
 import (
 	"context"
 	"net/http"
+	"os"
 	"runtime"
+	"runtime/debug"
+	"strings"
 	"time"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -67,6 +71,22 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	var memstats runtime.MemStats
 	runtime.ReadMemStats(&memstats)
 
+	gcStats := debug.GCStats{
+		// If stats.PauseQuantiles is non-empty, ReadGCStats fills
+		// it with quantiles summarizing the distribution of pause time.
+		// For example, if len(stats.PauseQuantiles) is 5, it will be
+		// filled with the minimum, 25%, 50%, 75%, and maximum pause times.
+		PauseQuantiles: make([]time.Duration, 5),
+	}
+	debug.ReadGCStats(&gcStats)
+	// Truncate GC stats to max 5 entries.
+	if len(gcStats.PauseEnd) > 5 {
+		gcStats.PauseEnd = gcStats.PauseEnd[len(gcStats.PauseEnd)-5:]
+	}
+	if len(gcStats.Pause) > 5 {
+		gcStats.Pause = gcStats.Pause[len(gcStats.Pause)-5:]
+	}
+
 	props := madmin.ServerProperties{
 		State:    string(madmin.ItemInitializing),
 		Endpoint: addr,
@@ -81,10 +101,48 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			Frees:      memstats.Frees,
 			HeapAlloc:  memstats.HeapAlloc,
 		},
+		GoMaxProcs:     runtime.GOMAXPROCS(0),
+		NumCPU:         runtime.NumCPU(),
+		RuntimeVersion: runtime.Version(),
+		GCStats: &madmin.GCStats{
+			LastGC:     gcStats.LastGC,
+			NumGC:      gcStats.NumGC,
+			PauseTotal: gcStats.PauseTotal,
+			Pause:      gcStats.Pause,
+			PauseEnd:   gcStats.PauseEnd,
+		},
+		MinioEnvVars: make(map[string]string, 10),
+	}
+
+	sensitive := map[string]struct{}{
+		config.EnvAccessKey:         {},
+		config.EnvSecretKey:         {},
+		config.EnvRootUser:          {},
+		config.EnvRootPassword:      {},
+		config.EnvMinIOSubnetAPIKey: {},
+		config.EnvKMSSecretKey:      {},
+	}
+	for _, v := range os.Environ() {
+		if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {
+			continue
+		}
+		split := strings.SplitN(v, "=", 2)
+		key := split[0]
+		value := ""
+		if len(split) > 1 {
+			value = split[1]
+		}
+
+		// Do not send sensitive creds.
+		if _, ok := sensitive[key]; ok || strings.Contains(strings.ToLower(key), "password") || strings.HasSuffix(strings.ToLower(key), "key") {
+			props.MinioEnvVars[key] = "*** EXISTS, REDACTED ***"
+			continue
+		}
+		props.MinioEnvVars[key] = value
 	}
 
 	objLayer := newObjectLayerFn()
-	if objLayer != nil && !globalIsGateway {
+	if objLayer != nil {
 		// only need Disks information in server mode.
 		storageInfo, _ := objLayer.LocalStorageInfo(GlobalContext)
 		props.State = string(madmin.ItemOnline)

cmd/api-errors.go

@@ -30,6 +30,7 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"google.golang.org/api/googleapi"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
@@ -263,6 +264,7 @@ const (
 	ErrAdminNoSuchUser
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
+	ErrAdminNoSuchJob
 	ErrAdminNoSuchPolicy
 	ErrAdminInvalidArgument
 	ErrAdminInvalidAccessKey
@@ -273,6 +275,10 @@ const (
 	ErrAdminNoSuchConfigTarget
 	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
+	ErrAdminConfigInvalidIDPType
+	ErrAdminConfigLDAPValidation
+	ErrAdminConfigIDPCfgNameAlreadyExists
+	ErrAdminConfigIDPCfgNameDoesNotExist
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
@@ -287,6 +293,10 @@ const (
 	ErrSiteReplicationIAMError
 	ErrSiteReplicationConfigMissing
 
+	// Pool rebalance errors
+	ErrAdminRebalanceAlreadyStarted
+	ErrAdminRebalanceNotStarted
+
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
 	ErrAdminNoSuchQuotaConfiguration
@@ -1226,6 +1236,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified group does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrAdminNoSuchJob: {
+		Code:           "XMinioAdminNoSuchJob",
+		Description:    "The specified job does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrAdminGroupNotEmpty: {
 		Code:           "XMinioAdminGroupNotEmpty",
 		Description:    "The specified group is not empty - cannot remove it.",
@@ -1282,6 +1297,26 @@ var errorCodes = errorCodeMap{
 		Description:    "JSON configuration provided has objects with duplicate keys",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigInvalidIDPType: {
+		Code:           "XMinioAdminConfigInvalidIDPType",
+		Description:    fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigLDAPValidation: {
+		Code:           "XMinioAdminConfigLDAPValidation",
+		Description:    "LDAP Configuration validation failed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigIDPCfgNameAlreadyExists: {
+		Code:           "XMinioAdminConfigIDPCfgNameAlreadyExists",
+		Description:    "An IDP configuration with the given name aleady exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigIDPCfgNameDoesNotExist: {
+		Code:           "XMinioAdminConfigIDPCfgNameDoesNotExist",
+		Description:    "No such IDP configuration exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigNotificationTargetsFailed: {
 		Code:           "XMinioAdminNotificationTargetsTestFailed",
 		Description:    "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",
@@ -1378,6 +1413,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Site not found in site replication configuration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminRebalanceAlreadyStarted: {
+		Code:           "XMinioAdminRebalanceAlreadyStarted",
+		Description:    "Pool rebalance is already started",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAdminRebalanceNotStarted: {
+		Code:           "XMinioAdminRebalanceNotStarted",
+		Description:    "Pool rebalance is not started",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
@@ -1923,6 +1968,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminNoSuchGroup
 	case errGroupNotEmpty:
 		apiErr = ErrAdminGroupNotEmpty
+	case errNoSuchJob:
+		apiErr = ErrAdminNoSuchJob
 	case errNoSuchPolicy:
 		apiErr = ErrAdminNoSuchPolicy
 	case errSignatureMismatch:
@@ -2217,9 +2264,14 @@ func toAPIError(ctx context.Context, err error) APIError {
 	if apiErr.Code == "InternalError" {
 		// If we see an internal error try to interpret
 		// any underlying errors if possible depending on
-		// their internal error types. This code is only
-		// useful with gateway implementations.
+		// their internal error types.
 		switch e := err.(type) {
+		case batchReplicationJobError:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Description,
+				HTTPStatusCode: e.HTTPStatusCode,
+			}
 		case InvalidArgument:
 			apiErr = APIError{
 				Code:           "InvalidArgument",
@@ -2282,7 +2334,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 				Description:    e.Message,
 				HTTPStatusCode: e.StatusCode,
 			}
-			if globalIsGateway && strings.Contains(e.Message, "KMS is not configured") {
+			if strings.Contains(e.Message, "KMS is not configured") {
 				apiErr = APIError{
 					Code:           "NotImplemented",
 					Description:    e.Message,
@@ -2306,7 +2358,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 				Description:    e.Error(),
 				HTTPStatusCode: e.Response().StatusCode,
 			}
-			// Add more Gateway SDKs here if any in future.
+			// Add more other SDK related errors here if any in future.
 		default:
 			//nolint:gocritic
 			if errors.Is(err, errMalformedEncoding) {

cmd/api-response.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	xxml "github.com/minio/xxml"
 )
 
 const (
@@ -90,8 +91,7 @@ type ListVersionsResponse struct {
 	IsTruncated bool
 
 	CommonPrefixes []CommonPrefix
-	DeleteMarkers  []DeleteMarkerVersion `xml:"DeleteMarker,omitempty"`
-	Versions       []ObjectVersion       `xml:"Version,omitempty"`
+	Versions       []ObjectVersion
 
 	// Encoding type used to encode object keys in the response.
 	EncodingType string `xml:"EncodingType,omitempty"`
@@ -166,10 +166,10 @@ type Part struct {
 	Size         int64
 
 	// Checksum values
-	ChecksumCRC32  string
-	ChecksumCRC32C string
-	ChecksumSHA1   string
-	ChecksumSHA256 string
+	ChecksumCRC32  string `xml:"ChecksumCRC32,omitempty"`
+	ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
+	ChecksumSHA1   string `xml:"ChecksumSHA1,omitempty"`
+	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
 }
 
 // ListPartsResponse - format for list parts response.
@@ -256,6 +256,19 @@ type ObjectVersion struct {
 	Object
 	IsLatest  bool
 	VersionID string `xml:"VersionId"`
+
+	isDeleteMarker bool
+}
+
+// MarshalXML - marshal ObjectVersion
+func (o ObjectVersion) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
+	if o.isDeleteMarker {
+		start.Name.Local = "DeleteMarker"
+	} else {
+		start.Name.Local = "Version"
+	}
+	type objectVersionWrapper ObjectVersion
+	return e.EncodeElement(objectVersionWrapper(o), start)
 }
 
 // DeleteMarkerVersion container for delete marker metadata
@@ -390,10 +403,10 @@ type CompleteMultipartUploadResponse struct {
 	Key      string
 	ETag     string
 
-	ChecksumCRC32  string
-	ChecksumCRC32C string
-	ChecksumSHA1   string
-	ChecksumSHA256 string
+	ChecksumCRC32  string `xml:"ChecksumCRC32,omitempty"`
+	ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
+	ChecksumSHA1   string `xml:"ChecksumSHA1,omitempty"`
+	ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
 }
 
 // DeleteError structure.
@@ -482,7 +495,6 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
-	deleteMarkers := make([]DeleteMarkerVersion, 0, len(resp.Objects))
 
 	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
@@ -494,21 +506,6 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		if object.Name == "" {
 			continue
 		}
-
-		if object.DeleteMarker {
-			deleteMarker := DeleteMarkerVersion{
-				Key:          s3EncodeName(object.Name, encodingType),
-				LastModified: object.ModTime.UTC().Format(iso8601TimeFormat),
-				Owner:        owner,
-				VersionID:    object.VersionID,
-			}
-			if deleteMarker.VersionID == "" {
-				deleteMarker.VersionID = nullVersionID
-			}
-			deleteMarker.IsLatest = object.IsLatest
-			deleteMarkers = append(deleteMarkers, deleteMarker)
-			continue
-		}
 		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
@@ -527,12 +524,12 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 			content.VersionID = nullVersionID
 		}
 		content.IsLatest = object.IsLatest
+		content.isDeleteMarker = object.DeleteMarker
 		versions = append(versions, content)
 	}
 
 	data.Name = bucket
 	data.Versions = versions
-	data.DeleteMarkers = deleteMarkers
 	data.EncodingType = encodingType
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.KeyMarker = s3EncodeName(marker, encodingType)
@@ -639,7 +636,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			case crypto.SSEC:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
-			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
+			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
 				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 					// Do not need to send any internal metadata
 					// values to client.

cmd/auth-handler.go

@@ -626,7 +626,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 
 	conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
 	conditions["object-lock-mode"] = []string{string(retMode)}
-	conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
+	conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
 	if retDays > 0 {
 		conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
 	}

cmd/auth-handler_test.go

@@ -359,7 +359,10 @@ func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int6
 
 // Tests is requested authenticated function, tests replies for s3 errors.
 func TestIsReqAuthenticated(t *testing.T) {
-	objLayer, fsDir, err := prepareFS()
+	ctx, cancel := context.WithCancel(GlobalContext)
+	defer cancel()
+
+	objLayer, fsDir, err := prepareFS(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -368,10 +371,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
 
-	initAllSubsystems()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	initAllSubsystems(ctx)
 
 	initConfigSubsystem(ctx, objLayer)
 
@@ -413,7 +413,10 @@ func TestIsReqAuthenticated(t *testing.T) {
 }
 
 func TestCheckAdminRequestAuthType(t *testing.T) {
-	objLayer, fsDir, err := prepareFS()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	objLayer, fsDir, err := prepareFS(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -439,7 +442,6 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 		{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
-	ctx := context.Background()
 	for i, testCase := range testCases {
 		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
@@ -451,7 +453,7 @@ func TestValidateAdminSignature(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	objLayer, fsDir, err := prepareFS()
+	objLayer, fsDir, err := prepareFS(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -461,8 +463,7 @@ func TestValidateAdminSignature(t *testing.T) {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
 
-	initAllSubsystems()
-
+	initAllSubsystems(ctx)
 	initConfigSubsystem(ctx, objLayer)
 
 	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)

cmd/background-heal-ops.go

@@ -22,7 +22,6 @@ import (
 	"runtime"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/pubsub"
 )
 
 // healTask represents what to heal along with options
@@ -54,7 +53,7 @@ type healRoutine struct {
 func activeListeners() int {
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	return int(globalHTTPListen.NumSubscribers(pubsub.MaskAll)) + int(globalTrace.NumSubscribers(pubsub.MaskAll))
+	return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
 }
 
 func waitForLowHTTPReq() {

cmd/background-newdisks-heal-ops.go

@@ -292,7 +292,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	if err != nil {
 		return fmt.Errorf("Error: %w, %s", err, endpoint)
 	}
-
+	defer disk.Close()
 	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
 	if poolIdx < 0 {
 		return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
@@ -310,7 +310,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	// Prevent parallel erasure set healing
-	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
+	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
 	if err != nil {
 		return err

cmd/batchreplicationmetric_string.go

@@ -0,0 +1,23 @@
+// Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[batchReplicationMetricObject-0]
+}
+
+const _batchReplicationMetric_name = "Object"
+
+var _batchReplicationMetric_index = [...]uint8{0, 6}
+
+func (i batchReplicationMetric) String() string {
+	if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
+		return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
+}

cmd/bitrot-streaming.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"hash"
 	"io"
+	"strings"
 	"sync"
 
 	xhttp "github.com/minio/minio/internal/http"
@@ -146,16 +147,28 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
 	}
+	ignoredErrs := []error{
+		errDiskNotFound,
+	}
+	if strings.HasPrefix(b.volume, minioMetaBucket) {
+		ignoredErrs = append(ignoredErrs,
+			errFileNotFound,
+			errVolumeNotFound,
+			errFileVersionNotFound,
+		)
+	}
 	if b.rc == nil {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		if len(b.data) == 0 && b.tillOffset != streamOffset {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
-			if err != nil && err != errDiskNotFound {
-				logger.LogIf(GlobalContext,
-					fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
-						b.disk, b.volume, b.filePath, err))
+			if err != nil {
+				if !IsErr(err, ignoredErrs...) {
+					logger.LogIf(GlobalContext,
+						fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
+							b.disk, b.volume, b.filePath, err))
+				}
 			}
 		} else {
 			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)

cmd/bootstrap-peer-server.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -64,6 +64,7 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 		return fmt.Errorf("Expected platform '%s', found to be running '%s'",
 			s1.MinioPlatform, s2.MinioPlatform)
 	}
+
 	if s1.MinioEndpoints.NEndpoints() != s2.MinioEndpoints.NEndpoints() {
 		return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.NEndpoints(),
 			s2.MinioEndpoints.NEndpoints())
@@ -200,15 +201,18 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	srcCfg := getServerSystemCfg()
 	clnts := newBootstrapRESTClients(endpointServerPools)
 	var onlineServers int
-	var offlineEndpoints []string
+	var offlineEndpoints []error
+	var incorrectConfigs []error
 	var retries int
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
 				if !isNetworkError(err) {
-					logger.LogIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
+					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
+					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
+				} else {
+					offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt.String(), err))
 				}
-				offlineEndpoints = append(offlineEndpoints, clnt.String())
 				continue
 			}
 			onlineServers++
@@ -221,15 +225,19 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 			// 100% CPU when half the endpoints are offline.
 			time.Sleep(100 * time.Millisecond)
 			retries++
-			// after 5 retries start logging that servers are not reachable yet
-			if retries >= 5 {
-				logger.Info(fmt.Sprintf("Waiting for atleast %d remote servers to be online for bootstrap check", len(clnts)/2))
+			// after 20 retries start logging that servers are not reachable yet
+			if retries >= 20 {
+				logger.Info(fmt.Sprintf("Waiting for atleast %d remote servers with valid configuration to be online", len(clnts)/2))
 				if len(offlineEndpoints) > 0 {
 					logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
 				}
+				if len(incorrectConfigs) > 0 {
+					logger.Info(fmt.Sprintf("Following servers mismatch in their configuration %s", incorrectConfigs))
+				}
 				retries = 0 // reset to log again after 5 retries.
 			}
 			offlineEndpoints = nil
+			incorrectConfigs = nil
 		}
 	}
 	return nil

cmd/bucket-encryption-handlers.go

@@ -204,7 +204,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Delete bucket encryption config from object layer
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
+	updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-encryption.go

@@ -34,15 +34,6 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 
 // Get - gets bucket encryption config for the given bucket.
 func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) {
-	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-
-		return nil, BucketSSEConfigNotFound{Bucket: bucket}
-	}
-
 	sseCfg, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
 	return sseCfg, err
 }

cmd/bucket-handlers.go

@@ -858,10 +858,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Call site replication hook
-	if err := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
 
 	// Make sure to add Location information here only for bucket
 	if cp := pathClean(r.URL.Path); cp != "" {
@@ -1065,10 +1062,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
 	sseConfig.Apply(r.Header, sse.ApplyOptions{
 		AutoEncrypt: globalAutoEncryption,
-		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
 	})
 
-	// get gateway encryption options
 	var opts ObjectOptions
 	opts, err = putOpts(ctx, r, bucket, object, metadata)
 	if err != nil {
@@ -1398,10 +1393,6 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -1618,7 +1609,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
+	updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-lifecycle-handlers.go

@@ -168,7 +168,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 		return
 	}
 
-	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle.go

@@ -58,15 +58,6 @@ type LifecycleSys struct{}
 
 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
-	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-
-		return nil, BucketLifecycleNotFound{Bucket: bucketName}
-	}
-
 	return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
 }
 
@@ -155,9 +146,14 @@ type newerNoncurrentTask struct {
 	versions []ObjectToDelete
 }
 
+type transitionTask struct {
+	tier    string
+	objInfo ObjectInfo
+}
+
 type transitionState struct {
 	once         sync.Once
-	transitionCh chan ObjectInfo
+	transitionCh chan transitionTask
 
 	ctx        context.Context
 	objAPI     ObjectLayer
@@ -171,13 +167,13 @@ type transitionState struct {
 	lastDayStats map[string]*lastDayTierStats
 }
 
-func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
+func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 	select {
 	case <-GlobalContext.Done():
 		t.once.Do(func() {
 			close(t.transitionCh)
 		})
-	case t.transitionCh <- oi:
+	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
 	default:
 	}
 }
@@ -186,7 +182,7 @@ var globalTransitionState *transitionState
 
 func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
 	return &transitionState{
-		transitionCh: make(chan ObjectInfo, 10000),
+		transitionCh: make(chan transitionTask, 10000),
 		ctx:          ctx,
 		objAPI:       objAPI,
 		killCh:       make(chan struct{}),
@@ -213,27 +209,25 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
 			return
 		case <-ctx.Done():
 			return
-		case oi, ok := <-t.transitionCh:
+		case task, ok := <-t.transitionCh:
 			if !ok {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			var tier string
-			var err error
-			if tier, err = transitionObject(ctx, objectAPI, oi); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
+			if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
+					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
-					TotalSize:   uint64(oi.Size),
+					TotalSize:   uint64(task.objInfo.Size),
 					NumVersions: 1,
 				}
-				if oi.IsLatest {
+				if task.objInfo.IsLatest {
 					ts.NumObjects = 1
 				}
-				t.addLastDayStats(tier, ts)
+				t.addLastDayStats(task.tier, ts)
 			}
 			atomic.AddInt32(&t.activeTasks, -1)
-
 		}
 	}
 }
@@ -304,9 +298,10 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
 // This is to be called after a successful upload of an object (version).
 func enqueueTransitionImmediate(obj ObjectInfo) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
-		switch lc.ComputeAction(obj.ToLifecycleOpts()) {
+		event := lc.Eval(obj.ToLifecycleOpts())
+		switch event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			globalTransitionState.queueTransitionTask(obj)
+			globalTransitionState.queueTransitionTask(obj, event.StorageClass)
 		}
 	}
 }
@@ -403,12 +398,7 @@ func genTransitionObjName(bucket string) (string, error) {
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) (string, error) {
-	lc, err := globalLifecycleSys.Get(oi.Bucket)
-	if err != nil {
-		return "", err
-	}
-	tier := lc.TransitionTier(oi.ToLifecycleOpts())
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
 	opts := ObjectOptions{
 		Transition: TransitionOptions{
 			Status: lifecycle.TransitionPending,
@@ -420,7 +410,7 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		MTime:            oi.ModTime,
 	}
-	return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
+	return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
 
 type auditTierOp struct {

cmd/bucket-listobjects-handlers.go

@@ -35,7 +35,7 @@ import (
 //   - delimiter if set should be equal to '/', otherwise the request is rejected.
 //   - marker if set should have a common prefix with 'prefix' param, otherwise
 //     the request is rejected.
-func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
+func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
 	// Max keys cannot be negative.
 	if maxKeys < 0 {
 		return ErrInvalidMaxKeys
@@ -48,6 +48,14 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
 		}
 	}
 
+	if !IsValidObjectPrefix(prefix) {
+		return ErrInvalidObjectName
+	}
+
+	if marker != "" && !HasPrefix(marker, prefix) {
+		return ErrNotImplemented
+	}
+
 	return ErrNone
 }
 
@@ -83,7 +91,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 	}
 
 	// Validate the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
@@ -148,7 +156,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
@@ -218,7 +226,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
@@ -325,7 +333,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	// Validate all the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -55,9 +54,6 @@ func (sys *BucketMetadataSys) Count() int {
 
 // Remove bucket metadata from memory.
 func (sys *BucketMetadataSys) Remove(bucket string) {
-	if globalIsGateway {
-		return
-	}
 	sys.Lock()
 	delete(sys.metadataMap, bucket)
 	globalBucketMonitor.DeleteBucket(bucket)
@@ -70,10 +66,6 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
 // so they should be replaced atomically and not appended to, etc.
 // Data is not persisted to disk.
 func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
-	if globalIsGateway {
-		return
-	}
-
 	if bucket != minioMetaBucket {
 		sys.Lock()
 		sys.metadataMap[bucket] = meta
@@ -81,33 +73,17 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 	}
 }
 
-// Update update bucket metadata for the specified config file.
-// The configData data should not be modified after being sent here.
-func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
+func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return updatedAt, errServerNotInitialized
 	}
 
-	if globalIsGateway && globalGatewayName != NASBackendGateway {
-		if configFile == bucketPolicyConfig {
-			if configData == nil {
-				return updatedAt, objAPI.DeleteBucketPolicy(ctx, bucket)
-			}
-			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
-			if err != nil {
-				return updatedAt, err
-			}
-			return updatedAt, objAPI.SetBucketPolicy(ctx, bucket, config)
-		}
-		return updatedAt, NotImplemented{}
-	}
-
 	if bucket == minioMetaBucket {
 		return updatedAt, errInvalidArgument
 	}
 
-	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
+	meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
 	if err != nil {
 		if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
 			// Only single drive mode needs this fallback.
@@ -165,6 +141,18 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 	return updatedAt, nil
 }
 
+// Delete delete the bucket metadata for the specified bucket.
+// must be used by all callers instead of using Update() with nil configData.
+func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
+	return sys.updateAndParse(ctx, bucket, configFile, nil, false)
+}
+
+// Update update bucket metadata for the specified bucket.
+// The configData data should not be modified after being sent here.
+func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
+	return sys.updateAndParse(ctx, bucket, configFile, configData, true)
+}
+
 // Get metadata for a bucket.
 // If no metadata exists errConfigNotFound is returned and a new metadata is returned.
 // Only a shallow copy is returned, so referenced data should not be modified,
@@ -176,7 +164,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 // For all other bucket specific metadata, use the relevant
 // calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
-	if globalIsGateway || bucket == minioMetaBucket {
+	if bucket == minioMetaBucket {
 		return newBucketMetadata(bucket), errConfigNotFound
 	}
 
@@ -239,22 +227,6 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
 // GetLifecycleConfig returns configured lifecycle config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
-	if globalIsGateway && globalGatewayName == NASBackendGateway {
-		// Only needed in case of NAS gateway.
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-		meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-		if err != nil {
-			return nil, err
-		}
-		if meta.lifecycleConfig == nil {
-			return nil, BucketLifecycleNotFound{Bucket: bucket}
-		}
-		return meta.lifecycleConfig, nil
-	}
-
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
@@ -271,19 +243,6 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
-	if globalIsGateway && globalGatewayName == NASBackendGateway {
-		// Only needed in case of NAS gateway.
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-		meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-		if err != nil {
-			return nil, err
-		}
-		return meta.notificationConfig, nil
-	}
-
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return nil, err
@@ -319,15 +278,6 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
-	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, time.Time{}, errServerNotInitialized
-		}
-		p, err := objAPI.GetBucketPolicy(GlobalContext, bucket)
-		return p, UTCNow(), err
-	}
-
 	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
@@ -395,10 +345,6 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (Buc
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}
 
-	if globalIsGateway {
-		return newBucketMetadata(bucket), NotImplemented{}
-	}
-
 	if bucket == minioMetaBucket {
 		return newBucketMetadata(bucket), errInvalidArgument
 	}
@@ -426,12 +372,6 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return errServerNotInitialized
 	}
 
-	// In gateway mode, we don't need to load bucket metadata except
-	// NAS gateway backend.
-	if globalIsGateway && !objAPI.IsNotificationSupported() {
-		return nil
-	}
-
 	// Load bucket metadata sys in background
 	go sys.load(ctx, buckets, objAPI)
 	return nil

cmd/bucket-metadata.go

@@ -162,8 +162,7 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 	return err
 }
 
-// loadBucketMetadata loads and migrates to bucket metadata.
-func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
+func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
 	err := b.Load(ctx, objectAPI, b.Name)
 	if err != nil && !errors.Is(err, errConfigNotFound) {
@@ -172,8 +171,22 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	if err == nil {
 		b.defaultTimestamps()
 	}
-	// Old bucket without bucket metadata. Hence we migrate existing settings.
-	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
+
+	configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
+	if err != nil {
+		return b, err
+	}
+
+	if len(configs) == 0 {
+		if parse {
+			// nothing to update, parse and proceed.
+			err = b.parseAllConfigs(ctx, objectAPI)
+		}
+	} else {
+		// Old bucket without bucket metadata. Hence we migrate existing settings.
+		err = b.convertLegacyConfigs(ctx, objectAPI, configs)
+	}
+	if err != nil {
 		return b, err
 	}
 
@@ -185,6 +198,11 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	return b, nil
 }
 
+// loadBucketMetadata loads and migrates to bucket metadata.
+func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
+	return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
+}
+
 // parseAllConfigs will parse all configs and populate the private fields.
 // The first error encountered is returned.
 func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
@@ -277,7 +295,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 	return nil
 }
 
-func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
+func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) (map[string][]byte, error) {
 	legacyConfigs := []string{
 		legacyBucketObjectLockEnabledConfigFile,
 		bucketPolicyConfig,
@@ -291,7 +309,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		objectLockConfig,
 	}
 
-	configs := make(map[string][]byte)
+	configs := make(map[string][]byte, len(legacyConfigs))
 
 	// Handle migration from lockEnabled to newer format.
 	if b.LockEnabled {
@@ -316,16 +334,15 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 				continue
 			}
 
-			return err
+			return nil, err
 		}
 		configs[legacyFile] = configData
 	}
 
-	if len(configs) == 0 {
-		// nothing to update, return right away.
-		return b.parseAllConfigs(ctx, objectAPI)
-	}
+	return configs, nil
+}
 
+func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
 	for legacyFile, configData := range configs {
 		switch legacyFile {
 		case legacyBucketObjectLockEnabledConfigFile:
@@ -425,23 +442,6 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 	return saveConfig(ctx, api, configFile, data)
 }
 
-// deleteBucketMetadata deletes bucket metadata
-// If config does not exist no error is returned.
-func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string) error {
-	metadataFiles := []string{
-		dataUsageCacheName,
-		bucketMetadataFile,
-		path.Join(replicationDir, resyncFileName),
-	}
-	for _, metaFile := range metadataFiles {
-		configFile := path.Join(bucketMetaPrefix, bucket, metaFile)
-		if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
-			return err
-		}
-	}
-	return nil
-}
-
 // migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
 func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
 	var err error

cmd/bucket-object-lock.go

@@ -36,15 +36,6 @@ type BucketObjectLockSys struct{}
 
 // Get - Get retention configuration.
 func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
-	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return r, errServerNotInitialized
-		}
-
-		return r, nil
-	}
-
 	config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
 	if err != nil {
 		if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucketName}) {

cmd/bucket-policy-handlers.go

@@ -149,7 +149,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
+	updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-quota.go

@@ -35,13 +35,6 @@ type BucketQuotaSys struct {
 
 // Get - Get quota configuration.
 func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
-	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-		return &madmin.BucketQuota{}, nil
-	}
 	qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
 	return qCfg, err
 }

cmd/bucket-replication-handlers.go

@@ -46,10 +46,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	if globalIsGateway {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -59,7 +55,10 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	if globalSiteReplicationSys.isEnabled() && logger.GetReqInfo(ctx).Cred.AccessKey != globalActiveCred.AccessKey {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
 		return
@@ -165,7 +164,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
 		return
 	}
-	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketReplicationConfig); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -277,7 +276,13 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if !config.HasExistingObjectReplication(arn) {
+	hasARN, hasExistingObjEnabled := config.HasExistingObjectReplication(arn)
+	if !hasARN {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrRemoteTargetNotFoundError), r.URL)
+		return
+	}
+
+	if !hasExistingObjEnabled {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoExistingObjects), r.URL)
 		return
 	}
@@ -316,7 +321,27 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		}
 	}
-	if err := startReplicationResync(ctx, bucket, arn, resetID, resetBeforeDate, objectAPI); err != nil {
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	tgtBytes, err := json.Marshal(&targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if err := globalReplicationPool.resyncer.start(ctx, objectAPI, resyncOpts{
+		bucket:       bucket,
+		arn:          arn,
+		resyncID:     resetID,
+		resyncBefore: resetBeforeDate,
+	}); err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
 			Bucket: bucket,
 			Err:    err,
@@ -365,10 +390,13 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	globalReplicationPool.resyncState.RLock()
-	brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
-	globalReplicationPool.resyncState.RUnlock()
+	var tgtStats map[string]TargetReplicationResyncStatus
+	globalReplicationPool.resyncer.RLock()
+	brs, ok := globalReplicationPool.resyncer.statusMap[bucket]
+	if ok {
+		tgtStats = brs.cloneTgtStats()
+	}
+	globalReplicationPool.resyncer.RUnlock()
 	if !ok {
 		brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
 		if err != nil {
@@ -378,10 +406,11 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 			}), r.URL)
 			return
 		}
+		tgtStats = brs.cloneTgtStats()
 	}
 
 	var rinfo ResyncTargetsInfo
-	for tarn, st := range brs.TargetsMap {
+	for tarn, st := range tgtStats {
 		if arn != "" && tarn != arn {
 			continue
 		}
@@ -389,7 +418,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 			Arn:             tarn,
 			ResetID:         st.ResyncID,
 			StartTime:       st.StartTime,
-			EndTime:         st.EndTime,
+			EndTime:         st.LastUpdate,
 			ResyncStatus:    st.ResyncStatus.String(),
 			ReplicatedSize:  st.ReplicatedSize,
 			ReplicatedCount: st.ReplicatedCount,

cmd/bucket-replication-stats.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/binary"
 	"math"
 	"sync"
 	"time"
@@ -37,12 +38,12 @@ func (b *BucketReplicationStats) hasReplicationUsage() bool {
 
 // ReplicationStats holds the global in-memory replication stats
 type ReplicationStats struct {
-	Cache           map[string]*BucketReplicationStats
-	UsageCache      map[string]*BucketReplicationStats
-	mostRecentStats BucketStatsMap
-	sync.RWMutex                 // mutex for Cache
-	ulock           sync.RWMutex // mutex for UsageCache
-	dlock           sync.RWMutex // mutex for mostRecentStats
+	Cache             map[string]*BucketReplicationStats
+	UsageCache        map[string]*BucketReplicationStats
+	mostRecentStats   BucketStatsMap
+	sync.RWMutex                   // mutex for Cache
+	ulock             sync.RWMutex // mutex for UsageCache
+	mostRecentStatsMu sync.Mutex   // mutex for mostRecentStats
 }
 
 // Delete deletes in-memory replication statistics for a bucket.
@@ -245,13 +246,24 @@ outer:
 	r.ulock.Unlock()
 }
 
-func (r *ReplicationStats) getAllCachedLatest() BucketStatsMap {
+// serializeStats will serialize the current stats.
+// Will return (nil, nil) if no data.
+func (r *ReplicationStats) serializeStats() ([]byte, error) {
 	if r == nil {
-		return BucketStatsMap{}
+		return nil, nil
 	}
-	r.dlock.RLock()
-	defer r.dlock.RUnlock()
-	return r.mostRecentStats
+	r.mostRecentStatsMu.Lock()
+	defer r.mostRecentStatsMu.Unlock()
+	bsm := r.mostRecentStats
+	if len(bsm.Stats) == 0 {
+		return nil, nil
+	}
+	data := make([]byte, 4, 4+bsm.Msgsize())
+	// Add the replication stats meta header.
+	binary.LittleEndian.PutUint16(data[0:2], replStatsMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], replStatsVersion)
+	// Add data
+	return r.mostRecentStats.MarshalMsg(data)
 }
 
 func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
@@ -356,13 +368,13 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u Buck
 	// normalize overall stats
 	s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
 	s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
-	r.dlock.Lock()
+	r.mostRecentStatsMu.Lock()
 	if len(r.mostRecentStats.Stats) == 0 {
 		r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
 	}
 	r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
 	r.mostRecentStats.Timestamp = UTCNow()
-	r.dlock.Unlock()
+	r.mostRecentStatsMu.Unlock()
 	return s
 }
 

cmd/bucket-replication-utils.go

@@ -628,9 +628,12 @@ func (v VersionPurgeStatusType) Pending() bool {
 	return v == Pending || v == Failed
 }
 
-type replicationResyncState struct {
+type replicationResyncer struct {
 	// map of bucket to their resync status
-	statusMap map[string]BucketReplicationResyncStatus
+	statusMap      map[string]BucketReplicationResyncStatus
+	workerSize     int
+	resyncCancelCh chan struct{}
+	workerCh       chan struct{}
 	sync.RWMutex
 }
 
@@ -642,12 +645,23 @@ const (
 	resyncMetaVersion   = resyncMetaVersionV1
 )
 
+type resyncOpts struct {
+	bucket       string
+	arn          string
+	resyncID     string
+	resyncBefore time.Time
+}
+
 // ResyncStatusType status of resync operation
 type ResyncStatusType int
 
 const (
 	// NoResync - no resync in progress
 	NoResync ResyncStatusType = iota
+	// ResyncPending - resync pending
+	ResyncPending
+	// ResyncCanceled - resync canceled
+	ResyncCanceled
 	// ResyncStarted -  resync in progress
 	ResyncStarted
 	// ResyncCompleted -  resync finished
@@ -656,6 +670,10 @@ const (
 	ResyncFailed
 )
 
+func (rt ResyncStatusType) isValid() bool {
+	return rt != NoResync
+}
+
 func (rt ResyncStatusType) String() string {
 	switch rt {
 	case ResyncStarted:
@@ -664,6 +682,10 @@ func (rt ResyncStatusType) String() string {
 		return "Completed"
 	case ResyncFailed:
 		return "Failed"
+	case ResyncPending:
+		return "Pending"
+	case ResyncCanceled:
+		return "Canceled"
 	default:
 		return ""
 	}
@@ -671,8 +693,8 @@ func (rt ResyncStatusType) String() string {
 
 // TargetReplicationResyncStatus status of resync of bucket for a specific target
 type TargetReplicationResyncStatus struct {
-	StartTime time.Time `json:"startTime" msg:"st"`
-	EndTime   time.Time `json:"endTime" msg:"et"`
+	StartTime  time.Time `json:"startTime" msg:"st"`
+	LastUpdate time.Time `json:"lastUpdated" msg:"lst"`
 	// Resync ID assigned to this reset
 	ResyncID string `json:"resyncID" msg:"id"`
 	// ResyncBeforeDate - resync all objects created prior to this date
@@ -701,6 +723,14 @@ type BucketReplicationResyncStatus struct {
 	LastUpdate time.Time                                `json:"lastUpdate" msg:"lu"`
 }
 
+func (rs *BucketReplicationResyncStatus) cloneTgtStats() (m map[string]TargetReplicationResyncStatus) {
+	m = make(map[string]TargetReplicationResyncStatus)
+	for arn, st := range rs.TargetsMap {
+		m[arn] = st
+	}
+	return
+}
+
 func newBucketResyncStatus(bucket string) BucketReplicationResyncStatus {
 	return BucketReplicationResyncStatus{
 		TargetsMap: make(map[string]TargetReplicationResyncStatus),

cmd/bucket-replication-utils_gen.go

@@ -2117,10 +2117,10 @@ func (z *TargetReplicationResyncStatus) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "StartTime")
 				return
 			}
-		case "et":
-			z.EndTime, err = dc.ReadTime()
+		case "lst":
+			z.LastUpdate, err = dc.ReadTime()
 			if err != nil {
-				err = msgp.WrapError(err, "EndTime")
+				err = msgp.WrapError(err, "LastUpdate")
 				return
 			}
 		case "id":
@@ -2205,14 +2205,14 @@ func (z *TargetReplicationResyncStatus) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "StartTime")
 		return
 	}
-	// write "et"
-	err = en.Append(0xa2, 0x65, 0x74)
+	// write "lst"
+	err = en.Append(0xa3, 0x6c, 0x73, 0x74)
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.EndTime)
+	err = en.WriteTime(z.LastUpdate)
 	if err != nil {
-		err = msgp.WrapError(err, "EndTime")
+		err = msgp.WrapError(err, "LastUpdate")
 		return
 	}
 	// write "id"
@@ -2315,9 +2315,9 @@ func (z *TargetReplicationResyncStatus) MarshalMsg(b []byte) (o []byte, err erro
 	// string "st"
 	o = append(o, 0x8b, 0xa2, 0x73, 0x74)
 	o = msgp.AppendTime(o, z.StartTime)
-	// string "et"
-	o = append(o, 0xa2, 0x65, 0x74)
-	o = msgp.AppendTime(o, z.EndTime)
+	// string "lst"
+	o = append(o, 0xa3, 0x6c, 0x73, 0x74)
+	o = msgp.AppendTime(o, z.LastUpdate)
 	// string "id"
 	o = append(o, 0xa2, 0x69, 0x64)
 	o = msgp.AppendString(o, z.ResyncID)
@@ -2372,10 +2372,10 @@ func (z *TargetReplicationResyncStatus) UnmarshalMsg(bts []byte) (o []byte, err
 				err = msgp.WrapError(err, "StartTime")
 				return
 			}
-		case "et":
-			z.EndTime, bts, err = msgp.ReadTimeBytes(bts)
+		case "lst":
+			z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "EndTime")
+				err = msgp.WrapError(err, "LastUpdate")
 				return
 			}
 		case "id":
@@ -2450,7 +2450,7 @@ func (z *TargetReplicationResyncStatus) UnmarshalMsg(bts []byte) (o []byte, err
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *TargetReplicationResyncStatus) Msgsize() (s int) {
-	s = 1 + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.StringPrefixSize + len(z.ResyncID) + 4 + msgp.TimeSize + 4 + msgp.IntSize + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object)
+	s = 1 + 3 + msgp.TimeSize + 4 + msgp.TimeSize + 3 + msgp.StringPrefixSize + len(z.ResyncID) + 4 + msgp.TimeSize + 4 + msgp.IntSize + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object)
 	return
 }
 

cmd/bucket-targets.go

@@ -30,7 +30,6 @@ import (
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/bucket/replication"
-	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -209,9 +208,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
 
 // SetTarget - sets a new minio-go client target for this bucket.
 func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error {
-	if globalIsGateway {
-		return nil
-	}
 	if !tgt.Type.IsValid() && !update {
 		return BucketRemoteArnTypeInvalid{Bucket: bucket}
 	}
@@ -220,12 +216,16 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 	}
 	// validate if target credentials are ok
-	if _, err = clnt.BucketExists(ctx, tgt.TargetBucket); err != nil {
+	exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)
+	if err != nil {
 		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
 			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 		}
 		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 	}
+	if !exists {
+		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+	}
 	if tgt.Type == madmin.ReplicationService {
 		if !globalBucketVersioningSys.Enabled(bucket) {
 			return BucketReplicationSourceNotVersioned{Bucket: bucket}
@@ -234,7 +234,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		if err != nil {
 			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 		}
-		if vcfg.Status != string(versioning.Enabled) {
+		if !vcfg.Enabled() {
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
 		}
 	}
@@ -268,9 +268,6 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 }
 
 func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
-	if globalIsGateway {
-		return
-	}
 	if limit == 0 {
 		globalBucketMonitor.DeleteBucket(bucket)
 		return
@@ -282,10 +279,6 @@ func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
 
 // RemoveTarget - removes a remote bucket target for this source bucket.
 func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
-	if globalIsGateway {
-		return nil
-	}
-
 	if arnStr == "" {
 		return BucketRemoteArnInvalid{Bucket: bucket}
 	}
@@ -494,6 +487,22 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 	return generateARN(target)
 }
 
+// getRemoteARNForPeer returns the remote target for a peer site in site replication
+func (sys *BucketTargetSys) getRemoteARNForPeer(bucket string, peer madmin.PeerInfo) string {
+	tgts := sys.targetsMap[bucket]
+	for _, target := range tgts {
+		ep, _ := url.Parse(peer.Endpoint)
+		if target.SourceBucket == bucket &&
+			target.TargetBucket == bucket &&
+			target.Endpoint == ep.Host &&
+			target.Secure == (ep.Scheme == "https") &&
+			target.Type == madmin.ReplicationService {
+			return target.Arn
+		}
+	}
+	return ""
+}
+
 // generate ARN that is unique to this target type
 func generateARN(t *madmin.BucketTarget) string {
 	arn := madmin.ARN{

cmd/bucket-versioning.go

@@ -69,11 +69,6 @@ func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool {
 
 // Get returns stored bucket policy
 func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) {
-	if globalIsGateway {
-		// Gateway does not implement versioning.
-		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
-	}
-
 	if bucket == minioMetaBucket || strings.HasPrefix(bucket, minioMetaBucket) {
 		return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil
 	}

cmd/callhome.go

@@ -18,53 +18,28 @@
 package cmd
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"math/rand"
+	"net/url"
 	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
-	uatomic "go.uber.org/atomic"
 )
 
-const (
-	// callhomeSchemaVersion1 is callhome schema version 1
-	callhomeSchemaVersion1 = "1"
-
-	// callhomeSchemaVersion is current callhome schema version.
-	callhomeSchemaVersion = callhomeSchemaVersion1
-
-	// callhomeCycleDefault is the default interval between two callhome cycles (24hrs)
-	callhomeCycleDefault = 24 * time.Hour
-)
-
-// CallhomeInfo - Contains callhome information
-type CallhomeInfo struct {
-	SchemaVersion string             `json:"schema_version"`
-	AdminInfo     madmin.InfoMessage `json:"admin_info"`
-}
-
-var (
-	enableCallhome            = uatomic.NewBool(false)
-	callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
-	callhomeFreq              = uatomic.NewDuration(callhomeCycleDefault)
-)
-
-func updateCallhomeParams(ctx context.Context, objAPI ObjectLayer) {
-	alreadyEnabled := enableCallhome.Load()
-	enableCallhome.Store(globalCallhomeConfig.Enable)
-	callhomeFreq.Store(globalCallhomeConfig.Frequency)
-
-	// If callhome was disabled earlier and has now been enabled,
-	// initialize the callhome process again.
-	if !alreadyEnabled && enableCallhome.Load() {
-		initCallhome(ctx, objAPI)
-	}
-}
+var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
 // initCallhome will start the callhome task in the background.
 func initCallhome(ctx context.Context, objAPI ObjectLayer) {
+	if !globalCallhomeConfig.Enabled() {
+		return
+	}
+
 	go func() {
 		r := rand.New(rand.NewSource(time.Now().UnixNano()))
 		// Leader node (that successfully acquires the lock inside runCallhome)
@@ -72,79 +47,154 @@ func initCallhome(ctx context.Context, objAPI ObjectLayer) {
 		// the lock will be released and another node will acquire it and take over
 		// because of this loop.
 		for {
-			runCallhome(ctx, objAPI)
-			if !enableCallhome.Load() {
+			if !globalCallhomeConfig.Enabled() {
+				return
+			}
+
+			if !runCallhome(ctx, objAPI) {
+				// callhome was disabled or context was canceled
 				return
 			}
 
 			// callhome running on a different node.
 			// sleep for some time and try again.
-			duration := time.Duration(r.Float64() * float64(callhomeFreq.Load()))
+			duration := time.Duration(r.Float64() * float64(globalCallhomeConfig.FrequencyDur()))
 			if duration < time.Second {
 				// Make sure to sleep atleast a second to avoid high CPU ticks.
 				duration = time.Second
 			}
 			time.Sleep(duration)
-
-			if !enableCallhome.Load() {
-				return
-			}
 		}
 	}()
 }
 
-func runCallhome(ctx context.Context, objAPI ObjectLayer) {
+func runCallhome(ctx context.Context, objAPI ObjectLayer) bool {
 	// Make sure only 1 callhome is running on the cluster.
 	locker := objAPI.NewNSLock(minioMetaBucket, "callhome/runCallhome.lock")
 	lkctx, err := locker.GetLock(ctx, callhomeLeaderLockTimeout)
 	if err != nil {
-		return
+		// lock timedout means some other node is the leader,
+		// cycle back return 'true'
+		return true
 	}
 
 	ctx = lkctx.Context()
 	defer locker.Unlock(lkctx.Cancel)
 
-	callhomeTimer := time.NewTimer(callhomeFreq.Load())
+	callhomeTimer := time.NewTimer(globalCallhomeConfig.FrequencyDur())
 	defer callhomeTimer.Stop()
 
 	for {
+		if !globalCallhomeConfig.Enabled() {
+			// Stop the processing as callhome got disabled
+			return false
+		}
+
 		select {
 		case <-ctx.Done():
-			return
+			// indicates that we do not need to run callhome anymore
+			return false
 		case <-callhomeTimer.C:
-			if !enableCallhome.Load() {
+			if !globalCallhomeConfig.Enabled() {
 				// Stop the processing as callhome got disabled
-				return
+				return false
 			}
+
 			performCallhome(ctx)
 
 			// Reset the timer for next cycle.
-			callhomeTimer.Reset(callhomeFreq.Load())
+			callhomeTimer.Reset(globalCallhomeConfig.FrequencyDur())
 		}
 	}
 }
 
 func performCallhome(ctx context.Context) {
-	err := sendCallhomeInfo(
-		CallhomeInfo{
-			SchemaVersion: callhomeSchemaVersion,
-			AdminInfo:     getServerInfo(ctx, nil),
-		})
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
+	deadline := 10 * time.Second // Default deadline is 10secs for callhome
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		logger.LogIf(ctx, errors.New("Callhome: object layer not ready"))
+		return
+	}
+
+	healthCtx, healthCancel := context.WithTimeout(ctx, deadline)
+	defer healthCancel()
+
+	healthInfoCh := make(chan madmin.HealthInfo)
+
+	query := url.Values{}
+	for _, k := range madmin.HealthDataTypesList {
+		query.Set(string(k), "true")
+	}
+
+	healthInfo := madmin.HealthInfo{
+		TimeStamp: time.Now().UTC(),
+		Version:   madmin.HealthInfoVersion,
+		Minio: madmin.MinioHealthInfo{
+			Info: madmin.MinioInfo{
+				DeploymentID: globalDeploymentID,
+			},
+		},
+	}
+
+	go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo)
+
+	for {
+		select {
+		case hi, hasMore := <-healthInfoCh:
+			if !hasMore {
+				// Received all data. Send to SUBNET and return
+				err := sendHealthInfo(ctx, healthInfo)
+				if err != nil {
+					logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
+				}
+				return
+			}
+			healthInfo = hi
+		case <-healthCtx.Done():
+			return
+		}
 	}
 }
 
 const (
-	callhomeURL    = "https://subnet.min.io/api/callhome"
-	callhomeURLDev = "http://localhost:9000/api/callhome"
+	healthURL    = "https://subnet.min.io/api/health/upload"
+	healthURLDev = "http://localhost:9000/api/health/upload"
 )
 
-func sendCallhomeInfo(ch CallhomeInfo) error {
-	url := callhomeURL
+func sendHealthInfo(ctx context.Context, healthInfo madmin.HealthInfo) error {
+	url := healthURL
 	if globalIsCICD {
-		url = callhomeURLDev
+		url = healthURLDev
 	}
-	_, err := globalSubnetConfig.Post(url, ch)
+
+	filename := fmt.Sprintf("health_%s.json.gz", UTCNow().Format("20060102150405"))
+	url += "?filename=" + filename
+
+	_, err := globalSubnetConfig.Upload(url, filename, createHealthJSONGzip(ctx, healthInfo))
 	return err
 }
+
+func createHealthJSONGzip(ctx context.Context, healthInfo madmin.HealthInfo) []byte {
+	var b bytes.Buffer
+	gzWriter := gzip.NewWriter(&b)
+
+	header := struct {
+		Version string `json:"version"`
+	}{Version: healthInfo.Version}
+
+	enc := json.NewEncoder(gzWriter)
+	if e := enc.Encode(header); e != nil {
+		logger.LogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e))
+		return nil
+	}
+
+	if e := enc.Encode(healthInfo); e != nil {
+		logger.LogIf(ctx, fmt.Errorf("Could not encode health info: %w", e))
+		return nil
+	}
+
+	gzWriter.Flush()
+	gzWriter.Close()
+
+	return b.Bytes()
+}

cmd/common-main.go

@@ -134,7 +134,7 @@ func init() {
 
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
-		RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
+		RoundTripper: newHTTPTransport(1 * time.Hour),
 		Logger: func(err error) {
 			if err != nil && !errors.Is(err, context.Canceled) {
 				logger.LogIf(GlobalContext, err)
@@ -151,7 +151,7 @@ func init() {
 	defaultAWSCredProvider = []credentials.Provider{
 		&credentials.IAM{
 			Client: &http.Client{
-				Transport: NewGatewayHTTPTransport(),
+				Transport: NewHTTPTransport(),
 			},
 		},
 	}
@@ -175,7 +175,9 @@ func minioConfigToConsoleFeatures() {
 	if globalMinioEndpoint != "" {
 		os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
 	} else {
-		os.Setenv("CONSOLE_MINIO_SERVER", getAPIEndpoints()[0])
+		// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
+		// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
+		os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
 	}
 	if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {
 		os.Setenv("CONSOLE_LOG_QUERY_URL", value)
@@ -202,20 +204,13 @@ func minioConfigToConsoleFeatures() {
 		}
 	}
 	// Enable if LDAP is enabled.
-	if globalLDAPConfig.Enabled {
+	if globalLDAPConfig.Enabled() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
-	if globalSubnetConfig.License != "" {
-		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
-	}
-	if globalSubnetConfig.APIKey != "" {
-		os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
-	}
-	if globalSubnetConfig.ProxyURL != nil {
-		os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.ProxyURL.String())
-	}
+
+	globalSubnetConfig.ApplyEnv()
 }
 
 func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
@@ -236,6 +231,7 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
 			Userinfo:                cfg.ClaimUserinfo,
 			RedirectCallbackDynamic: cfg.RedirectURIDynamic,
 			RedirectCallback:        callback,
+			RoleArn:                 cfg.GetRoleArn(),
 		}
 	}
 	return m
@@ -305,22 +301,6 @@ func initConsoleServer() (*restapi.Server, error) {
 	return server, nil
 }
 
-func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
-	if strings.HasPrefix(name, "gateway") {
-		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
-			uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
-			logger.Fatal(uiErr, "Unable to start gateway with SSE")
-		}
-	}
-
-	globalCompressConfigMu.Lock()
-	if globalCompressConfig.Enabled && !objAPI.IsCompressionSupported() {
-		logger.Fatal(errInvalidArgument,
-			"Compression support is requested but '%s' does not support compression", name)
-	}
-	globalCompressConfigMu.Unlock()
-}
-
 // Check for updates and print a notification message
 func checkUpdate(mode string) {
 	updateURL := minioReleaseInfoURL

cmd/config-common.go

@@ -29,10 +29,14 @@ import (
 
 var errConfigNotFound = errors.New("config file not found")
 
-func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string) ([]byte, ObjectInfo, error) {
-	r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
+func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string, opts ObjectOptions) ([]byte, ObjectInfo, error) {
+	lockType := readLock
+	if opts.NoLock {
+		lockType = noLock // erasureObjects.GetObjectNInfo honors lockType argument but not opts.NoLock.
+	}
+
+	r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, lockType, opts)
 	if err != nil {
-		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {
 			return nil, ObjectInfo{}, errConfigNotFound
 		}
@@ -52,7 +56,7 @@ func readConfigWithMetadata(ctx context.Context, store objectIO, configFile stri
 }
 
 func readConfig(ctx context.Context, store objectIO, configFile string) ([]byte, error) {
-	buf, _, err := readConfigWithMetadata(ctx, store, configFile)
+	buf, _, err := readConfigWithMetadata(ctx, store, configFile, ObjectOptions{})
 	return buf, err
 }
 
@@ -70,16 +74,20 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
 	return err
 }
 
-func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
+func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error {
 	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
 	if err != nil {
 		return err
 	}
 
-	_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{MaxParity: true})
+	_, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), opts)
 	return err
 }
 
+func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error {
+	return saveConfigWithOpts(ctx, store, configFile, data, ObjectOptions{MaxParity: true})
+}
+
 func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
 	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
 		// Treat object not found as config not found.

cmd/config-current.go

@@ -331,16 +331,16 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 		}
 	case config.IdentityOpenIDSubSys:
 		if _, err := openid.LookupConfig(s,
-			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
+			NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 			return err
 		}
 	case config.IdentityLDAPSubSys:
-		cfg, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default], globalRootCAs)
+		cfg, err := xldap.Lookup(s, globalRootCAs)
 		if err != nil {
 			return err
 		}
-		if cfg.Enabled {
-			conn, cerr := cfg.Connect()
+		if cfg.Enabled() {
+			conn, cerr := cfg.LDAP.Connect()
 			if cerr != nil {
 				return cerr
 			}
@@ -352,7 +352,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 		}
 	case config.IdentityPluginSubSys:
 		if _, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
-			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
+			NewHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 			return err
 		}
 	case config.SubnetSubSys:
@@ -360,9 +360,14 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	case config.CallhomeSubSys:
-		if _, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default]); err != nil {
+		cfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
+		if err != nil {
 			return err
 		}
+		// callhome cannot be enabled if license is not registered yet, throw an error.
+		if cfg.Enabled() && !globalSubnetConfig.Registered() {
+			return errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'")
+		}
 	case config.PolicyOPASubSys:
 		// In case legacy OPA config is being set, we treat it as if the
 		// AuthZPlugin is being set.
@@ -370,12 +375,12 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 		fallthrough
 	case config.PolicyPluginSubSys:
 		if ppargs, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
-			NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+			NewHTTPTransport(), xhttp.DrainBody); err != nil {
 			return err
 		} else if ppargs.URL == nil {
 			// Check if legacy opa is configured.
 			if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-				NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+				NewHTTPTransport(), xhttp.DrainBody); err != nil {
 				return err
 			}
 		}
@@ -388,7 +393,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	}
 
 	if config.NotifySubSystems.Contains(subSys) {
-		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewGatewayHTTPTransport()); err != nil {
+		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
 			return err
 		}
 	}
@@ -434,42 +439,26 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
 	if err != nil {
-		if globalIsGateway {
-			logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
-		} else {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
-		}
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 	}
 	if err == nil && dnsURL != "" {
 		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
 		if err != nil {
-			if globalIsGateway {
-				logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
-			} else {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
-			}
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 		}
 	}
 
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
-		if globalIsGateway {
-			logger.FatalIf(err, "Unable to initialize etcd config")
-		} else {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
-		}
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 	}
 
 	if etcdCfg.Enabled {
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
-			if globalIsGateway {
-				logger.FatalIf(err, "Unable to initialize etcd config")
-			} else {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
-			}
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 		}
 
 		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
@@ -485,12 +474,8 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 					dns.CoreDNSPath(etcdCfg.CoreDNSPath),
 				)
 				if err != nil {
-					if globalIsGateway {
-						logger.FatalIf(err, "Unable to initialize DNS config")
-					} else {
-						logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-							globalDomainNames, err))
-					}
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+						globalDomainNames, err))
 				}
 			}
 		}
@@ -510,11 +495,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
-		if globalIsGateway {
-			logger.FatalIf(err, "Unable to setup cache")
-		} else {
-			logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
-		}
+		logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
 	}
 
 	if globalCacheConfig.Enabled {
@@ -540,12 +521,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
-	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
-	}
-
-	transport := NewGatewayHTTPTransport()
+	transport := NewHTTPTransport()
 
 	globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
@@ -554,11 +530,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
-		if globalIsGateway {
-			logger.FatalIf(err, "Unable to initialize dynamic configuration")
-		} else {
-			logger.LogIf(ctx, err)
-		}
+		logger.LogIf(ctx, err)
 	}
 }
 
@@ -579,7 +551,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 
 		// Initialize remote instance transport once.
 		getRemoteInstanceTransportOnce.Do(func() {
-			getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
+			getRemoteInstanceTransport = newHTTPTransport(apiConfig.RemoteTransportDeadline)
 		})
 	case config.CompressionSubSys:
 		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
@@ -617,12 +589,12 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			if l.Enabled {
 				l.LogOnce = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
-				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
-				loggerCfg.HTTP[n] = l
+				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
+			loggerCfg.HTTP[n] = l
 		}
-		if err = logger.UpdateSystemTargets(loggerCfg); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
+		if errs := logger.UpdateSystemTargets(loggerCfg); len(errs) > 0 {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditWebhookSubSys)
@@ -634,13 +606,13 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			if l.Enabled {
 				l.LogOnce = logger.LogOnceConsoleIf
 				l.UserAgent = userAgent
-				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
-				loggerCfg.AuditWebhook[n] = l
+				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
+			loggerCfg.AuditWebhook[n] = l
 		}
 
-		if err = logger.UpdateAuditWebhookTargets(loggerCfg); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
+		if errs := logger.UpdateAuditWebhookTargets(loggerCfg); len(errs) > 0 {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditKafkaSubSys)
@@ -653,8 +625,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		if err = logger.UpdateAuditKafkaTargets(loggerCfg); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
+		if errs := logger.UpdateAuditKafkaTargets(loggerCfg); len(errs) > 0 {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
 		}
 	case config.StorageClassSubSys:
 		for i, setDriveCount := range setDriveCounts {
@@ -669,13 +641,24 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				globalStorageClass.Update(sc)
 			}
 		}
+	case config.SubnetSubSys:
+		subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
+		} else {
+			globalSubnetConfig.Update(subnetConfig)
+			globalSubnetConfig.ApplyEnv() // update environment settings for Console UI
+		}
 	case config.CallhomeSubSys:
 		callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
 		} else {
-			globalCallhomeConfig = callhomeCfg
-			updateCallhomeParams(ctx, objAPI)
+			enable := callhomeCfg.Enable && !globalCallhomeConfig.Enabled()
+			globalCallhomeConfig.Update(callhomeCfg)
+			if enable {
+				initCallhome(ctx, objAPI)
+			}
 		}
 	}
 	globalServerConfigMu.Lock()

cmd/config-current_test.go

@@ -26,7 +26,10 @@ import (
 )
 
 func TestServerConfig(t *testing.T) {
-	objLayer, fsDir, err := prepareFS()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	objLayer, fsDir, err := prepareFS(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/config-migrate_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
@@ -27,7 +28,10 @@ import (
 
 // Test if config v1 is purged
 func TestServerConfigMigrateV1(t *testing.T) {
-	objLayer, fsDir, err := prepareFS()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	objLayer, fsDir, err := prepareFS(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -154,10 +158,12 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 // Test if a config migration from v2 to v33 is successfully done
 func TestServerConfigMigrateV2toV33(t *testing.T) {
 	rootPath := t.TempDir()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 
-	objLayer, fsDir, err := prepareFS()
+	objLayer, fsDir, err := prepareFS(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/config-versions.go

@@ -855,5 +855,5 @@ type serverConfigV33 struct {
 		// Add new external policy enforcements here.
 	} `json:"policy"`
 
-	LDAPServerConfig xldap.Config `json:"ldapserverconfig"`
+	LDAPServerConfig xldap.LegacyConfig `json:"ldapserverconfig"`
 }

cmd/consolelogger.go

@@ -21,6 +21,7 @@ import (
 	"container/ring"
 	"context"
 	"sync"
+	"sync/atomic"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
@@ -36,8 +37,11 @@ const defaultLogBufferCount = 10000
 
 // HTTPConsoleLoggerSys holds global console logger state
 type HTTPConsoleLoggerSys struct {
+	totalMessages  int64
+	failedMessages int64
+
 	sync.RWMutex
-	pubsub   *pubsub.PubSub
+	pubsub   *pubsub.PubSub[log.Info, madmin.LogMask]
 	console  *console.Target
 	nodeName string
 	logBuf   *ring.Ring
@@ -46,14 +50,18 @@ type HTTPConsoleLoggerSys struct {
 // NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to
 // the console logging pub sub system
 func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
-	ps := pubsub.New(8)
 	return &HTTPConsoleLoggerSys{
-		pubsub:  ps,
+		pubsub:  pubsub.New[log.Info, madmin.LogMask](8),
 		console: console.New(),
 		logBuf:  ring.New(defaultLogBufferCount),
 	}
 }
 
+// IsOnline always true in case of console logger
+func (sys *HTTPConsoleLoggerSys) IsOnline() bool {
+	return true
+}
+
 // SetNodeName - sets the node name if any after distributed setup has initialized
 func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
 	if !globalIsDistErasure {
@@ -72,11 +80,11 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
 // HasLogListeners returns true if console log listeners are registered
 // for this node or peers
 func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
-	return sys != nil && sys.pubsub.NumSubscribers(madmin.LogMaskAll) > 0
+	return sys != nil && sys.pubsub.Subscribers() > 0
 }
 
 // Subscribe starts console logging for this node.
-func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry pubsub.Maskable) bool) error {
+func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry log.Info) bool) error {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
 		logger.AddSystemTarget(sys)
@@ -116,7 +124,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-
 			}
 		}
 	}
-	return sys.pubsub.Subscribe(pubsub.MaskFromMaskable(madmin.LogMaskAll), subCh, doneCh, filter)
+	return sys.pubsub.Subscribe(madmin.LogMaskAll, subCh, doneCh, filter)
 }
 
 // Init if HTTPConsoleLoggerSys is valid, always returns nil right now
@@ -134,6 +142,15 @@ func (sys *HTTPConsoleLoggerSys) String() string {
 	return logger.ConsoleLoggerTgt
 }
 
+// Stats returns the target statistics.
+func (sys *HTTPConsoleLoggerSys) Stats() types.TargetStats {
+	return types.TargetStats{
+		TotalMessages:  atomic.LoadInt64(&sys.totalMessages),
+		FailedMessages: atomic.LoadInt64(&sys.failedMessages),
+		QueueLength:    0,
+	}
+}
+
 // Content returns the console stdout log
 func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
 	sys.RLock()
@@ -171,6 +188,7 @@ func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
 	case string:
 		lg = log.Info{ConsoleMsg: e, NodeName: sys.nodeName}
 	}
+	atomic.AddInt64(&sys.totalMessages, 1)
 
 	sys.pubsub.Publish(lg)
 	sys.Lock()
@@ -178,6 +196,9 @@ func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
 	sys.logBuf.Value = lg
 	sys.logBuf = sys.logBuf.Next()
 	sys.Unlock()
-
-	return sys.console.Send(entry, string(logger.All))
+	err := sys.console.Send(entry, string(logger.All))
+	if err != nil {
+		atomic.AddInt64(&sys.failedMessages, 1)
+	}
+	return err
 }

cmd/data-scanner-metric.go

@@ -47,6 +47,8 @@ const (
 	scannerMetricILM
 	scannerMetricCheckReplication
 	scannerMetricYield
+	scannerMetricCleanAbandoned
+	scannerMetricApplyNonCurrent
 
 	// START Trace metrics:
 	scannerMetricStartTrace
@@ -119,13 +121,6 @@ func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
 	}
 }
 
-func (p *scannerMetrics) incNoTime(s scannerMetric) {
-	atomic.AddUint64(&p.operations[s], 1)
-	if s < scannerMetricLastRealtime {
-		p.latency[s].add(0)
-	}
-}
-
 // timeILM times an ILM action.
 // lifecycle.NoneAction is ignored.
 // Use for s < scannerMetricLastRealtime

cmd/data-scanner.go

@@ -64,11 +64,6 @@ const (
 var (
 	globalHealConfig heal.Config
 
-	dataScannerLeaderLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
-		timeout:       30 * time.Second,
-		minimum:       10 * time.Second,
-		retryInterval: time.Second,
-	})
 	// Sleeper values are updated when config is loaded.
 	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
 	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
@@ -118,6 +113,10 @@ type backgroundHealInfo struct {
 }
 
 func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundHealInfo {
+	if globalIsErasureSD {
+		return backgroundHealInfo{}
+	}
+
 	// Get last healing information
 	buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
 	if err != nil {
@@ -127,15 +126,17 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
 		return backgroundHealInfo{}
 	}
 	var info backgroundHealInfo
-	err = json.Unmarshal(buf, &info)
-	if err != nil {
+	if err = json.Unmarshal(buf, &info); err != nil {
 		logger.LogIf(ctx, err)
-		return backgroundHealInfo{}
 	}
 	return info
 }
 
 func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgroundHealInfo) {
+	if globalIsErasureSD {
+		return
+	}
+
 	b, err := json.Marshal(info)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -151,19 +152,9 @@ func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgr
 // runDataScanner will start a data scanner.
 // The function will block until the context is canceled.
 // There should only ever be one scanner running per cluster.
-func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
-	// Make sure only 1 scanner is running on the cluster.
-	locker := objAPI.NewNSLock(minioMetaBucket, "scanner/runDataScanner.lock")
-	lkctx, err := locker.GetLock(pctx, dataScannerLeaderLockTimeout)
-	if err != nil {
-		if intDataUpdateTracker.debug {
-			logger.LogIf(pctx, err)
-		}
-		return
-	}
-	ctx := lkctx.Context()
-	defer lkctx.Cancel()
-	// No unlock for "leader" lock.
+func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
+	ctx, cancel := globalLeaderLock.GetLock(ctx)
+	defer cancel()
 
 	// Load current bloom cycle
 	var cycleInfo currentScannerCycle
@@ -175,7 +166,7 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 	} else if len(buf) > 8 {
 		cycleInfo.next = binary.LittleEndian.Uint64(buf[:8])
 		buf = buf[8:]
-		_, err = cycleInfo.UnmarshalMsg(buf)
+		_, err := cycleInfo.UnmarshalMsg(buf)
 		logger.LogIf(ctx, err)
 	}
 
@@ -429,7 +420,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		filter := f.withFilter
 		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
 			if f.dataUsageScannerDebug {
 				console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
@@ -938,7 +929,10 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
 		ScanMode: scanMode,
 	}
 	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
-	if err != nil && !errors.Is(err, NotImplemented{}) {
+	if err != nil {
+		if errors.Is(err, NotImplemented{}) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			err = nil
+		}
 		logger.LogIf(ctx, err)
 		return 0
 	}
@@ -960,21 +954,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
-	action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
+	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
 	if i.debug {
 		if versionID != "" {
-			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
 		} else {
-			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
+			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
 		}
 	}
-	defer globalScannerMetrics.timeILM(action)
+	defer globalScannerMetrics.timeILM(lcEvt.Action)
 
-	switch action {
+	switch lcEvt.Action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		return applyLifecycleAction(action, oi), 0
+		return applyLifecycleAction(lcEvt.Action, oi, ""), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		return applyLifecycleAction(action, oi), size
+		return applyLifecycleAction(lcEvt.Action, oi, lcEvt.StorageClass), size
 	default:
 		// No action.
 		return false, size
@@ -1021,6 +1015,8 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 	if i.lifeCycle == nil {
 		return fivs, nil
 	}
+	done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
+	defer done()
 
 	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
 	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
@@ -1077,6 +1073,17 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
 func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+	if i.heal.enabled {
+		if healDeleteDangling {
+			done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
+			err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
+			done()
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
+			}
+		}
+	}
+
 	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 }
 
@@ -1106,42 +1113,42 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	return size
 }
 
-func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo, debug bool) (action lifecycle.Action) {
-	action = lc.ComputeAction(obj.ToLifecycleOpts())
-	if debug {
-		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
+	event := lc.Eval(obj.ToLifecycleOpts())
+	if serverDebugLog {
+		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
 	}
 
-	if action == lifecycle.NoneAction {
-		return action
+	if event.Action == lifecycle.NoneAction {
+		return event
 	}
 
-	switch action {
+	switch event.Action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
 		if obj.VersionID == "" {
-			return lifecycle.NoneAction
+			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
 		if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
-			if debug {
+			if serverDebugLog {
 				if obj.VersionID != "" {
 					console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
 				} else {
 					console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 				}
 			}
-			return lifecycle.NoneAction
+			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
 	}
 
-	return action
+	return event
 }
 
-func applyTransitionRule(obj ObjectInfo) bool {
+func applyTransitionRule(obj ObjectInfo, storageClass string) bool {
 	if obj.DeleteMarker {
 		return false
 	}
-	globalTransitionState.queueTransitionTask(obj)
+	globalTransitionState.queueTransitionTask(obj, storageClass)
 	return true
 }
 
@@ -1210,14 +1217,14 @@ func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
 }
 
 // Perform actions (removal or transitioning of objects), return true the action is successfully performed
-func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
+func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo, storageClass string) (success bool) {
 	switch action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
 		success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
 	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 		success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		success = applyTransitionRule(obj)
+		success = applyTransitionRule(obj, storageClass)
 	}
 	return
 }
@@ -1438,9 +1445,11 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
 	case ILMTransition:
 		apiName = "ILMTransition"
 	}
-	auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
+	auditLogInternal(ctx, AuditLogOptions{
 		Event:     event,
 		APIName:   apiName,
+		Bucket:    oi.Bucket,
+		Object:    oi.Name,
 		VersionID: oi.VersionID,
 	})
 }

cmd/data-update-tracker_test.go

@@ -62,6 +62,15 @@ func (t *testingLogger) Type() types.TargetType {
 	return types.TargetHTTP
 }
 
+func (t *testingLogger) IsOnline() bool {
+	return true
+}
+
+// Stats returns the target statistics.
+func (t *testingLogger) Stats() types.TargetStats {
+	return types.TargetStats{}
+}
+
 func (t *testingLogger) Send(entry interface{}) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()

cmd/data-usage-cache.go

@@ -392,13 +392,6 @@ func (e *dataUsageEntry) addChild(hash dataUsageHash) {
 	e.Children[hash.Key()] = struct{}{}
 }
 
-// removeChild will remove a child based on its hash.
-func (e *dataUsageEntry) removeChild(hash dataUsageHash) {
-	if len(e.Children) > 0 {
-		delete(e.Children, hash.Key())
-	}
-}
-
 // Create a clone of the entry.
 func (e dataUsageEntry) clone() dataUsageEntry {
 	// We operate on a copy from the receiver.
@@ -488,43 +481,6 @@ func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {
 	}
 }
 
-// keepBuckets will keep only the buckets specified specified by delete all others.
-func (d *dataUsageCache) keepBuckets(b []BucketInfo) {
-	lu := make(map[dataUsageHash]struct{})
-	for _, v := range b {
-		lu[hashPath(v.Name)] = struct{}{}
-	}
-	d.keepRootChildren(lu)
-}
-
-// keepRootChildren will keep the root children specified by delete all others.
-func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
-	root := d.root()
-	if root == nil {
-		return
-	}
-	rh := d.rootHash()
-	for k := range d.Cache {
-		h := dataUsageHash(k)
-		if h == rh {
-			continue
-		}
-		if _, ok := list[h]; !ok {
-			delete(d.Cache, k)
-			d.deleteRecursive(h)
-			root.removeChild(h)
-		}
-	}
-	// Clean up abandoned children.
-	for k := range root.Children {
-		h := dataUsageHash(k)
-		if _, ok := list[h]; !ok {
-			delete(root.Children, k)
-		}
-	}
-	d.Cache[rh.Key()] = *root
-}
-
 // dui converts the flattened version of the path to madmin.DataUsageInfo.
 // As a side effect d will be flattened, use a clone if this is not ok.
 func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {

cmd/disk-cache-backend.go

@@ -953,7 +953,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 		objSize, _ = sio.EncryptedSize(uint64(size))
 
 	}
-	cacheFile := MustGetUUID()
+	cacheFile := mustGetUUID()
 	n, _, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		// take the cache drive offline
@@ -1003,6 +1003,7 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 	if err != nil {
 		return err
 	}
+	defer rc.Close()
 	bufp := c.pool.Get().(*[]byte)
 	defer c.pool.Put(bufp)
 

cmd/dynamic-timeouts_test.go

@@ -130,7 +130,6 @@ func TestDynamicTimeoutConcurrent(t *testing.T) {
 		go func() {
 			defer wg.Done()
 			for i := 0; i < 100; i++ {
-				timeout.LogFailure()
 				for j := 0; j < 100; j++ {
 					timeout.LogSuccess(time.Duration(float64(time.Second) * rng.Float64()))
 				}

cmd/encryption-v1.go

@@ -1043,18 +1043,6 @@ func DecryptObjectInfo(info *ObjectInfo, r *http.Request) (encrypted bool, err e
 	return encrypted, nil
 }
 
-// The customer key in the header is used by the gateway for encryption in the case of
-// s3 gateway double encryption. A new client key is derived from the customer provided
-// key to be sent to the s3 backend for encryption at the backend.
-func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
-	var key [32]byte
-	mac := hmac.New(sha256.New, clientKey[:])
-	mac.Write([]byte(crypto.SSEC.String()))
-	mac.Write([]byte(path.Join(bucket, object)))
-	mac.Sum(key[:0])
-	return key
-}
-
 type (
 	objectMetaEncryptFn func(baseKey string, data []byte) []byte
 	objectMetaDecryptFn func(baseKey string, data []byte) ([]byte, error)

cmd/endpoint-ellipses.go

@@ -343,7 +343,13 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		return nil, -1, errInvalidArgument
 	}
 
-	if !ellipses.HasEllipses(args...) {
+	ok := true
+	for _, arg := range args {
+		ok = ok && !ellipses.HasEllipses(arg)
+	}
+
+	// None of the args have ellipses use the old style.
+	if ok {
 		setArgs, err := GetAllSets(args...)
 		if err != nil {
 			return nil, -1, err
@@ -365,6 +371,10 @@ func createServerEndpoints(serverAddr string, args ...string) (
 
 	var foundPrevLocal bool
 	for _, arg := range args {
+		if !ellipses.HasEllipses(arg) && len(args) > 1 {
+			// TODO: support SNSD deployments to be decommissioned in future
+			return nil, -1, fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args)
+		}
 		setArgs, err := GetAllSets(arg)
 		if err != nil {
 			return nil, -1, err

cmd/endpoint.go

@@ -592,14 +592,14 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 			return endpoints, setupType, err
 		}
 		if endpoint.Type() != PathEndpointType {
-			return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
+			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
 		}
 		endpoints = append(endpoints, endpoint)
 		setupType = ErasureSDSetupType
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error())
+			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg(err.Error())
 		}
 
 		return endpoints, setupType, nil

cmd/erasure-healing-common.go

@@ -179,53 +179,6 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error)
 	return onlineDisks, modTime
 }
 
-// Returns the latest updated FileInfo files and error in case of failure.
-func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultParityCount int, errs []error) (FileInfo, error) {
-	// There should be atleast half correct entries, if not return failure
-	expectedRQuorum := len(partsMetadata) / 2
-	if defaultParityCount == 0 {
-		// if parity count is '0', we expected all entries to be present.
-		expectedRQuorum = len(partsMetadata)
-	}
-
-	reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, expectedRQuorum)
-	if reducedErr != nil {
-		return FileInfo{}, reducedErr
-	}
-
-	// List all the file commit ids from parts metadata.
-	modTimes := listObjectModtimes(partsMetadata, errs)
-
-	// Count all latest updated FileInfo values
-	var count int
-	var latestFileInfo FileInfo
-
-	// Reduce list of UUIDs to a single common value - i.e. the last updated Time
-	modTime := commonTime(modTimes)
-
-	if modTime.IsZero() || modTime.Equal(timeSentinel) {
-		return FileInfo{}, errErasureReadQuorum
-	}
-
-	// Interate through all the modTimes and count the FileInfo(s) with latest time.
-	for index, t := range modTimes {
-		if partsMetadata[index].IsValid() && t.Equal(modTime) {
-			latestFileInfo = partsMetadata[index]
-			count++
-		}
-	}
-
-	if !latestFileInfo.IsValid() {
-		return FileInfo{}, errErasureReadQuorum
-	}
-
-	if count < latestFileInfo.Erasure.DataBlocks {
-		return FileInfo{}, errErasureReadQuorum
-	}
-
-	return latestFileInfo, nil
-}
-
 // disksWithAllParts - This function needs to be called with
 // []StorageAPI returned by listOnlineDisks. Returns,
 //

cmd/erasure-healing-common_test.go

@@ -30,6 +30,53 @@ import (
 	"github.com/minio/madmin-go"
 )
 
+// Returns the latest updated FileInfo files and error in case of failure.
+func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultParityCount int, errs []error) (FileInfo, error) {
+	// There should be atleast half correct entries, if not return failure
+	expectedRQuorum := len(partsMetadata) / 2
+	if defaultParityCount == 0 {
+		// if parity count is '0', we expected all entries to be present.
+		expectedRQuorum = len(partsMetadata)
+	}
+
+	reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, expectedRQuorum)
+	if reducedErr != nil {
+		return FileInfo{}, reducedErr
+	}
+
+	// List all the file commit ids from parts metadata.
+	modTimes := listObjectModtimes(partsMetadata, errs)
+
+	// Count all latest updated FileInfo values
+	var count int
+	var latestFileInfo FileInfo
+
+	// Reduce list of UUIDs to a single common value - i.e. the last updated Time
+	modTime := commonTime(modTimes)
+
+	if modTime.IsZero() || modTime.Equal(timeSentinel) {
+		return FileInfo{}, errErasureReadQuorum
+	}
+
+	// Interate through all the modTimes and count the FileInfo(s) with latest time.
+	for index, t := range modTimes {
+		if partsMetadata[index].IsValid() && t.Equal(modTime) {
+			latestFileInfo = partsMetadata[index]
+			count++
+		}
+	}
+
+	if !latestFileInfo.IsValid() {
+		return FileInfo{}, errErasureReadQuorum
+	}
+
+	if count < latestFileInfo.Erasure.DataBlocks {
+		return FileInfo{}, errErasureReadQuorum
+	}
+
+	return latestFileInfo, nil
+}
+
 // validates functionality provided to find most common
 // time occurrence from a list of time.
 func TestCommonTime(t *testing.T) {

cmd/erasure-healing.go

@@ -40,6 +40,7 @@ type healingMetric uint8
 const (
 	healingMetricBucket healingMetric = iota
 	healingMetricObject
+	healingMetricCheckAbandonedParts
 )
 
 // AcceptableDelta returns 'true' if the fi.DiskMTime is under
@@ -93,7 +94,7 @@ func (er erasureObjects) healBucket(ctx context.Context, storageDisks []StorageA
 	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
 		startTime := time.Now()
 		defer func() {
-			healTrace(healingMetricBucket, startTime, bucket, "", "", opts, err, &res)
+			healTrace(healingMetricBucket, startTime, bucket, "", "", &opts, err, &res)
 		}()
 	}
 
@@ -294,7 +295,7 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 }
 
 // Heals an object by re-writing corrupt/missing erasure blocks.
-func (er erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
+func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
 	scanMode := opts.ScanMode
 
@@ -304,7 +305,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
 		startTime := time.Now()
 		defer func() {
-			healTrace(healingMetricObject, startTime, bucket, object, versionID, opts, err, &result)
+			healTrace(healingMetricObject, startTime, bucket, object, versionID, &opts, err, &result)
 		}()
 	}
 	// Initialize heal result object
@@ -339,7 +340,24 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
-		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, nil, opts)
+		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, nil, ObjectOptions{
+			VersionID: versionID,
+		})
+		errs = make([]error, len(errs))
+		for i := range errs {
+			errs[i] = err
+		}
+		if err == nil {
+			// Dangling object successfully purged, size is '0'
+			m.Size = 0
+		}
+		// Generate file/version not found with default heal result
+		err = errFileNotFound
+		if versionID != "" {
+			err = errFileVersionNotFound
+		}
+		return er.defaultHealResult(m, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), err
 	}
 
 	result.ParityBlocks = result.DiskCount - readQuorum
@@ -392,14 +410,12 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
 	outDatedDisks := make([]StorageAPI, len(storageDisks))
-	numAvailableDisks := 0
 	disksToHealCount := 0
 	for i, v := range availableDisks {
 		driveState := ""
 		switch {
 		case v != nil:
 			driveState = madmin.DriveStateOk
-			numAvailableDisks++
 			// If data is sane on any one disk, we can
 			// extract the correct object size.
 			result.ObjectSize = partsMetadata[i].Size
@@ -451,12 +467,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			bucket, object, versionID), err
 	}
 
-	// If less than read quorum number of disks have all the parts
-	// of the data, we can't reconstruct the erasure-coded data.
-	if numAvailableDisks < readQuorum {
-		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, dataErrs, opts)
-	}
-
 	if disksToHealCount == 0 {
 		// Nothing to heal!
 		return result, nil
@@ -613,7 +623,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	}
 
-	defer er.renameAll(context.Background(), minioMetaTmpBucket, tmpID)
+	defer er.deleteAll(context.Background(), minioMetaTmpBucket, tmpID)
 
 	// Rename from tmp location to the actual location.
 	for i, disk := range outDatedDisks {
@@ -664,9 +674,45 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	return result, nil
 }
 
+// checkAbandonedParts will check if an object has abandoned parts,
+// meaning data-dirs or inlined data that are no longer referenced by the xl.meta
+// Errors are generally ignored by this function.
+func (er *erasureObjects) checkAbandonedParts(ctx context.Context, bucket string, object string, opts madmin.HealOpts) (err error) {
+	if !opts.Remove || opts.DryRun {
+		return nil
+	}
+	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
+		startTime := time.Now()
+		defer func() {
+			healTrace(healingMetricCheckAbandonedParts, startTime, bucket, object, "", nil, err, nil)
+		}()
+	}
+	if !opts.NoLock {
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx.Cancel)
+	}
+	var wg sync.WaitGroup
+	for _, disk := range er.getDisks() {
+		if disk != nil {
+			wg.Add(1)
+			go func(disk StorageAPI) {
+				defer wg.Done()
+				_ = disk.CleanAbandonedData(ctx, bucket, object)
+			}(disk)
+		}
+	}
+	wg.Wait()
+	return nil
+}
+
 // healObjectDir - heals object directory specifically, this special call
 // is needed since we do not have a special backend format for directories.
-func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object string, dryRun bool, remove bool) (hr madmin.HealResultItem, err error) {
+func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object string, dryRun bool, remove bool) (hr madmin.HealResultItem, err error) {
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
@@ -757,7 +803,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 
 // Populates default heal result item entries with possible values when we are returning prematurely.
 // This is to ensure that in any circumstance we are not returning empty arrays with wrong values.
-func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []Endpoint, errs []error, bucket, object, versionID string) madmin.HealResultItem {
+func (er *erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []Endpoint, errs []error, bucket, object, versionID string) madmin.HealResultItem {
 	// Initialize heal result object
 	result := madmin.HealResultItem{
 		Type:       madmin.HealItemObject,
@@ -879,32 +925,6 @@ func isObjectDirDangling(errs []error) (ok bool) {
 	return found < notFound && found > 0
 }
 
-func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object, versionID string,
-	metaArr []FileInfo, errs []error, dataErrs []error, opts madmin.HealOpts) (madmin.HealResultItem, error,
-) {
-	storageDisks := er.getDisks()
-	storageEndpoints := er.getEndpoints()
-
-	m, err := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, dataErrs, ObjectOptions{
-		VersionID: versionID,
-	})
-	errs = make([]error, len(errs))
-	for i := range errs {
-		errs[i] = err
-	}
-	if err == nil {
-		// Dangling object successfully purged, size is '0'
-		m.Size = 0
-	}
-	// Generate file/version not found with default heal result
-	err = errFileNotFound
-	if versionID != "" {
-		err = errFileVersionNotFound
-	}
-	return er.defaultHealResult(m, storageDisks, storageEndpoints,
-		errs, bucket, object, versionID), err
-}
-
 // Object is considered dangling/corrupted if any only
 // if total disks - a combination of corrupted and missing
 // files is lesser than number of data blocks.
@@ -1022,16 +1042,18 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 }
 
 // healTrace sends healing results to trace output.
-func healTrace(funcName healingMetric, startTime time.Time, bucket, object, versionID string, opts madmin.HealOpts, err error, result *madmin.HealResultItem) {
+func healTrace(funcName healingMetric, startTime time.Time, bucket, object, versionID string, opts *madmin.HealOpts, err error, result *madmin.HealResultItem) {
 	tr := madmin.TraceInfo{
 		TraceType: madmin.TraceHealing,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  "heal." + funcName.String(),
 		Duration:  time.Since(startTime),
-		Message:   fmt.Sprintf("dry:%v, rm:%v, recreate:%v mode:%v", opts.DryRun, opts.Remove, opts.Recreate, opts.ScanMode),
 		Path:      pathJoin(bucket, decodeDirObject(object)),
 	}
+	if opts != nil {
+		tr.Message = fmt.Sprintf("dry:%v, rm:%v, recreate:%v mode:%v", opts.DryRun, opts.Remove, opts.Recreate, opts.ScanMode)
+	}
 	if versionID != "" && versionID != "null" {
 		tr.Path += " v=" + versionID
 	}

cmd/erasure-healing_test.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	uuid2 "github.com/google/uuid"
 	"github.com/minio/madmin-go"
 )
 
@@ -276,6 +277,12 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Checking abandoned parts should do nothing
+	err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	_, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
 		t.Fatal(err)
@@ -320,6 +327,214 @@ func TestHealing(t *testing.T) {
 		t.Fatal("HealObject failed")
 	}
 
+	uuid, _ := uuid2.NewRandom()
+	for _, drive := range fsDirs {
+		dir := path.Join(drive, bucket, object, uuid.String())
+		err = os.MkdirAll(dir, os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = os.WriteFile(pathJoin(dir, "part.1"), []byte("some data"), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// This should remove all the unreferenced parts.
+	err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, drive := range fsDirs {
+		dir := path.Join(drive, bucket, object, uuid.String())
+		_, err := os.ReadFile(pathJoin(dir, "part.1"))
+		if err == nil {
+			t.Fatal("expected data dit to be cleaned up")
+		}
+	}
+
+	// Remove the bucket - to simulate the case where bucket was
+	// created when the disk was down.
+	err = os.RemoveAll(path.Join(fsDirs[0], bucket))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// This would create the bucket.
+	_, err = er.HealBucket(ctx, bucket, madmin.HealOpts{
+		DryRun: false,
+		Remove: false,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Stat the bucket to make sure that it was created.
+	_, err = er.getDisks()[0].StatVol(context.Background(), bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests both object and bucket healing.
+func TestHealingVersioned(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	obj, fsDirs, err := prepareErasure16(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer obj.Shutdown(context.Background())
+
+	// initialize the server and obtain the credentials and root.
+	// credentials are necessary to sign the HTTP request.
+	if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil {
+		t.Fatalf("Unable to initialize server config. %s", err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	z := obj.(*erasureServerPools)
+	er := z.serverPools[0].sets[0]
+
+	// Create "bucket"
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{VersioningEnabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := "bucket"
+	object := "object"
+
+	data := make([]byte, 1*humanize.MiByte)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	oi1, err := obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), length, "", ""), ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// 2nd version.
+	_, _ = rand.Read(data)
+	oi2, err := obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), length, "", ""), ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	disk := er.getDisks()[0]
+	fileInfoPreHeal1, err := disk.ReadVersion(context.Background(), bucket, object, oi1.VersionID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fileInfoPreHeal2, err := disk.ReadVersion(context.Background(), bucket, object, oi2.VersionID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove the object - to simulate the case where the disk was down when the object
+	// was created.
+	err = removeAll(pathJoin(disk.String(), bucket, object))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Checking abandoned parts should do nothing
+	err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fileInfoPostHeal1, err := disk.ReadVersion(context.Background(), bucket, object, oi1.VersionID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fileInfoPostHeal2, err := disk.ReadVersion(context.Background(), bucket, object, oi2.VersionID, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// After heal the meta file should be as expected.
+	if !fileInfoPreHeal1.Equals(fileInfoPostHeal1) {
+		t.Fatal("HealObject failed")
+	}
+	if !fileInfoPreHeal1.Equals(fileInfoPostHeal2) {
+		t.Fatal("HealObject failed")
+	}
+
+	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "xl.meta"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Write xl.meta with different modtime to simulate the case where a disk had
+	// gone down when an object was replaced by a new object.
+	fileInfoOutDated := fileInfoPreHeal1
+	fileInfoOutDated.ModTime = time.Now()
+	err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fileInfoPostHeal1, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// After heal the meta file should be as expected.
+	if !fileInfoPreHeal1.Equals(fileInfoPostHeal1) {
+		t.Fatal("HealObject failed")
+	}
+
+	fileInfoPostHeal2, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// After heal the meta file should be as expected.
+	if !fileInfoPreHeal2.Equals(fileInfoPostHeal2) {
+		t.Fatal("HealObject failed")
+	}
+
+	uuid, _ := uuid2.NewRandom()
+	for _, drive := range fsDirs {
+		dir := path.Join(drive, bucket, object, uuid.String())
+		err = os.MkdirAll(dir, os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = os.WriteFile(pathJoin(dir, "part.1"), []byte("some data"), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// This should remove all the unreferenced parts.
+	err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, drive := range fsDirs {
+		dir := path.Join(drive, bucket, object, uuid.String())
+		_, err := os.ReadFile(pathJoin(dir, "part.1"))
+		if err == nil {
+			t.Fatal("expected data dit to be cleaned up")
+		}
+	}
+
 	// Remove the bucket - to simulate the case where bucket was
 	// created when the disk was down.
 	err = os.RemoveAll(path.Join(fsDirs[0], bucket))

cmd/erasure-metadata-utils.go

@@ -22,6 +22,8 @@ import (
 	"errors"
 	"fmt"
 	"hash/crc32"
+	"io"
+	"strings"
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -165,18 +167,23 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 		}, index)
 	}
 
+	ignoredErrs := []error{
+		errFileNotFound,
+		errVolumeNotFound,
+		errFileVersionNotFound,
+		errDiskNotFound,
+		errUnformattedDisk,
+	}
+	if strings.HasPrefix(bucket, minioMetaBucket) {
+		// listing object might be truncated, ignore such errors from logging.
+		ignoredErrs = append(ignoredErrs, io.ErrUnexpectedEOF)
+	}
 	errs := g.Wait()
 	for index, err := range errs {
 		if err == nil {
 			continue
 		}
-		if !IsErr(err, []error{
-			errFileNotFound,
-			errVolumeNotFound,
-			errFileVersionNotFound,
-			errDiskNotFound,
-			errUnformattedDisk,
-		}...) {
+		if !IsErr(err, ignoredErrs...) {
 			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], bucket, object, err),
 				disks[index].String())

cmd/erasure-metadata.go

@@ -168,7 +168,9 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 	objInfo.Parts = fi.Parts
 
 	// Update storage class
-	if sc, ok := fi.Metadata[xhttp.AmzStorageClass]; ok {
+	if fi.TransitionTier != "" {
+		objInfo.StorageClass = fi.TransitionTier
+	} else if sc, ok := fi.Metadata[xhttp.AmzStorageClass]; ok {
 		objInfo.StorageClass = sc
 	} else {
 		objInfo.StorageClass = globalMinioDefaultStorageClass
@@ -394,21 +396,73 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 	return evalDisks(disks, mErrs), err
 }
 
+func commonParity(parities []int) int {
+	occMap := make(map[int]int)
+	for _, p := range parities {
+		occMap[p]++
+	}
+
+	var maxOcc, commonParity int
+
+	for parity, occ := range occMap {
+		if parity == -1 {
+			// Ignore non defined parity
+			continue
+		}
+		if occ >= maxOcc {
+			maxOcc = occ
+			commonParity = parity
+		}
+	}
+
+	if maxOcc == 0 {
+		// Did not found anything useful
+		return -1
+	}
+	return commonParity
+}
+
+func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int) {
+	parities = make([]int, len(partsMetadata))
+	for index, metadata := range partsMetadata {
+		if errs[index] != nil {
+			parities[index] = -1
+			continue
+		}
+		parities[index] = metadata.Erasure.ParityBlocks
+	}
+	return
+}
+
 // Returns per object readQuorum and writeQuorum
 // readQuorum is the min required disks to read data.
 // writeQuorum is the min required disks to write data.
 func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
-	// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
-	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, defaultParityCount, errs)
-	if err != nil {
-		return 0, 0, err
+	// There should be atleast half correct entries, if not return failure
+	expectedRQuorum := len(partsMetaData) / 2
+	if defaultParityCount == 0 {
+		// if parity count is '0', we expected all entries to be present.
+		expectedRQuorum = len(partsMetaData)
 	}
 
-	if latestFileInfo.Deleted {
-		// special case when parity is '0'
-		if defaultParityCount == 0 {
-			return len(partsMetaData), len(partsMetaData), nil
-		}
+	reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, expectedRQuorum)
+	if reducedErr != nil {
+		return -1, -1, reducedErr
+	}
+
+	// special case when parity is '0'
+	if defaultParityCount == 0 {
+		return len(partsMetaData), len(partsMetaData), nil
+	}
+
+	parities := listObjectParities(partsMetaData, errs)
+	parityBlocks := commonParity(parities)
+
+	if parityBlocks < 0 {
+		return -1, -1, errErasureReadQuorum
+	}
+
+	if parityBlocks == 0 {
 		// For delete markers do not use 'defaultParityCount' as it is not expected to be the case.
 		// Use maximum allowed read quorum instead, writeQuorum+1 is returned for compatibility sake
 		// but there are no callers that shall be using this.
@@ -416,23 +470,7 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 		return readQuorum, readQuorum + 1, nil
 	}
 
-	parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass])
-	if parityBlocks < 0 {
-		parityBlocks = defaultParityCount
-	}
-
-	// For erasure code upgraded objects choose the parity
-	// blocks saved internally, instead of 'defaultParityCount'
-	if _, ok := latestFileInfo.Metadata[minIOErasureUpgraded]; ok {
-		if latestFileInfo.Erasure.ParityBlocks != 0 {
-			parityBlocks = latestFileInfo.Erasure.ParityBlocks
-		}
-	}
-
-	dataBlocks := latestFileInfo.Erasure.DataBlocks
-	if dataBlocks == 0 {
-		dataBlocks = len(partsMetaData) - parityBlocks
-	}
+	dataBlocks := len(partsMetaData) - parityBlocks
 
 	writeQuorum := dataBlocks
 	if dataBlocks == parityBlocks {

cmd/erasure-multipart.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -41,7 +42,15 @@ import (
 )
 
 func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
-	return pathJoin(er.getMultipartSHADir(bucket, object), uploadID)
+	uploadUUID := uploadID
+	uploadBytes, err := base64.RawURLEncoding.DecodeString(uploadID)
+	if err == nil {
+		slc := strings.SplitN(string(uploadBytes), ".", 2)
+		if len(slc) == 2 {
+			uploadUUID = slc[1]
+		}
+	}
+	return pathJoin(er.getMultipartSHADir(bucket, object), uploadUUID)
 }
 
 func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
@@ -166,21 +175,6 @@ func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Du
 	wg.Wait()
 }
 
-func (er erasureObjects) renameAll(ctx context.Context, bucket, prefix string) {
-	var wg sync.WaitGroup
-	for _, disk := range er.getDisks() {
-		if disk == nil {
-			continue
-		}
-		wg.Add(1)
-		go func(disk StorageAPI) {
-			defer wg.Done()
-			disk.RenameFile(ctx, bucket, prefix, minioMetaTmpDeletedBucket, mustGetUUID())
-		}(disk)
-	}
-	wg.Wait()
-}
-
 func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 	var wg sync.WaitGroup
 	for _, disk := range er.getDisks() {
@@ -209,11 +203,12 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
 			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "", false)
 			if err != nil {
+				er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
 				return nil
 			}
-			wait := er.deletedCleanupSleeper.Timer(ctx)
+			wait := deletedCleanupSleeper.Timer(ctx)
 			if now.Sub(fi.ModTime) > expiry {
-				er.renameAll(ctx, minioMetaMultipartBucket, uploadIDPath)
+				er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
 			}
 			wait()
 			return nil
@@ -228,7 +223,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		if err != nil {
 			return nil
 		}
-		wait := er.deletedCleanupSleeper.Timer(ctx)
+		wait := deletedCleanupSleeper.Timer(ctx)
 		if now.Sub(vi.Created) > expiry {
 			er.deleteAll(ctx, minioMetaTmpBucket, tmpDir)
 		}
@@ -355,7 +350,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		rctx := lkctx.Context()
 		obj, err := er.getObjectInfo(rctx, bucket, object, opts)
 		lk.RUnlock(lkctx.Cancel)
-		if err != nil {
+		if err != nil && !isErrVersionNotFound(err) {
 			return nil, err
 		}
 		if opts.CheckPrecondFn(obj) {
@@ -364,7 +359,9 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	userDefined := cloneMSS(opts.UserDefined)
-
+	if opts.PreserveETag != "" {
+		userDefined["etag"] = opts.PreserveETag
+	}
 	onlineDisks := er.getDisks()
 	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
 	if parityDrives < 0 {
@@ -436,9 +433,9 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		partsMetadata[index].ModTime = modTime
 		partsMetadata[index].Metadata = userDefined
 	}
-
-	uploadID := mustGetUUID()
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
+	uploadUUID := mustGetUUID()
+	uploadID := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID, uploadUUID)))
+	uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID)
 
 	// Write updated `xl.meta` to all disks.
 	if _, err := writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
@@ -572,16 +569,6 @@ func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)
 
-	// Write lock for this part ID.
-	// Held throughout the operation.
-	partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID)))
-	plkctx, err := partIDLock.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return PartInfo{}, err
-	}
-	pctx := plkctx.Context()
-	defer partIDLock.Unlock(plkctx.Cancel)
-
 	// Read lock for upload id.
 	// Only held while reading the upload metadata.
 	uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
@@ -592,6 +579,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	rctx := rlkctx.Context()
 	defer uploadIDRLock.RUnlock(rlkctx.Cancel)
 
+	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
+	// Validates if upload ID exists.
+	fi, _, err := er.checkUploadIDExists(rctx, bucket, object, uploadID, true)
+	if err != nil {
+		return pi, toObjectErr(err, bucket, object, uploadID)
+	}
+
 	data := r.Reader
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
@@ -599,13 +593,17 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return pi, toObjectErr(errInvalidArgument)
 	}
 
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-
-	// Validates if upload ID exists.
-	fi, _, err := er.checkUploadIDExists(rctx, bucket, object, uploadID, true)
+	// Write lock for this part ID, only hold it if we are planning to read from the
+	// streamto avoid any concurrent updates.
+	//
+	// Must be held throughout this call.
+	partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID)))
+	plkctx, err := partIDLock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
-		return pi, toObjectErr(err, bucket, object, uploadID)
+		return PartInfo{}, err
 	}
+	pctx := plkctx.Context()
+	defer partIDLock.Unlock(plkctx.Cancel)
 
 	onlineDisks := er.getDisks()
 	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
@@ -632,7 +630,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.renameAll(context.Background(), minioMetaTmpBucket, tmpPart)
+			er.deleteAll(context.Background(), minioMetaTmpBucket, tmpPart)
 		}
 	}()
 
@@ -1103,7 +1101,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 					PartNumber: part.PartNumber,
 				}
 			}
-			checksumCombined = append(checksumCombined, cs.Raw()...)
+			checksumCombined = append(checksumCombined, cs.Raw...)
 		}
 
 		// All parts except the last part has to be at least 5MB.
@@ -1155,9 +1153,13 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Save successfully calculated md5sum.
-	fi.Metadata["etag"] = opts.UserDefined["etag"]
+	// for replica, newMultipartUpload would have already sent the replication ETag
 	if fi.Metadata["etag"] == "" {
-		fi.Metadata["etag"] = getCompleteMultipartMD5(parts)
+		if opts.UserDefined["etag"] != "" {
+			fi.Metadata["etag"] = opts.UserDefined["etag"]
+		} else { // fallback if not already calculated in handler.
+			fi.Metadata["etag"] = getCompleteMultipartMD5(parts)
+		}
 	}
 
 	// Save the consolidated actual size.
@@ -1171,6 +1173,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			partsMetadata[index].ModTime = fi.ModTime
 			partsMetadata[index].Metadata = fi.Metadata
 			partsMetadata[index].Parts = fi.Parts
+			partsMetadata[index].Checksum = fi.Checksum
 		}
 	}
 
@@ -1264,7 +1267,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 	}
 
 	// Cleanup all uploaded parts.
-	er.renameAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
+	er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
 
 	// Successfully purged.
 	return nil

cmd/erasure-object.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"net/http"
 	"path"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -439,10 +440,37 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
 
+func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]interface{}) {
+	if len(logger.AuditTargets()) == 0 {
+		return
+	}
+
+	opts := AuditLogOptions{
+		Event:     "DeleteDanglingObject",
+		Bucket:    bucket,
+		Object:    object,
+		VersionID: versionID,
+		Tags:      tags,
+	}
+
+	auditLogInternal(ctx, opts)
+}
+
 func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
+	_, file, line, cok := runtime.Caller(1)
 	var err error
 	m, ok := isObjectDangling(metaArr, errs, dataErrs)
 	if ok {
+		tags := make(map[string]interface{}, 4)
+		tags["set"] = er.setIndex
+		tags["pool"] = er.poolIndex
+		tags["parity"] = m.Erasure.ParityBlocks
+		if cok {
+			tags["caller"] = fmt.Sprintf("%s:%d", file, line)
+		}
+
+		defer auditDanglingObjectDeletion(ctx, bucket, object, m.VersionID, tags)
+
 		err = errFileNotFound
 		if opts.VersionID != "" {
 			err = errFileVersionNotFound
@@ -505,6 +533,7 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 
 	ignoredErrs := []error{
 		errFileNotFound,
+		errFileNameTooLong,
 		errVolumeNotFound,
 		errFileVersionNotFound,
 		errDiskNotFound,
@@ -914,7 +943,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	if opts.CheckPrecondFn != nil {
 		obj, err := er.getObjectInfo(ctx, bucket, object, opts)
-		if err != nil {
+		if err != nil && !isErrVersionNotFound(err) {
 			return objInfo, err
 		}
 		if opts.CheckPrecondFn(obj) {
@@ -1044,7 +1073,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.renameAll(context.Background(), minioMetaTmpBucket, tempObj)
+			er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
 		}
 	}()
 
@@ -1224,17 +1253,21 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 				return nil
 			}
 		}
+
 		fivs, err := entry.fileInfoVersions(bucket)
 		if err != nil {
-			_, err = er.HealObject(ctx, bucket, entry.name, "", madmin.HealOpts{NoLock: true})
+			healObject(bucket, entry.name, "", madmin.HealNormalScan)
 			return err
 		}
 
+		if len(fivs.Versions) > 2 {
+			return fmt.Errorf("bucket(%s)/object(%s) object with %d versions needs healing, allowing lazy healing via scanner instead here for versions greater than 2",
+				bucket, entry.name,
+				len(fivs.Versions))
+		}
+
 		for _, version := range fivs.Versions {
-			_, err = er.HealObject(ctx, bucket, version.Name, version.VersionID, madmin.HealOpts{NoLock: true})
-			if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-				return err
-			}
+			healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
 		}
 
 		return nil
@@ -1527,12 +1560,15 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	storageDisks := er.getDisks()
 
+	//  Determine whether to mark object deleted for replication
+	var markDelete bool
+
 	if opts.Expiration.Expire {
 		goi, _, err := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
 		if err == nil {
-			action := evalActionFromLifecycle(ctx, *lc, rcfg, goi, false)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
 			var isErr bool
-			switch action {
+			switch evt.Action {
 			case lifecycle.NoneAction:
 				isErr = true
 			case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
@@ -1551,20 +1587,21 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 					Object: object,
 				}
 			}
+			markDelete = goi.VersionID != ""
 		}
 	}
 
 	versionFound := !(opts.DeleteMarker && opts.VersionID != "")
 
-	// Determine whether to mark object deleted for replication
-	markDelete := !opts.DeleteMarker && opts.VersionID != ""
-
+	if !markDelete {
+		markDelete = !opts.DeleteMarker && opts.VersionID != ""
+	}
 	// Default deleteMarker to true if object is under versioning
 	// versioning suspended means we add `null` version as
 	// delete marker, if its not decided already.
 	deleteMarker := (opts.Versioned || opts.VersionSuspended) && opts.VersionID == "" || (opts.DeleteMarker && opts.VersionID != "")
 
-	if markDelete {
+	if markDelete && opts.VersionID != "" {
 		// case where replica version needs to be deleted on target cluster
 		if versionFound && opts.DeleteMarkerReplicationStatus() == replication.Replica {
 			markDelete = false

cmd/erasure-object_test.go

@@ -27,6 +27,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"testing"
 
@@ -1123,6 +1124,10 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 
 // Test reading an object with some outdated data in some disks
 func TestGetObjectWithOutdatedDisks(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip()
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 

cmd/erasure-server-pool-decom.go

@@ -317,7 +317,7 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 	}
 
 	replaceScheme := func(k string) string {
-		// This is needed as fallback when users are changeing
+		// This is needed as fallback when users are updating
 		// from http->https or https->http, we need to verify
 		// both because MinIO remembers the command-line in
 		// "exact" order - as long as this order is not disturbed
@@ -359,7 +359,7 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 			}
 		}
 		if !ok {
-			return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
+			update = true
 		}
 	}
 
@@ -374,9 +374,9 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 				}
 			}
 			if !ok {
-				return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
+				update = true
 			}
-			if pos != pi.position {
+			if ok && pos != pi.position {
 				return false, fmt.Errorf("pool order change detected for %s, expected position is (%s) but found (%s)", k, humanize.Ordinal(pi.position+1), humanize.Ordinal(pos+1))
 			}
 		}
@@ -392,6 +392,7 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 			}
 		}
 	}
+
 	return update, nil
 }
 
@@ -499,6 +500,15 @@ const (
 // Init() initializes pools and saves additional information about them
 // in 'pool.bin', this is eventually used for decommissioning the pool.
 func (z *erasureServerPools) Init(ctx context.Context) error {
+	// Load rebalance metadata if present
+	err := z.loadRebalanceMeta(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to load rebalance data: %w", err)
+	}
+
+	// Start rebalance routine
+	z.StartRebalance()
+
 	meta := poolMeta{}
 
 	if err := meta.load(ctx, z.serverPools[0], z.serverPools); err != nil {
@@ -529,21 +539,18 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 					r := rand.New(rand.NewSource(time.Now().UnixNano()))
 					for {
 						if err := z.Decommission(ctx, pool.ID); err != nil {
-							switch err {
-							// we already started decommission
-							case errDecommissionAlreadyRunning:
+							if errors.Is(err, errDecommissionAlreadyRunning) {
 								// A previous decommission running found restart it.
 								z.doDecommissionInRoutine(ctx, idx)
 								return
-							default:
-								if configRetriableErrors(err) {
-									logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w: retrying..", pool, err))
-									time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
-									continue
-								}
-								logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
-								return
 							}
+							if configRetriableErrors(err) {
+								logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w: retrying..", pool, err))
+								time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
+								continue
+							}
+							logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+							return
 						}
 						break
 					}
@@ -573,6 +580,19 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 	return nil
 }
 
+func (z *erasureServerPools) IsDecommissionRunning() bool {
+	z.poolMetaMutex.RLock()
+	defer z.poolMetaMutex.RUnlock()
+	meta := z.poolMeta
+	for _, pool := range meta.Pools {
+		if pool.Decommission != nil {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
 	objInfo := gr.ObjInfo
 
@@ -701,14 +721,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			}
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
-			action := evalActionFromLifecycle(ctx, *lc, lr, objInfo, false)
-			switch action {
+			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			switch evt.Action {
 			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+				globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
 				// Skip this entry.
 				return true
 			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
 				// Skip this entry.
 				return true
 			}
@@ -768,6 +788,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							DeleteMarker:       true, // make sure we create a delete marker
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 						})
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+						// object/version already deleted by the application, nothing to do here we move on.
+						continue
+					}
 					var failure bool
 					if err != nil {
 						logger.LogIf(ctx, err)
@@ -783,7 +807,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					continue
 				}
 
-				var failure bool
+				var failure, ignore bool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
 					gr, err := set.GetObjectNInfo(ctx,
@@ -796,9 +820,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							VersionID:    version.VersionID,
 							NoDecryption: true,
 						})
-					if isErrObjectNotFound(err) {
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
-						return
+						ignore = true
+						break
 					}
 					if err != nil {
 						failure = true
@@ -816,6 +841,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					failure = false
 					break
 				}
+				if ignore {
+					continue
+				}
 				z.poolMetaMutex.Lock()
 				z.poolMeta.CountItem(idx, version.Size, failure)
 				z.poolMetaMutex.Unlock()
@@ -1266,9 +1294,11 @@ func auditLogDecom(ctx context.Context, apiName, bucket, object, versionID strin
 	if err != nil {
 		errStr = err.Error()
 	}
-	auditLogInternal(ctx, bucket, object, AuditLogOptions{
+	auditLogInternal(ctx, AuditLogOptions{
 		Event:     "decommission",
 		APIName:   apiName,
+		Bucket:    bucket,
+		Object:    object,
 		VersionID: versionID,
 		Error:     errStr,
 	})

cmd/erasure-server-pool-decom_test.go

@@ -112,16 +112,16 @@ func TestPoolMetaValidate(t *testing.T) {
 		{
 			meta:           meta,
 			pools:          newPools,
-			name:           "Invalid-Commandline",
-			expectedErr:    true,
-			expectedUpdate: false,
+			name:           "Correct-Update",
+			expectedErr:    false,
+			expectedUpdate: true,
 		},
 		{
 			meta:           meta,
 			pools:          reducedPools,
-			name:           "Invalid-Reduced",
-			expectedErr:    true,
-			expectedUpdate: false,
+			name:           "Correct-Update",
+			expectedErr:    false,
+			expectedUpdate: true,
 		},
 		{
 			meta:           meta,

cmd/erasure-server-pool-rebalance.go

@@ -0,0 +1,876 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math"
+	"math/rand"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/lithammer/shortuuid/v4"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/env"
+)
+
+//go:generate msgp -file $GOFILE -unexported
+
+// rebalanceStats contains per-pool rebalance statistics like number of objects,
+// versions and bytes rebalanced out of a pool
+type rebalanceStats struct {
+	InitFreeSpace uint64 `json:"initFreeSpace" msg:"ifs"` // Pool free space at the start of rebalance
+	InitCapacity  uint64 `json:"initCapacity" msg:"ic"`   // Pool capacity at the start of rebalance
+
+	Buckets           []string      `json:"buckets" msg:"bus"`           // buckets being rebalanced or to be rebalanced
+	RebalancedBuckets []string      `json:"rebalancedBuckets" msg:"rbs"` // buckets rebalanced
+	Bucket            string        `json:"bucket" msg:"bu"`             // Last rebalanced bucket
+	Object            string        `json:"object" msg:"ob"`             // Last rebalanced object
+	NumObjects        uint64        `json:"numObjects" msg:"no"`         // Number of objects rebalanced
+	NumVersions       uint64        `json:"numVersions" msg:"nv"`        // Number of versions rebalanced
+	Bytes             uint64        `json:"bytes" msg:"bs"`              // Number of bytes rebalanced
+	Participating     bool          `json:"participating" msg:"par"`
+	Info              rebalanceInfo `json:"info" msg:"inf"`
+}
+
+func (rs *rebalanceStats) update(bucket string, oi ObjectInfo) {
+	if oi.IsLatest {
+		rs.NumObjects++
+	}
+
+	rs.NumVersions++
+	rs.Bytes += uint64(oi.Size)
+	rs.Bucket = bucket
+	rs.Object = oi.Name
+}
+
+type rstats []*rebalanceStats
+
+//go:generate stringer -type=rebalStatus -trimprefix=rebal $GOFILE
+type rebalStatus uint8
+
+const (
+	rebalNone rebalStatus = iota
+	rebalStarted
+	rebalCompleted
+	rebalStopped
+	rebalFailed
+)
+
+type rebalanceInfo struct {
+	StartTime time.Time   `msg:"startTs"` // Time at which rebalance-start was issued
+	EndTime   time.Time   `msg:"stopTs"`  // Time at which rebalance operation completed or rebalance-stop was called
+	Status    rebalStatus `msg:"status"`  // Current state of rebalance operation. One of Started|Stopped|Completed|Failed.
+}
+
+// rebalanceMeta contains information pertaining to an ongoing rebalance operation.
+type rebalanceMeta struct {
+	cancel          context.CancelFunc `msg:"-"` // to be invoked on rebalance-stop
+	lastRefreshedAt time.Time          `msg:"-"`
+	StoppedAt       time.Time          `msg:"stopTs"` // Time when rebalance-stop was issued.
+	ID              string             `msg:"id"`     // ID of the ongoing rebalance operation
+	PercentFreeGoal float64            `msg:"pf"`     // Computed from total free space and capacity at the start of rebalance
+	PoolStats       []*rebalanceStats  `msg:"rss"`    // Per-pool rebalance stats keyed by pool index
+}
+
+var errRebalanceNotStarted = errors.New("rebalance not started")
+
+func (z *erasureServerPools) loadRebalanceMeta(ctx context.Context) error {
+	r := &rebalanceMeta{}
+	err := r.load(ctx, z.serverPools[0])
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil
+		}
+		return err
+	}
+
+	z.rebalMu.Lock()
+	z.rebalMeta = r
+	z.rebalMu.Unlock()
+
+	return nil
+}
+
+// initRebalanceMeta initializes rebalance metadata for a new rebalance
+// operation and saves it in the object store.
+func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []string) (arn string, err error) {
+	r := &rebalanceMeta{
+		ID:        shortuuid.New(),
+		PoolStats: make([]*rebalanceStats, len(z.serverPools)),
+	}
+
+	// Fetch disk capacity and available space.
+	si, _ := z.StorageInfo(ctx)
+	diskStats := make([]struct {
+		AvailableSpace uint64
+		TotalSpace     uint64
+	}, len(z.serverPools))
+	var totalCap, totalFree uint64
+	for _, disk := range si.Disks {
+		totalCap += disk.TotalSpace
+		totalFree += disk.AvailableSpace
+
+		diskStats[disk.PoolIndex].AvailableSpace += disk.AvailableSpace
+		diskStats[disk.PoolIndex].TotalSpace += disk.TotalSpace
+	}
+	r.PercentFreeGoal = float64(totalFree) / float64(totalCap)
+
+	now := time.Now()
+	for idx := range z.serverPools {
+		r.PoolStats[idx] = &rebalanceStats{
+			Buckets:           make([]string, len(buckets)),
+			RebalancedBuckets: make([]string, 0, len(buckets)),
+			InitFreeSpace:     diskStats[idx].AvailableSpace,
+			InitCapacity:      diskStats[idx].TotalSpace,
+		}
+		copy(r.PoolStats[idx].Buckets, buckets)
+
+		if pfi := float64(diskStats[idx].AvailableSpace) / float64(diskStats[idx].TotalSpace); pfi < r.PercentFreeGoal {
+			r.PoolStats[idx].Participating = true
+			r.PoolStats[idx].Info = rebalanceInfo{
+				StartTime: now,
+				Status:    rebalStarted,
+			}
+		}
+	}
+
+	err = r.save(ctx, z.serverPools[0])
+	if err != nil {
+		return arn, err
+	}
+
+	z.rebalMeta = r
+	return r.ID, nil
+}
+
+func (z *erasureServerPools) updatePoolStats(poolIdx int, bucket string, oi ObjectInfo) {
+	z.rebalMu.Lock()
+	defer z.rebalMu.Unlock()
+
+	r := z.rebalMeta
+	if r == nil {
+		return
+	}
+
+	r.PoolStats[poolIdx].update(bucket, oi)
+}
+
+const (
+	rebalMetaName = "rebalance.bin"
+	rebalMetaFmt  = 1
+	rebalMetaVer  = 1
+)
+
+func (z *erasureServerPools) nextRebalBucket(poolIdx int) (string, bool) {
+	z.rebalMu.RLock()
+	defer z.rebalMu.RUnlock()
+
+	r := z.rebalMeta
+	if r == nil {
+		return "", false
+	}
+
+	ps := r.PoolStats[poolIdx]
+	if ps == nil {
+		return "", false
+	}
+
+	if ps.Info.Status == rebalCompleted || !ps.Participating {
+		return "", false
+	}
+
+	if len(ps.Buckets) == 0 {
+		return "", false
+	}
+
+	return ps.Buckets[0], true
+}
+
+func (z *erasureServerPools) bucketRebalanceDone(bucket string, poolIdx int) {
+	z.rebalMu.Lock()
+	defer z.rebalMu.Unlock()
+
+	ps := z.rebalMeta.PoolStats[poolIdx]
+	if ps == nil {
+		return
+	}
+
+	for i, b := range ps.Buckets {
+		if b == bucket {
+			ps.Buckets = append(ps.Buckets[:i], ps.Buckets[i+1:]...)
+			ps.RebalancedBuckets = append(ps.RebalancedBuckets, bucket)
+			break
+		}
+	}
+}
+
+func (r *rebalanceMeta) load(ctx context.Context, store objectIO) error {
+	return r.loadWithOpts(ctx, store, ObjectOptions{})
+}
+
+func (r *rebalanceMeta) loadWithOpts(ctx context.Context, store objectIO, opts ObjectOptions) error {
+	data, _, err := readConfigWithMetadata(ctx, store, rebalMetaName, opts)
+	if err != nil {
+		return err
+	}
+
+	if len(data) == 0 {
+		return nil
+	}
+	if len(data) <= 4 {
+		return fmt.Errorf("rebalanceMeta: no data")
+	}
+
+	// Read header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case rebalMetaFmt:
+	default:
+		return fmt.Errorf("rebalanceMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case rebalMetaVer:
+	default:
+		return fmt.Errorf("rebalanceMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+
+	// OK, parse data.
+	if _, err = r.UnmarshalMsg(data[4:]); err != nil {
+		return err
+	}
+
+	r.lastRefreshedAt = time.Now()
+
+	return nil
+}
+
+func (r *rebalanceMeta) saveWithOpts(ctx context.Context, store objectIO, opts ObjectOptions) error {
+	data := make([]byte, 4, r.Msgsize()+4)
+
+	// Initialize the header.
+	binary.LittleEndian.PutUint16(data[0:2], rebalMetaFmt)
+	binary.LittleEndian.PutUint16(data[2:4], rebalMetaVer)
+
+	buf, err := r.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	return saveConfigWithOpts(ctx, store, rebalMetaName, buf, opts)
+}
+
+func (r *rebalanceMeta) save(ctx context.Context, store objectIO) error {
+	return r.saveWithOpts(ctx, store, ObjectOptions{})
+}
+
+func (z *erasureServerPools) IsRebalanceStarted() bool {
+	z.rebalMu.RLock()
+	defer z.rebalMu.RUnlock()
+
+	if r := z.rebalMeta; r != nil {
+		if r.StoppedAt.IsZero() {
+			return true
+		}
+	}
+	return false
+}
+
+func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool {
+	z.rebalMu.RLock()
+	defer z.rebalMu.RUnlock()
+
+	if r := z.rebalMeta; r != nil {
+		if !r.StoppedAt.IsZero() {
+			return false
+		}
+		ps := z.rebalMeta.PoolStats[poolIndex]
+		return ps.Participating && ps.Info.Status == rebalStarted
+	}
+	return false
+}
+
+func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) (err error) {
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Save rebalance.bin periodically.
+	go func() {
+		// Update rebalance.bin periodically once every 5-10s, chosen randomly
+		// to avoid multiple pool leaders herding to update around the same
+		// time.
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		randSleepFor := func() time.Duration {
+			return 5*time.Second + time.Duration(float64(5*time.Second)*r.Float64())
+		}
+
+		timer := time.NewTimer(randSleepFor())
+		defer timer.Stop()
+		var rebalDone bool
+		var traceMsg string
+
+		for {
+			select {
+			case <-doneCh:
+				// rebalance completed for poolIdx
+				now := time.Now()
+				z.rebalMu.Lock()
+				z.rebalMeta.PoolStats[poolIdx].Info.Status = rebalCompleted
+				z.rebalMeta.PoolStats[poolIdx].Info.EndTime = now
+				z.rebalMu.Unlock()
+
+				rebalDone = true
+				traceMsg = fmt.Sprintf("completed at %s", now)
+
+			case <-ctx.Done():
+
+				// rebalance stopped for poolIdx
+				now := time.Now()
+				z.rebalMu.Lock()
+				z.rebalMeta.PoolStats[poolIdx].Info.Status = rebalStopped
+				z.rebalMeta.PoolStats[poolIdx].Info.EndTime = now
+				z.rebalMu.Unlock()
+
+				rebalDone = true
+				traceMsg = fmt.Sprintf("stopped at %s", now)
+
+			case <-timer.C:
+				traceMsg = fmt.Sprintf("saved at %s", time.Now())
+			}
+
+			stopFn := globalRebalanceMetrics.log(rebalanceMetricSaveMetadata, poolIdx, traceMsg)
+			err := z.saveRebalanceStats(ctx, poolIdx, rebalSaveStats)
+			stopFn(err)
+			logger.LogIf(ctx, err)
+			timer.Reset(randSleepFor())
+
+			if rebalDone {
+				return
+			}
+		}
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		bucket, ok := z.nextRebalBucket(poolIdx)
+		if !ok {
+			// no more buckets to rebalance or target free_space/capacity reached
+			break
+		}
+
+		stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBucket, poolIdx, bucket)
+		err = z.rebalanceBucket(ctx, bucket, poolIdx)
+		if err != nil {
+			stopFn(err)
+			logger.LogIf(ctx, err)
+			return
+		}
+		stopFn(nil)
+		z.bucketRebalanceDone(bucket, poolIdx)
+	}
+
+	return err
+}
+
+func (z *erasureServerPools) checkIfRebalanceDone(poolIdx int) bool {
+	z.rebalMu.Lock()
+	defer z.rebalMu.Unlock()
+
+	// check if enough objects have been rebalanced
+	r := z.rebalMeta
+	poolStats := r.PoolStats[poolIdx]
+	if poolStats.Info.Status == rebalCompleted {
+		return true
+	}
+
+	pfi := float64(poolStats.InitFreeSpace+poolStats.Bytes) / float64(poolStats.InitCapacity)
+	// Mark pool rebalance as done if within 5% from PercentFreeGoal.
+	if diff := math.Abs(pfi - r.PercentFreeGoal); diff <= 0.05 {
+		r.PoolStats[poolIdx].Info.Status = rebalCompleted
+		r.PoolStats[poolIdx].Info.EndTime = time.Now()
+		return true
+	}
+
+	return false
+}
+
+// rebalanceBucket rebalances objects under bucket in poolIdx pool
+func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) error {
+	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
+	vc, _ := globalBucketVersioningSys.Get(bucket)
+	// Check if the current bucket has a configured lifecycle policy
+	lc, _ := globalLifecycleSys.Get(bucket)
+	// Check if bucket is object locked.
+	lr, _ := globalBucketObjectLockSys.Get(bucket)
+
+	pool := z.serverPools[poolIdx]
+	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
+	wStr := env.Get(envRebalanceWorkers, strconv.Itoa(len(pool.sets)))
+	workerSize, err := strconv.Atoi(wStr)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("invalid %s value: %s err: %v, defaulting to %d", envRebalanceWorkers, wStr, err, len(pool.sets)))
+		workerSize = len(pool.sets)
+	}
+	workers := make(chan struct{}, workerSize)
+	var wg sync.WaitGroup
+	for _, set := range pool.sets {
+		set := set
+		disks := set.getOnlineDisks()
+		if len(disks) == 0 {
+			logger.LogIf(ctx, fmt.Errorf("no online disks found for set with endpoints %s",
+				set.getEndpoints()))
+			continue
+		}
+
+		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
+			if lc == nil {
+				return false
+			}
+			versioned := vc != nil && vc.Versioned(object)
+			objInfo := fi.ToObjectInfo(bucket, object, versioned)
+			event := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			switch action := event.Action; action {
+			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+				// Skip this entry.
+				return true
+			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				// Skip this entry.
+				return true
+			}
+			return false
+		}
+
+		rebalanceEntry := func(entry metaCacheEntry) {
+			defer func() {
+				<-workers
+				wg.Done()
+			}()
+
+			if entry.isDir() {
+				return
+			}
+
+			// rebalance on poolIdx has reached its goal
+			if z.checkIfRebalanceDone(poolIdx) {
+				return
+			}
+
+			fivs, err := entry.fileInfoVersions(bucket)
+			if err != nil {
+				return
+			}
+
+			// We need a reversed order for rebalance,
+			// to create the appropriate stack.
+			versionsSorter(fivs.Versions).reverse()
+
+			var rebalanced int
+			for _, version := range fivs.Versions {
+				// Skip transitioned objects for now. TBD
+				if version.IsRemote() {
+					continue
+				}
+
+				// Apply lifecycle rules on the objects that are expired.
+				if filterLifecycle(bucket, version.Name, version) {
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s (%s) expired object based on ILM rules, skipping and scheduled for deletion", bucket, version.Name, version.VersionID))
+					continue
+				}
+
+				// We will skip rebalancing delete markers
+				// with single version, its as good as there
+				// is no data associated with the object.
+				if version.Deleted && len(fivs.Versions) == 1 {
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no data to be rebalanced", bucket, version.Name))
+					continue
+				}
+
+				if version.Deleted {
+					_, err := z.DeleteObject(ctx,
+						bucket,
+						version.Name,
+						ObjectOptions{
+							Versioned:         vc.PrefixEnabled(version.Name),
+							VersionID:         version.VersionID,
+							MTime:             version.ModTime,
+							DeleteReplication: version.ReplicationState,
+							DeleteMarker:      true, // make sure we create a delete marker
+							SkipRebalancing:   true, // make sure we skip the decommissioned pool
+						})
+					var failure bool
+					if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+						logger.LogIf(ctx, err)
+						failure = true
+					}
+
+					if !failure {
+						z.updatePoolStats(poolIdx, bucket, version.ToObjectInfo(bucket, version.Name, vc.PrefixEnabled(version.Name)))
+						rebalanced++
+					}
+					continue
+				}
+
+				var failure bool
+				var oi ObjectInfo
+				for try := 0; try < 3; try++ {
+					// GetObjectReader.Close is called by rebalanceObject
+					gr, err := set.GetObjectNInfo(ctx,
+						bucket,
+						encodeDirObject(version.Name),
+						nil,
+						http.Header{},
+						noLock, // all mutations are blocked reads are safe without locks.
+						ObjectOptions{
+							VersionID:    version.VersionID,
+							NoDecryption: true,
+						})
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+						// object deleted by the application, nothing to do here we move on.
+						return
+					}
+					if err != nil {
+						failure = true
+						logger.LogIf(ctx, err)
+						continue
+					}
+
+					stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name)
+					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
+						stopFn(err)
+						failure = true
+						logger.LogIf(ctx, err)
+						continue
+					}
+
+					stopFn(nil)
+					failure = false
+					oi = gr.ObjInfo
+					break
+				}
+
+				if failure {
+					break // break out on first error
+				}
+				z.updatePoolStats(poolIdx, bucket, oi)
+				rebalanced++
+			}
+
+			// if all versions were rebalanced, we can delete the object versions.
+			if rebalanced == len(fivs.Versions) {
+				stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceRemoveObject, poolIdx, bucket, entry.name)
+				_, err := set.DeleteObject(ctx,
+					bucket,
+					encodeDirObject(entry.name),
+					ObjectOptions{
+						DeletePrefix: true, // use prefix delete to delete all versions at once.
+					},
+				)
+				stopFn(err)
+				auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
+				if err != nil {
+					logger.LogIf(ctx, err)
+				}
+			}
+		}
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			// How to resolve partial results.
+			resolver := metadataResolutionParams{
+				dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+				objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+				bucket:    bucket,
+			}
+			err := listPathRaw(ctx, listPathRawOptions{
+				disks:          disks,
+				bucket:         bucket,
+				recursive:      true,
+				forwardTo:      "",
+				minDisks:       len(disks) / 2, // to capture all quorum ratios
+				reportNotFound: false,
+				agreed: func(entry metaCacheEntry) {
+					workers <- struct{}{}
+					wg.Add(1)
+					go rebalanceEntry(entry)
+				},
+				partial: func(entries metaCacheEntries, _ []error) {
+					entry, ok := entries.resolve(&resolver)
+					if ok {
+						workers <- struct{}{}
+						wg.Add(1)
+						go rebalanceEntry(*entry)
+					}
+				},
+				finished: nil,
+			})
+			logger.LogIf(ctx, err)
+		}()
+	}
+	wg.Wait()
+	return nil
+}
+
+type rebalSaveOpts uint8
+
+const (
+	rebalSaveStats rebalSaveOpts = iota
+	rebalSaveStoppedAt
+)
+
+func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int, opts rebalSaveOpts) error {
+	lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
+	lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
+		return err
+	}
+	defer lock.Unlock(lkCtx.Cancel)
+
+	ctx = lkCtx.Context()
+	noLockOpts := ObjectOptions{NoLock: true}
+	r := &rebalanceMeta{}
+	if err := r.loadWithOpts(ctx, z.serverPools[0], noLockOpts); err != nil {
+		return err
+	}
+
+	z.rebalMu.Lock()
+	defer z.rebalMu.Unlock()
+
+	switch opts {
+	case rebalSaveStoppedAt:
+		r.StoppedAt = time.Now()
+	case rebalSaveStats:
+		r.PoolStats[poolIdx] = z.rebalMeta.PoolStats[poolIdx]
+	}
+	z.rebalMeta = r
+
+	err = z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
+	return err
+}
+
+func auditLogRebalance(ctx context.Context, apiName, bucket, object, versionID string, err error) {
+	errStr := ""
+	if err != nil {
+		errStr = err.Error()
+	}
+	auditLogInternal(ctx, AuditLogOptions{
+		Event:     "rebalance",
+		APIName:   apiName,
+		Bucket:    bucket,
+		Object:    object,
+		VersionID: versionID,
+		Error:     errStr,
+	})
+}
+
+func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
+	oi := gr.ObjInfo
+
+	defer func() {
+		gr.Close()
+		auditLogRebalance(ctx, "RebalanceCopyData", oi.Bucket, oi.Name, oi.VersionID, err)
+	}()
+
+	actualSize, err := oi.GetActualSize()
+	if err != nil {
+		return err
+	}
+
+	if oi.isMultipart() {
+		res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
+			VersionID:   oi.VersionID,
+			MTime:       oi.ModTime,
+			UserDefined: oi.UserDefined,
+		})
+		if err != nil {
+			return fmt.Errorf("rebalanceObject: NewMultipartUpload() %w", err)
+		}
+		defer z.AbortMultipartUpload(ctx, bucket, oi.Name, res.UploadID, ObjectOptions{})
+
+		parts := make([]CompletePart, len(oi.Parts))
+		for i, part := range oi.Parts {
+			hr, err := hash.NewReader(gr, part.Size, "", "", part.ActualSize)
+			if err != nil {
+				return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
+			}
+			pi, err := z.PutObjectPart(ctx, bucket, oi.Name, res.UploadID,
+				part.Number,
+				NewPutObjReader(hr),
+				ObjectOptions{
+					PreserveETag: part.ETag, // Preserve original ETag to ensure same metadata.
+					IndexCB: func() []byte {
+						return part.Index // Preserve part Index to ensure decompression works.
+					},
+				})
+			if err != nil {
+				return fmt.Errorf("rebalanceObject: PutObjectPart() %w", err)
+			}
+			parts[i] = CompletePart{
+				ETag:       pi.ETag,
+				PartNumber: pi.PartNumber,
+			}
+		}
+		_, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{
+			MTime: oi.ModTime,
+		})
+		if err != nil {
+			err = fmt.Errorf("rebalanceObject: CompleteMultipartUpload() %w", err)
+		}
+		return err
+	}
+
+	hr, err := hash.NewReader(gr, oi.Size, "", "", actualSize)
+	if err != nil {
+		return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
+	}
+	_, err = z.PutObject(ctx,
+		bucket,
+		oi.Name,
+		NewPutObjReader(hr),
+		ObjectOptions{
+			VersionID:    oi.VersionID,
+			MTime:        oi.ModTime,
+			UserDefined:  oi.UserDefined,
+			PreserveETag: oi.ETag, // Preserve original ETag to ensure same metadata.
+			IndexCB: func() []byte {
+				return oi.Parts[0].Index // Preserve part Index to ensure decompression works.
+			},
+		})
+	if err != nil {
+		err = fmt.Errorf("rebalanceObject: PutObject() %w", err)
+	}
+	return err
+}
+
+func (z *erasureServerPools) StartRebalance() {
+	z.rebalMu.Lock()
+	if z.rebalMeta == nil || !z.rebalMeta.StoppedAt.IsZero() { // rebalance not running, nothing to do
+		z.rebalMu.Unlock()
+		return
+	}
+	ctx, cancel := context.WithCancel(GlobalContext)
+	z.rebalMeta.cancel = cancel // to be used when rebalance-stop is called
+	z.rebalMu.Unlock()
+
+	z.rebalMu.RLock()
+	participants := make([]bool, len(z.rebalMeta.PoolStats))
+	for i, ps := range z.rebalMeta.PoolStats {
+		// skip pools which have completed rebalancing
+		if ps.Info.Status != rebalStarted {
+			continue
+		}
+
+		participants[i] = ps.Participating
+	}
+	z.rebalMu.RUnlock()
+
+	for poolIdx, doRebalance := range participants {
+		if !doRebalance {
+			continue
+		}
+		// nothing to do if this node is not pool's first node (i.e pool's rebalance 'leader').
+		if !globalEndpoints[poolIdx].Endpoints[0].IsLocal {
+			continue
+		}
+
+		go func(idx int) {
+			stopfn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBuckets, idx)
+			err := z.rebalanceBuckets(ctx, idx)
+			stopfn(err)
+		}(poolIdx)
+	}
+	return
+}
+
+// StopRebalance signals the rebalance goroutine running on this node (if any)
+// to stop, using the context.CancelFunc(s) saved at the time ofStartRebalance.
+func (z *erasureServerPools) StopRebalance() error {
+	z.rebalMu.Lock()
+	defer z.rebalMu.Unlock()
+
+	r := z.rebalMeta
+	if r == nil { // rebalance not running in this node, nothing to do
+		return nil
+	}
+
+	if cancel := r.cancel; cancel != nil {
+		// cancel != nil only on pool leaders
+		r.cancel = nil
+		cancel()
+	}
+	return nil
+}
+
+// for rebalance trace support
+type rebalanceMetrics struct{}
+
+var globalRebalanceMetrics rebalanceMetrics
+
+//go:generate stringer -type=rebalanceMetric -trimprefix=rebalanceMetric $GOFILE
+type rebalanceMetric uint8
+
+const (
+	rebalanceMetricRebalanceBuckets rebalanceMetric = iota
+	rebalanceMetricRebalanceBucket
+	rebalanceMetricRebalanceObject
+	rebalanceMetricRebalanceRemoveObject
+	rebalanceMetricSaveMetadata
+)
+
+func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string) madmin.TraceInfo {
+	var errStr string
+	if err != nil {
+		errStr = err.Error()
+	}
+	return madmin.TraceInfo{
+		TraceType: madmin.TraceRebalance,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  fmt.Sprintf("rebalance.%s (pool-id=%d)", r.String(), poolIdx),
+		Duration:  duration,
+		Path:      path,
+		Error:     errStr,
+	}
+}
+
+func (p *rebalanceMetrics) log(r rebalanceMetric, poolIdx int, paths ...string) func(err error) {
+	startTime := time.Now()
+	return func(err error) {
+		duration := time.Since(startTime)
+		if globalTrace.NumSubscribers(madmin.TraceRebalance) > 0 {
+			globalTrace.Publish(rebalanceTrace(r, poolIdx, startTime, duration, err, strings.Join(paths, " ")))
+		}
+	}
+}

cmd/erasure-server-pool-rebalance_gen_test.go

@@ -0,0 +1,575 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalrebalanceInfo(t *testing.T) {
+	v := rebalanceInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgrebalanceInfo(b *testing.B) {
+	v := rebalanceInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgrebalanceInfo(b *testing.B) {
+	v := rebalanceInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalrebalanceInfo(b *testing.B) {
+	v := rebalanceInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecoderebalanceInfo(t *testing.T) {
+	v := rebalanceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecoderebalanceInfo Msgsize() is inaccurate")
+	}
+
+	vn := rebalanceInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncoderebalanceInfo(b *testing.B) {
+	v := rebalanceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecoderebalanceInfo(b *testing.B) {
+	v := rebalanceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalrebalanceMeta(t *testing.T) {
+	v := rebalanceMeta{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgrebalanceMeta(b *testing.B) {
+	v := rebalanceMeta{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgrebalanceMeta(b *testing.B) {
+	v := rebalanceMeta{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalrebalanceMeta(b *testing.B) {
+	v := rebalanceMeta{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecoderebalanceMeta(t *testing.T) {
+	v := rebalanceMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecoderebalanceMeta Msgsize() is inaccurate")
+	}
+
+	vn := rebalanceMeta{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncoderebalanceMeta(b *testing.B) {
+	v := rebalanceMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecoderebalanceMeta(b *testing.B) {
+	v := rebalanceMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalrebalanceMetrics(t *testing.T) {
+	v := rebalanceMetrics{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgrebalanceMetrics(b *testing.B) {
+	v := rebalanceMetrics{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgrebalanceMetrics(b *testing.B) {
+	v := rebalanceMetrics{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalrebalanceMetrics(b *testing.B) {
+	v := rebalanceMetrics{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecoderebalanceMetrics(t *testing.T) {
+	v := rebalanceMetrics{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecoderebalanceMetrics Msgsize() is inaccurate")
+	}
+
+	vn := rebalanceMetrics{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncoderebalanceMetrics(b *testing.B) {
+	v := rebalanceMetrics{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecoderebalanceMetrics(b *testing.B) {
+	v := rebalanceMetrics{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalrebalanceStats(t *testing.T) {
+	v := rebalanceStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgrebalanceStats(b *testing.B) {
+	v := rebalanceStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgrebalanceStats(b *testing.B) {
+	v := rebalanceStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalrebalanceStats(b *testing.B) {
+	v := rebalanceStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecoderebalanceStats(t *testing.T) {
+	v := rebalanceStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecoderebalanceStats Msgsize() is inaccurate")
+	}
+
+	vn := rebalanceStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncoderebalanceStats(b *testing.B) {
+	v := rebalanceStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecoderebalanceStats(b *testing.B) {
+	v := rebalanceStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalrstats(t *testing.T) {
+	v := rstats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgrstats(b *testing.B) {
+	v := rstats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgrstats(b *testing.B) {
+	v := rstats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalrstats(b *testing.B) {
+	v := rstats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecoderstats(t *testing.T) {
+	v := rstats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecoderstats Msgsize() is inaccurate")
+	}
+
+	vn := rstats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncoderstats(b *testing.B) {
+	v := rstats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecoderstats(b *testing.B) {
+	v := rstats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/erasure-server-pool.go

@@ -31,6 +31,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -42,11 +43,13 @@ import (
 )
 
 type erasureServerPools struct {
-	GatewayUnsupported
-
 	poolMetaMutex sync.RWMutex
 	poolMeta      poolMeta
-	serverPools   []*erasureSets
+
+	rebalMu   sync.RWMutex
+	rebalMeta *rebalanceMeta
+
+	serverPools []*erasureSets
 
 	// Shut down async operations
 	shutdown context.CancelFunc
@@ -61,22 +64,6 @@ func (z *erasureServerPools) SinglePool() bool {
 
 // Initialize new pool of erasure sets.
 func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) {
-	if endpointServerPools.NEndpoints() == 1 {
-		ep := endpointServerPools[0]
-		storageDisks, format, err := waitForFormatErasure(true, ep.Endpoints, 1, ep.SetCount, ep.DrivesPerSet, "", "")
-		if err != nil {
-			return nil, err
-		}
-
-		objLayer, err := newErasureSingle(ctx, storageDisks[0], format)
-		if err != nil {
-			return nil, err
-		}
-
-		globalLocalDrives = storageDisks
-		return objLayer, nil
-	}
-
 	var (
 		deploymentID       string
 		distributionAlgo   string
@@ -103,7 +90,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		}
 
 		if err = storageclass.ValidateParity(commonParityDrives, ep.DrivesPerSet); err != nil {
-			return nil, fmt.Errorf("All current serverPools should have same parity ratio - expected %d, got %d", commonParityDrives, ecDrivesNoConfig(ep.DrivesPerSet))
+			return nil, fmt.Errorf("parity validation returned an error %w <- (%d, %d), for pool(%s)", err, commonParityDrives, ep.DrivesPerSet, humanize.Ordinal(i+1))
 		}
 
 		storageDisks[i], formats[i], err = waitForFormatErasure(local, ep.Endpoints, i+1,
@@ -129,7 +116,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 		// Validate if users brought different DeploymentID pools.
 		if deploymentID != formats[i].ID {
-			return nil, fmt.Errorf("All serverPools should have same deployment ID expected %s, got %s", deploymentID, formats[i].ID)
+			return nil, fmt.Errorf("all pools must have same deployment ID - expected %s, got %s for pool(%s)", deploymentID, formats[i].ID, humanize.Ordinal(i+1))
 		}
 
 		z.serverPools[i], err = newErasureSets(ctx, ep, storageDisks[i], formats[i], commonParityDrives, i)
@@ -329,8 +316,9 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 	g := errgroup.WithNErrs(len(z.serverPools))
 	for index := range z.serverPools {
 		index := index
-		// skip suspended pools for any new I/O.
-		if z.IsSuspended(index) {
+		// Skip suspended pools or pools participating in rebalance for any new
+		// I/O.
+		if z.IsSuspended(index) || z.IsPoolRebalancing(index) {
 			continue
 		}
 		pool := z.serverPools[index]
@@ -428,6 +416,10 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 		if z.IsSuspended(pinfo.Index) && opts.SkipDecommissioned {
 			continue
 		}
+		// Skip object if it's from pools participating in a rebalance operation.
+		if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) {
+			continue
+		}
 
 		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
 			return pinfo, pinfo.Err
@@ -468,6 +460,7 @@ func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucke
 	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{
 		NoLock:             true,
 		SkipDecommissioned: true,
+		SkipRebalancing:    true,
 	})
 }
 
@@ -491,7 +484,10 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
 // if none are found falls back to most available space pool, this function is
 // designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
 func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
-	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{SkipDecommissioned: true})
+	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{
+		SkipDecommissioned: true,
+		SkipRebalancing:    true,
+	})
 	if err != nil && !isErrObjectNotFound(err) {
 		return idx, err
 	}
@@ -1080,9 +1076,12 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 				NoLock: true,
 			})
 			if err != nil {
-				derrs[j] = err
+				if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+					derrs[j] = err
+				}
 				dobjects[j] = DeletedObject{
 					ObjectName: obj.ObjectName,
+					VersionID:  obj.VersionID,
 				}
 				return nil
 			}
@@ -1255,7 +1254,20 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	}
 	for _, obj := range objects {
 		if obj.IsDir && obj.ModTime.IsZero() && delimiter != "" {
-			loi.Prefixes = append(loi.Prefixes, obj.Name)
+			// Only add each once.
+			// With slash delimiter we only get the directory once.
+			found := false
+			if delimiter != slashSeparator {
+				for _, p := range loi.Prefixes {
+					if found {
+						break
+					}
+					found = p == obj.Name
+				}
+			}
+			if !found {
+				loi.Prefixes = append(loi.Prefixes, obj.Name)
+			}
 		} else {
 			loi.Objects = append(loi.Objects, obj)
 		}
@@ -1302,8 +1314,8 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
 			if opts.Lifecycle != nil {
-				action := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo, false)
-				switch action {
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
+				switch evt.Action {
 				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
 					fallthrough
 				case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
@@ -1335,7 +1347,20 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	for _, obj := range objects {
 		if obj.IsDir && obj.ModTime.IsZero() && delimiter != "" {
-			loi.Prefixes = append(loi.Prefixes, obj.Name)
+			// Only add each once.
+			// With slash delimiter we only get the directory once.
+			found := false
+			if delimiter != slashSeparator {
+				for _, p := range loi.Prefixes {
+					if found {
+						break
+					}
+					found = p == obj.Name
+				}
+			}
+			if !found {
+				loi.Prefixes = append(loi.Prefixes, obj.Name)
+			}
 		} else {
 			loi.Objects = append(loi.Objects, obj)
 		}
@@ -1389,9 +1414,10 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	for idx, pool := range z.serverPools {
-		if z.IsSuspended(idx) {
+		if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) {
 			continue
 		}
+
 		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
 		if err != nil {
 			return nil, err
@@ -1438,9 +1464,9 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		if z.IsSuspended(idx) {
 			continue
 		}
-		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+		pi, err := pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 		if err == nil {
-			return pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
+			return pi, nil
 		}
 		switch err.(type) {
 		case InvalidUploadID:
@@ -1502,9 +1528,9 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		if z.IsSuspended(idx) {
 			continue
 		}
-		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+		result, err := pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 		if err == nil {
-			return pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
+			return result, nil
 		}
 		switch err.(type) {
 		case InvalidUploadID:
@@ -1533,9 +1559,9 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		if z.IsSuspended(idx) {
 			continue
 		}
-		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+		err := pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
+			return nil
 		}
 		switch err.(type) {
 		case InvalidUploadID:
@@ -1565,10 +1591,16 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		if z.IsSuspended(idx) {
 			continue
 		}
-		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+		objInfo, err = pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 		if err == nil {
-			return pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
+			return objInfo, nil
 		}
+		switch err.(type) {
+		case InvalidUploadID:
+			// upload id not found move to next pool
+			continue
+		}
+		return objInfo, err
 	}
 
 	return objInfo, InvalidUploadID{
@@ -1588,6 +1620,8 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, o
 		meta, err := globalBucketMetadataSys.Get(bucket)
 		if err == nil {
 			bucketInfo.Created = meta.Created
+			bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
+			bucketInfo.ObjectLocking = meta.LockEnabled
 		}
 		return bucketInfo, nil
 	}
@@ -1602,6 +1636,8 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, o
 		meta, err := globalBucketMetadataSys.Get(bucket)
 		if err == nil {
 			bucketInfo.Created = meta.Created
+			bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
+			bucketInfo.ObjectLocking = meta.LockEnabled
 		}
 		return bucketInfo, nil
 	}
@@ -1665,7 +1701,7 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	}
 
 	// Purge the entire bucket metadata entirely.
-	z.renameAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
+	z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
 	// If site replication is configured, hold on to deleted bucket state until sites sync
 	switch opts.SRDeleteOp {
 	case MarkDelete:
@@ -1675,15 +1711,15 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	return nil
 }
 
-// renameAll will rename bucket+prefix unconditionally across all disks to
+// deleteAll will rename bucket+prefix unconditionally across all disks to
 // minioMetaTmpDeletedBucket + unique uuid,
 // Note that set distribution is ignored so it should only be used in cases where
 // data is not distributed across sets. Errors are logged but individual
 // disk failures are not returned.
-func (z *erasureServerPools) renameAll(ctx context.Context, bucket, prefix string) {
+func (z *erasureServerPools) deleteAll(ctx context.Context, bucket, prefix string) {
 	for _, servers := range z.serverPools {
 		for _, set := range servers.sets {
-			set.renameAll(ctx, bucket, prefix)
+			set.deleteAll(ctx, bucket, prefix)
 		}
 	}
 }
@@ -1875,12 +1911,22 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 						versionsSorter(fivs.Versions).reverse()
 
 						for _, version := range fivs.Versions {
+							send := true
+							if opts.WalkFilter != nil && !opts.WalkFilter(version) {
+								send = false
+							}
+
+							if !send {
+								continue
+							}
+
 							versioned := vcfg != nil && vcfg.Versioned(version.Name)
+							objInfo := version.ToObjectInfo(bucket, version.Name, versioned)
 
 							select {
 							case <-ctx.Done():
 								return
-							case results <- version.ToObjectInfo(bucket, version.Name, versioned):
+							case results <- objInfo:
 							}
 						}
 					}
@@ -1904,7 +1950,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 						path:           path,
 						filterPrefix:   filterPrefix,
 						recursive:      true,
-						forwardTo:      "",
+						forwardTo:      opts.WalkMarker,
 						minDisks:       1,
 						reportNotFound: false,
 						agreed:         loadEntry,
@@ -2026,7 +2072,12 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		if err != nil {
 			return healObjectFn(bucket, entry.name, "")
 		}
-
+		if opts.Remove && !opts.DryRun {
+			err := z.CheckAbandonedParts(ctx, bucket, entry.name, opts)
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", bucket, entry.name, err))
+			}
+		}
 		for _, version := range fivs.Versions {
 			err := healObjectFn(bucket, version.Name, version.VersionID)
 			if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
@@ -2118,12 +2169,6 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 	}
 }
 
-// GetMetrics - returns metrics of local disks
-func (z *erasureServerPools) GetMetrics(ctx context.Context) (*BackendMetrics, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return &BackendMetrics{}, NotImplemented{}
-}
-
 func (z *erasureServerPools) getPoolAndSet(id string) (poolIdx, setIdx, diskIdx int, err error) {
 	for poolIdx := range z.serverPools {
 		format := z.serverPools[poolIdx].format
@@ -2384,3 +2429,30 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 
 	return z.serverPools[idx].RestoreTransitionedObject(ctx, bucket, object, opts)
 }
+
+func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error {
+	object = encodeDirObject(object)
+	if z.SinglePool() {
+		return z.serverPools[0].CheckAbandonedParts(ctx, bucket, object, opts)
+	}
+	errs := make([]error, len(z.serverPools))
+	var wg sync.WaitGroup
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
+		wg.Add(1)
+		go func(idx int, pool *erasureSets) {
+			defer wg.Done()
+			err := pool.CheckAbandonedParts(ctx, bucket, object, opts)
+			if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+				errs[idx] = err
+			}
+		}(idx, pool)
+	}
+	wg.Wait()
+	for _, err := range errs {
+		return err
+	}
+	return nil
+}

cmd/erasure-sets.go

@@ -51,8 +51,6 @@ type setsDsyncLockers [][]dsync.NetLocker
 // object sets. NOTE: There is no dynamic scaling allowed or intended in
 // current design.
 type erasureSets struct {
-	GatewayUnsupported
-
 	sets []*erasureObjects
 
 	// Reference format.
@@ -119,7 +117,7 @@ func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 // Initializes a new StorageAPI from the endpoint argument, returns
 // StorageAPI and also `format` which exists on the disk.
 func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
-	disk, err := newStorageAPIWithoutHealthCheck(endpoint)
+	disk, err := newStorageAPI(endpoint, false)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -253,7 +251,7 @@ func (s *erasureSets) connectDisks() {
 				s.erasureDisks[setIndex][diskIndex] = disk
 			} else {
 				// Enable healthcheck disk for remote endpoint.
-				disk, err = newStorageAPI(endpoint)
+				disk, err = newStorageAPI(endpoint, true)
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					s.erasureDisksMu.Unlock()
@@ -463,17 +461,16 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 
 			// Initialize erasure objects for a given set.
 			s.sets[i] = &erasureObjects{
-				setIndex:              i,
-				poolIndex:             poolIdx,
-				setDriveCount:         setDriveCount,
-				defaultParityCount:    defaultParityCount,
-				getDisks:              s.GetDisks(i),
-				getLockers:            s.GetLockers(i),
-				getEndpoints:          s.GetEndpoints(i),
-				deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second, false),
-				nsMutex:               mutex,
-				bp:                    bp,
-				bpOld:                 bpOld,
+				setIndex:           i,
+				poolIndex:          poolIdx,
+				setDriveCount:      setDriveCount,
+				defaultParityCount: defaultParityCount,
+				getDisks:           s.GetDisks(i),
+				getLockers:         s.GetLockers(i),
+				getEndpoints:       s.GetEndpoints(i),
+				nsMutex:            mutex,
+				bp:                 bp,
+				bpOld:              bpOld,
 			}
 		}(i)
 	}
@@ -1252,7 +1249,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
-	storageDisks, _ := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints.Endpoints)
+	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, false)
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {
@@ -1415,3 +1412,8 @@ func (s *erasureSets) TransitionObject(ctx context.Context, bucket, object strin
 func (s *erasureSets) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
 	return s.getHashedSet(object).RestoreTransitionedObject(ctx, bucket, object, opts)
 }
+
+// CheckAbandonedParts - check object for abandoned parts.
+func (s *erasureSets) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error {
+	return s.getHashedSet(object).checkAbandonedParts(ctx, bucket, object, opts)
+}

cmd/erasure-utils.go

@@ -20,8 +20,10 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/klauspost/reedsolomon"
 	xioutil "github.com/minio/minio/internal/ioutil"
@@ -117,3 +119,16 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 	// Success.
 	return totalWritten, nil
 }
+
+// returns deploymentID from uploadID
+func getDeplIDFromUpload(uploadID string) (string, error) {
+	uploadBytes, err := base64.RawURLEncoding.DecodeString(uploadID)
+	if err != nil {
+		return "", fmt.Errorf("error parsing uploadID %s (%w)", uploadID, err)
+	}
+	slc := strings.SplitN(string(uploadBytes), ".", 2)
+	if len(slc) != 2 {
+		return "", fmt.Errorf("uploadID %s has incorrect format", uploadID)
+	}
+	return slc[0], nil
+}

cmd/erasure.go

@@ -41,8 +41,6 @@ var OfflineDisk StorageAPI // zero value is nil
 
 // erasureObjects - Implements ER object layer.
 type erasureObjects struct {
-	GatewayUnsupported
-
 	setDriveCount      int
 	defaultParityCount int
 
@@ -68,8 +66,6 @@ type erasureObjects struct {
 	// Byte pools used for temporary i/o buffers,
 	// legacy objects.
 	bpOld *bpool.BytePoolCap
-
-	deletedCleanupSleeper *dynamicSleeper
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
@@ -107,9 +103,8 @@ func (d byDiskTotal) Less(i, j int) bool {
 }
 
 func diskErrToDriveState(err error) (state string) {
-	state = madmin.DriveStateUnknown
 	switch {
-	case errors.Is(err, errDiskNotFound):
+	case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded):
 		state = madmin.DriveStateOffline
 	case errors.Is(err, errCorruptedFormat):
 		state = madmin.DriveStateCorrupt
@@ -121,7 +116,10 @@ func diskErrToDriveState(err error) (state string) {
 		state = madmin.DriveStateFaulty
 	case err == nil:
 		state = madmin.DriveStateOk
+	default:
+		state = fmt.Sprintf("%s (cause: %s)", madmin.DriveStateUnknown, err)
 	}
+
 	return
 }
 
@@ -184,18 +182,19 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 	for index := range disks {
 		index := index
 		g.Go(func() error {
+			diskEndpoint := endpoints[index].String()
 			if disks[index] == OfflineDisk {
 				logger.LogIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]))
 				disksInfo[index] = madmin.Disk{
 					State:    diskErrToDriveState(errDiskNotFound),
-					Endpoint: endpoints[index].String(),
+					Endpoint: diskEndpoint,
 				}
 				// Storage disk is empty, perhaps ignored disk or not available.
 				return errDiskNotFound
 			}
 			info, err := disks[index].DiskInfo(context.TODO())
 			di := madmin.Disk{
-				Endpoint:       info.Endpoint,
+				Endpoint:       diskEndpoint,
 				DrivePath:      info.MountPath,
 				TotalSpace:     info.Total,
 				UsedSpace:      info.Used,
@@ -337,7 +336,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 				defer wg.Done()
 				diskPath := disk.Endpoint().Path
 				readDirFn(pathJoin(diskPath, minioMetaTmpDeletedBucket), func(ddir string, typ os.FileMode) error {
-					wait := er.deletedCleanupSleeper.Timer(ctx)
+					wait := deletedCleanupSleeper.Timer(ctx)
 					removeAll(pathJoin(diskPath, minioMetaTmpDeletedBucket, ddir))
 					wait()
 					return nil

cmd/event-notification.go

@@ -29,6 +29,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/pkg/bucket/policy"
 )
 
@@ -100,11 +101,6 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
 		return errServerNotInitialized
 	}
 
-	// In gateway mode, notifications are not supported - except NAS gateway.
-	if globalIsGateway && !objAPI.IsNotificationSupported() {
-		return nil
-	}
-
 	if err := evnot.targetList.Add(globalConfigTargetList.Targets()...); err != nil {
 		return err
 	}
@@ -326,11 +322,7 @@ func sendEvent(args eventArgs) {
 	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
 	crypto.RemoveInternalEntries(args.Object.UserDefined)
 
-	// globalNotificationSys is not initialized in gateway mode.
-	if globalNotificationSys == nil {
-		return
-	}
-	if globalHTTPListen.NumSubscribers(args.EventName) > 0 {
+	if globalHTTPListen.NumSubscribers(pubsub.MaskFromMaskable(args.EventName)) > 0 {
 		globalHTTPListen.Publish(args.ToEvent(false))
 	}
 

cmd/format-erasure.go

@@ -177,7 +177,7 @@ func formatGetBackendErasureVersion(b []byte) (string, error) {
 		return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)
 	}
 	if meta.Format != formatBackendErasure && meta.Format != formatBackendErasureSingle {
-		return "", fmt.Errorf(`found backend type %s, expected %s or %s`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
+		return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://min.io/docs/minio/linux/operations/install-deploy-manage/migrate-fs-gateway.html`, meta.Format, formatBackendErasure, formatBackendErasureSingle)
 	}
 	// Erasure backend found, proceed to detect version.
 	format := &formatErasureVersionDetect{}
@@ -666,30 +666,16 @@ func closeStorageDisks(storageDisks ...StorageAPI) {
 	wg.Wait()
 }
 
-func initStorageDisksWithErrorsWithoutHealthCheck(endpoints Endpoints) ([]StorageAPI, []error) {
-	// Bootstrap disks.
-	storageDisks := make([]StorageAPI, len(endpoints))
-	g := errgroup.WithNErrs(len(endpoints))
-	for index := range endpoints {
-		index := index
-		g.Go(func() (err error) {
-			storageDisks[index], err = newStorageAPIWithoutHealthCheck(endpoints[index])
-			return err
-		}, index)
-	}
-	return storageDisks, g.Wait()
-}
-
 // Initialize storage disks for each endpoint.
 // Errors are returned for each endpoint with matching index.
-func initStorageDisksWithErrors(endpoints Endpoints) ([]StorageAPI, []error) {
+func initStorageDisksWithErrors(endpoints Endpoints, healthCheck bool) ([]StorageAPI, []error) {
 	// Bootstrap disks.
 	storageDisks := make([]StorageAPI, len(endpoints))
 	g := errgroup.WithNErrs(len(endpoints))
 	for index := range endpoints {
 		index := index
 		g.Go(func() (err error) {
-			storageDisks[index], err = newStorageAPI(endpoints[index])
+			storageDisks[index], err = newStorageAPI(endpoints[index], healthCheck)
 			return err
 		}, index)
 	}

cmd/format-erasure_test.go

@@ -38,7 +38,7 @@ func TestFixFormatV3(t *testing.T) {
 	}
 	endpoints := mustGetNewEndpoints(erasureDirs...)
 
-	storageDisks, errs := initStorageDisksWithErrors(endpoints)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints, true)
 	for _, err := range errs {
 		if err != nil && err != errDiskNotFound {
 			t.Fatal(err)
@@ -559,7 +559,7 @@ func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
 	b.RunParallel(func(pb *testing.PB) {
 		endpoints := endpoints
 		for pb.Next() {
-			initStorageDisksWithErrors(endpoints)
+			initStorageDisksWithErrors(endpoints, true)
 		}
 	})
 }

cmd/format-fs.go

@@ -1,387 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"math/rand"
-	"os"
-	"path"
-	"time"
-
-	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/lock"
-	"github.com/minio/minio/internal/logger"
-)
-
-// FS format version strings.
-const (
-	formatBackendFS   = "fs"
-	formatFSVersionV1 = "1"
-	formatFSVersionV2 = "2"
-)
-
-// formatFSV1 - structure holds format version '1'.
-type formatFSV1 struct {
-	formatMetaV1
-	FS struct {
-		Version string `json:"version"`
-	} `json:"fs"`
-}
-
-// formatFSV2 - structure is same as formatFSV1. But the multipart backend
-// structure is flat instead of hierarchy now.
-// In .minio.sys/multipart we have:
-// sha256(bucket/object)/uploadID/[fs.json, 1.etag, 2.etag ....]
-type formatFSV2 = formatFSV1
-
-// Used to detect the version of "fs" format.
-type formatFSVersionDetect struct {
-	FS struct {
-		Version string `json:"version"`
-	} `json:"fs"`
-}
-
-// Generic structure to manage both v1 and v2 structures
-type formatFS struct {
-	formatMetaV1
-	FS interface{} `json:"fs"`
-}
-
-// Returns the latest "fs" format V1
-func newFormatFSV1() (format *formatFSV1) {
-	f := &formatFSV1{}
-	f.Version = formatMetaVersionV1
-	f.Format = formatBackendFS
-	f.ID = mustGetUUID()
-	f.FS.Version = formatFSVersionV1
-	return f
-}
-
-// Returns the field formatMetaV1.Format i.e the string "fs" which is never likely to change.
-// We do not use this function in Erasure to get the format as the file is not fcntl-locked on Erasure.
-func formatMetaGetFormatBackendFS(r io.ReadSeeker) (string, error) {
-	format := &formatMetaV1{}
-	if err := jsonLoad(r, format); err != nil {
-		return "", err
-	}
-	if format.Version == formatMetaVersionV1 {
-		return format.Format, nil
-	}
-	return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, format.Version)
-}
-
-// Returns formatFS.FS.Version
-func formatFSGetVersion(r io.ReadSeeker) (string, error) {
-	format := &formatFSVersionDetect{}
-	if err := jsonLoad(r, format); err != nil {
-		return "", err
-	}
-	return format.FS.Version, nil
-}
-
-// Migrate from V1 to V2. V2 implements new backend format for multipart
-// uploads. Delete the previous multipart directory.
-func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath string) error {
-	version, err := formatFSGetVersion(wlk)
-	if err != nil {
-		return err
-	}
-
-	if version != formatFSVersionV1 {
-		return fmt.Errorf(`format.json version expected %s, found %s`, formatFSVersionV1, version)
-	}
-
-	if err = fsRemoveAll(ctx, path.Join(fsPath, minioMetaMultipartBucket)); err != nil {
-		return err
-	}
-
-	if err = os.MkdirAll(path.Join(fsPath, minioMetaMultipartBucket), 0o755); err != nil {
-		return err
-	}
-
-	formatV1 := formatFSV1{}
-	if err = jsonLoad(wlk, &formatV1); err != nil {
-		return err
-	}
-
-	formatV2 := formatFSV2{}
-	formatV2.formatMetaV1 = formatV1.formatMetaV1
-	formatV2.FS.Version = formatFSVersionV2
-
-	return jsonSave(wlk.File, formatV2)
-}
-
-// Migrate the "fs" backend.
-// Migration should happen when formatFSV1.FS.Version changes. This version
-// can change when there is a change to the struct formatFSV1.FS or if there
-// is any change in the backend file system tree structure.
-func formatFSMigrate(ctx context.Context, wlk *lock.LockedFile, fsPath string) error {
-	// Add any migration code here in case we bump format.FS.Version
-	version, err := formatFSGetVersion(wlk)
-	if err != nil {
-		return err
-	}
-
-	switch version {
-	case formatFSVersionV1:
-		if err = formatFSMigrateV1ToV2(ctx, wlk, fsPath); err != nil {
-			return err
-		}
-		fallthrough
-	case formatFSVersionV2:
-		// We are at the latest version.
-	}
-
-	// Make sure that the version is what we expect after the migration.
-	version, err = formatFSGetVersion(wlk)
-	if err != nil {
-		return err
-	}
-	if version != formatFSVersionV2 {
-		return config.ErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version))
-	}
-	return nil
-}
-
-// Creates a new format.json if unformatted.
-func createFormatFS(fsFormatPath string) error {
-	// Attempt a write lock on formatConfigFile `format.json`
-	// file stored in minioMetaBucket(.minio.sys) directory.
-	lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o666)
-	if err != nil {
-		return err
-	}
-	// Close the locked file upon return.
-	defer lk.Close()
-
-	fi, err := lk.Stat()
-	if err != nil {
-		return err
-	}
-	if fi.Size() != 0 {
-		// format.json already got created because of another minio process's createFormatFS()
-		return nil
-	}
-
-	return jsonSave(lk.File, newFormatFSV1())
-}
-
-// This function returns a read-locked format.json reference to the caller.
-// The file descriptor should be kept open throughout the life
-// of the process so that another minio process does not try to
-// migrate the backend when we are actively working on the backend.
-func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, err error) {
-	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
-
-	// Add a deployment ID, if it does not exist.
-	if err := formatFSFixDeploymentID(ctx, fsFormatPath); err != nil {
-		return nil, err
-	}
-
-	// Any read on format.json should be done with read-lock.
-	// Any write on format.json should be done with write-lock.
-	for {
-		isEmpty := false
-		rlk, err := lock.RLockedOpenFile(fsFormatPath)
-		if err == nil {
-			// format.json can be empty in a rare condition when another
-			// minio process just created the file but could not hold lock
-			// and write to it.
-			var fi os.FileInfo
-			fi, err = rlk.Stat()
-			if err != nil {
-				return nil, err
-			}
-			isEmpty = fi.Size() == 0
-		}
-		if osIsNotExist(err) || isEmpty {
-			if err == nil {
-				rlk.Close()
-			}
-			// Fresh disk - create format.json
-			err = createFormatFS(fsFormatPath)
-			if err == lock.ErrAlreadyLocked {
-				// Lock already present, sleep and attempt again.
-				// Can happen in a rare situation when a parallel minio process
-				// holds the lock and creates format.json
-				time.Sleep(100 * time.Millisecond)
-				continue
-			}
-			if err != nil {
-				return nil, err
-			}
-			// After successfully creating format.json try to hold a read-lock on
-			// the file.
-			continue
-		}
-		if err != nil {
-			return nil, err
-		}
-
-		formatBackend, err := formatMetaGetFormatBackendFS(rlk)
-		if err != nil {
-			return nil, err
-		}
-		if formatBackend == formatBackendErasureSingle {
-			return nil, errFreshDisk
-		}
-		if formatBackend != formatBackendFS {
-			return nil, fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
-		}
-		version, err := formatFSGetVersion(rlk)
-		if err != nil {
-			return nil, err
-		}
-		if version != formatFSVersionV2 {
-			// Format needs migration
-			rlk.Close()
-			// Hold write lock during migration so that we do not disturb any
-			// minio processes running in parallel.
-			var wlk *lock.LockedFile
-			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0o666)
-			if err == lock.ErrAlreadyLocked {
-				// Lock already present, sleep and attempt again.
-				time.Sleep(100 * time.Millisecond)
-				continue
-			}
-			if err != nil {
-				return nil, err
-			}
-			err = formatFSMigrate(ctx, wlk, fsPath)
-			wlk.Close()
-			if err != nil {
-				// Migration failed, bail out so that the user can observe what happened.
-				return nil, err
-			}
-			// Successfully migrated, now try to hold a read-lock on format.json
-			continue
-		}
-		var id string
-		if id, err = formatFSGetDeploymentID(rlk); err != nil {
-			rlk.Close()
-			return nil, err
-		}
-		globalDeploymentID = id
-		return rlk, nil
-	}
-}
-
-func formatFSGetDeploymentID(rlk *lock.RLockedFile) (id string, err error) {
-	format := &formatFS{}
-	if err := jsonLoad(rlk, format); err != nil {
-		return "", err
-	}
-	return format.ID, nil
-}
-
-// Generate a deployment ID if one does not exist already.
-func formatFSFixDeploymentID(ctx context.Context, fsFormatPath string) error {
-	rlk, err := lock.RLockedOpenFile(fsFormatPath)
-	if err == nil {
-		// format.json can be empty in a rare condition when another
-		// minio process just created the file but could not hold lock
-		// and write to it.
-		var fi os.FileInfo
-		fi, err = rlk.Stat()
-		if err != nil {
-			rlk.Close()
-			return err
-		}
-		if fi.Size() == 0 {
-			rlk.Close()
-			return nil
-		}
-	}
-	if osIsNotExist(err) {
-		return nil
-	}
-	if err != nil {
-		return err
-	}
-
-	formatBackend, err := formatMetaGetFormatBackendFS(rlk)
-	if err != nil {
-		rlk.Close()
-		return err
-	}
-	if formatBackend == formatBackendErasureSingle {
-		rlk.Close()
-		return errFreshDisk
-	}
-	if formatBackend != formatBackendFS {
-		rlk.Close()
-		return fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
-	}
-
-	format := &formatFS{}
-	err = jsonLoad(rlk, format)
-	rlk.Close()
-	if err != nil {
-		return err
-	}
-
-	// Check if it needs to be updated
-	if format.ID != "" {
-		return nil
-	}
-
-	formatStartTime := time.Now().Round(time.Second)
-	getElapsedTime := func() string {
-		return time.Now().Round(time.Second).Sub(formatStartTime).String()
-	}
-
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-	var wlk *lock.LockedFile
-	var stop bool
-	for !stop {
-		select {
-		case <-ctx.Done():
-			return fmt.Errorf("Initializing FS format stopped gracefully")
-		default:
-			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0o666)
-			if err == lock.ErrAlreadyLocked {
-				// Lock already present, sleep and attempt again
-				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
-				time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
-				continue
-			}
-			if err != nil {
-				return err
-			}
-		}
-		stop = true
-	}
-	defer wlk.Close()
-
-	if err = jsonLoad(wlk, format); err != nil {
-		return err
-	}
-
-	// Check if format needs to be updated
-	if format.ID != "" {
-		return nil
-	}
-
-	// Set new UUID to the format and save it
-	format.ID = mustGetUUID()
-	return jsonSave(wlk, format)
-}

cmd/format-fs_test.go

@@ -1,104 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-// TestFSFormatFS - tests initFormatFS, formatMetaGetFormatBackendFS, formatFSGetVersion.
-func TestFSFormatFS(t *testing.T) {
-	// Prepare for testing
-	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer os.RemoveAll(disk)
-
-	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
-
-	// Assign a new UUID.
-	uuid := mustGetUUID()
-
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolumeFS(disk, uuid); err != nil {
-		t.Fatal(err)
-	}
-
-	rlk, err := initFormatFS(context.Background(), disk)
-	if err != nil {
-		t.Fatal(err)
-	}
-	rlk.Close()
-
-	// Do the basic sanity checks to check if initFormatFS() did its job.
-	f, err := os.OpenFile(fsFormatPath, os.O_RDWR|os.O_SYNC, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer f.Close()
-
-	format, err := formatMetaGetFormatBackendFS(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if format != formatBackendFS {
-		t.Fatalf(`expected: %s, got: %s`, formatBackendFS, format)
-	}
-	version, err := formatFSGetVersion(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if version != formatFSVersionV2 {
-		t.Fatalf(`expected: %s, got: %s`, formatFSVersionV2, version)
-	}
-
-	// Corrupt the format.json file and test the functions.
-	// formatMetaGetFormatBackendFS, formatFSGetVersion, initFormatFS should return errors.
-	if err = f.Truncate(0); err != nil {
-		t.Fatal(err)
-	}
-	if _, err = f.WriteString("b"); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, err = formatMetaGetFormatBackendFS(f); err == nil {
-		t.Fatal("expected to fail")
-	}
-	if _, err = formatFSGetVersion(rlk); err == nil {
-		t.Fatal("expected to fail")
-	}
-	if _, err = initFormatFS(context.Background(), disk); err == nil {
-		t.Fatal("expected to fail")
-	}
-
-	// With unknown formatMetaV1.Version formatMetaGetFormatBackendFS, initFormatFS should return error.
-	if err = f.Truncate(0); err != nil {
-		t.Fatal(err)
-	}
-	// Here we set formatMetaV1.Version to "2"
-	if _, err = f.WriteString(`{"version":"2","format":"fs","fs":{"version":"1"}}`); err != nil {
-		t.Fatal(err)
-	}
-	if _, err = formatMetaGetFormatBackendFS(f); err == nil {
-		t.Fatal("expected to fail")
-	}
-	if _, err = initFormatFS(context.Background(), disk); err == nil {
-		t.Fatal("expected to fail")
-	}
-}

cmd/fs-tree-walk-pool.go

@@ -1,212 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"errors"
-	"reflect"
-	"sync"
-	"time"
-)
-
-// Global lookup timeout.
-const (
-	globalLookupTimeout    = time.Minute * 30 // 30minutes.
-	treeWalkEntryLimit     = 50
-	treeWalkSameEntryLimit = 4
-)
-
-// listParams - list object params used for list object map
-type listParams struct {
-	bucket    string
-	recursive bool
-	marker    string
-	prefix    string
-}
-
-// errWalkAbort - returned by doTreeWalk() if it returns prematurely.
-// doTreeWalk() can return prematurely if
-// 1) treeWalk is timed out by the timer go-routine.
-// 2) there is an error during tree walk.
-var errWalkAbort = errors.New("treeWalk abort")
-
-// treeWalk - represents the go routine that does the file tree walk.
-type treeWalk struct {
-	added      time.Time
-	resultCh   chan TreeWalkResult
-	endWalkCh  chan struct{}   // To signal when treeWalk go-routine should end.
-	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
-}
-
-// TreeWalkPool - pool of treeWalk go routines.
-// A treeWalk is added to the pool by Set() and removed either by
-// doing a Release() or if the concerned timer goes off.
-// treeWalkPool's purpose is to maintain active treeWalk go-routines in a map so that
-// it can be looked up across related list calls.
-type TreeWalkPool struct {
-	mu      sync.Mutex
-	pool    map[listParams][]treeWalk
-	timeOut time.Duration
-}
-
-// NewTreeWalkPool - initialize new tree walk pool.
-func NewTreeWalkPool(timeout time.Duration) *TreeWalkPool {
-	tPool := &TreeWalkPool{
-		pool:    make(map[listParams][]treeWalk),
-		timeOut: timeout,
-	}
-	return tPool
-}
-
-// Release - selects a treeWalk from the pool based on the input
-// listParams, removes it from the pool, and returns the TreeWalkResult
-// channel.
-// Returns nil if listParams does not have an associated treeWalk.
-func (t *TreeWalkPool) Release(params listParams) (resultCh chan TreeWalkResult, endWalkCh chan struct{}) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	walks, ok := t.pool[params] // Pick the valid walks.
-	if !ok || len(walks) == 0 {
-		// Release return nil if params not found.
-		return nil, nil
-	}
-
-	// Pop out the first valid walk entry.
-	walk := walks[0]
-	walks = walks[1:]
-	if len(walks) > 0 {
-		t.pool[params] = walks
-	} else {
-		delete(t.pool, params)
-	}
-	walk.endTimerCh <- struct{}{}
-	return walk.resultCh, walk.endWalkCh
-}
-
-// Set - adds a treeWalk to the treeWalkPool.
-// Also starts a timer go-routine that ends when:
-//  1. time.After() expires after t.timeOut seconds.
-//     The expiration is needed so that the treeWalk go-routine resources are freed after a timeout
-//     if the S3 client does only partial listing of objects.
-//  2. Release() signals the timer go-routine to end on endTimerCh.
-//     During listing the timer should not timeout and end the treeWalk go-routine, hence the
-//     timer go-routine should be ended.
-func (t *TreeWalkPool) Set(params listParams, resultCh chan TreeWalkResult, endWalkCh chan struct{}) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	// If we are above the limit delete at least one entry from the pool.
-	if len(t.pool) > treeWalkEntryLimit {
-		age := time.Now()
-		var oldest listParams
-		for k, v := range t.pool {
-			if len(v) == 0 {
-				delete(t.pool, k)
-				continue
-			}
-			// The first element is the oldest, so we only check that.
-			e := v[0]
-			if e.added.Before(age) {
-				oldest = k
-				age = e.added
-			}
-		}
-		// Invalidate and delete oldest.
-		if walks, ok := t.pool[oldest]; ok && len(walks) > 0 {
-			endCh := walks[0].endTimerCh
-			endWalkCh := walks[0].endWalkCh
-			if len(walks) > 1 {
-				// Move walks forward
-				copy(walks, walks[1:])
-				walks = walks[:len(walks)-1]
-				t.pool[oldest] = walks
-			} else {
-				// Only entry, just delete.
-				delete(t.pool, oldest)
-			}
-			select {
-			case endCh <- struct{}{}:
-				close(endWalkCh)
-			default:
-			}
-		} else {
-			// Shouldn't happen, but just in case.
-			delete(t.pool, oldest)
-		}
-	}
-
-	// Should be a buffered channel so that Release() never blocks.
-	endTimerCh := make(chan struct{}, 1)
-	walkInfo := treeWalk{
-		added:      UTCNow(),
-		resultCh:   resultCh,
-		endWalkCh:  endWalkCh,
-		endTimerCh: endTimerCh,
-	}
-
-	// Append new walk info.
-	walks := t.pool[params]
-	if len(walks) < treeWalkSameEntryLimit {
-		t.pool[params] = append(walks, walkInfo)
-	} else {
-		// We are at limit, invalidate oldest, move list down and add new as last.
-		select {
-		case walks[0].endTimerCh <- struct{}{}:
-			close(walks[0].endWalkCh)
-		default:
-		}
-		copy(walks, walks[1:])
-		walks[len(walks)-1] = walkInfo
-	}
-
-	// Timer go-routine which times out after t.timeOut seconds.
-	go func(endTimerCh <-chan struct{}, walkInfo treeWalk) {
-		select {
-		// Wait until timeOut
-		case <-time.After(t.timeOut):
-			// Timeout has expired. Remove the treeWalk from treeWalkPool and
-			// end the treeWalk go-routine.
-			t.mu.Lock()
-			defer t.mu.Unlock()
-			walks, ok := t.pool[params]
-			if ok {
-				// Trick of filtering without allocating
-				// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
-				nwalks := walks[:0]
-				// Look for walkInfo, remove it from the walks list.
-				for _, walk := range walks {
-					if !reflect.DeepEqual(walk, walkInfo) {
-						nwalks = append(nwalks, walk)
-					}
-				}
-				if len(nwalks) == 0 {
-					// No more treeWalk go-routines associated with listParams
-					// hence remove map entry.
-					delete(t.pool, params)
-				} else {
-					// There are more treeWalk go-routines associated with listParams
-					// hence save the list in the map.
-					t.pool[params] = nwalks
-				}
-			}
-			// Signal the treeWalk go-routine to die.
-			close(endWalkCh)
-		case <-endTimerCh:
-			return
-		}
-	}(endTimerCh, walkInfo)
-}

cmd/fs-tree-walk-pool_test.go

@@ -1,148 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"testing"
-	"time"
-)
-
-// Test if tree walker go-routine is removed from the pool after timeout
-// and that is available in the pool before the timeout.
-func TestTreeWalkPoolBasic(t *testing.T) {
-	// Create a treeWalkPool
-	tw := NewTreeWalkPool(1 * time.Second)
-
-	// Create sample params
-	params := listParams{
-		bucket: "test-bucket",
-	}
-
-	// Add a treeWalk to the pool
-	resultCh := make(chan TreeWalkResult)
-	endWalkCh := make(chan struct{})
-	tw.Set(params, resultCh, endWalkCh)
-
-	// Wait for treeWalkPool timeout to happen
-	<-time.After(2 * time.Second)
-	if c1, _ := tw.Release(params); c1 != nil {
-		t.Error("treeWalk go-routine must have been freed")
-	}
-
-	// Add the treeWalk back to the pool
-	tw.Set(params, resultCh, endWalkCh)
-
-	// Release the treeWalk before timeout
-	select {
-	case <-time.After(1 * time.Second):
-		break
-	default:
-		if c1, _ := tw.Release(params); c1 == nil {
-			t.Error("treeWalk go-routine got freed before timeout")
-		}
-	}
-}
-
-// Test if multiple tree walkers for the same listParams are managed as expected by the pool.
-func TestManyWalksSameParam(t *testing.T) {
-	// Create a treeWalkPool.
-	tw := NewTreeWalkPool(5 * time.Second)
-
-	// Create sample params.
-	params := listParams{
-		bucket: "test-bucket",
-	}
-
-	select {
-	// This timeout is an upper-bound. This is started
-	// before the first treeWalk go-routine's timeout period starts.
-	case <-time.After(5 * time.Second):
-		break
-	default:
-		// Create many treeWalk go-routines for the same params.
-		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			resultCh := make(chan TreeWalkResult)
-			endWalkCh := make(chan struct{})
-			tw.Set(params, resultCh, endWalkCh)
-		}
-
-		tw.mu.Lock()
-		if walks, ok := tw.pool[params]; ok {
-			if len(walks) != treeWalkSameEntryLimit {
-				t.Error("There aren't as many walks as were Set")
-			}
-		}
-		tw.mu.Unlock()
-		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			tw.mu.Lock()
-			if walks, ok := tw.pool[params]; ok {
-				// Before ith Release we should have n-i treeWalk go-routines.
-				if treeWalkSameEntryLimit-i != len(walks) {
-					t.Error("There aren't as many walks as were Set")
-				}
-			}
-			tw.mu.Unlock()
-			tw.Release(params)
-		}
-	}
-}
-
-// Test if multiple tree walkers for the same listParams are managed as expected by the pool
-// but that treeWalkSameEntryLimit is respected.
-func TestManyWalksSameParamPrune(t *testing.T) {
-	// Create a treeWalkPool.
-	tw := NewTreeWalkPool(5 * time.Second)
-
-	// Create sample params.
-	params := listParams{
-		bucket: "test-bucket",
-	}
-
-	select {
-	// This timeout is an upper-bound. This is started
-	// before the first treeWalk go-routine's timeout period starts.
-	case <-time.After(5 * time.Second):
-		break
-	default:
-		// Create many treeWalk go-routines for the same params.
-		for i := 0; i < treeWalkSameEntryLimit*4; i++ {
-			resultCh := make(chan TreeWalkResult)
-			endWalkCh := make(chan struct{})
-			tw.Set(params, resultCh, endWalkCh)
-		}
-
-		tw.mu.Lock()
-		if walks, ok := tw.pool[params]; ok {
-			if len(walks) != treeWalkSameEntryLimit {
-				t.Error("There aren't as many walks as were Set")
-			}
-		}
-		tw.mu.Unlock()
-		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			tw.mu.Lock()
-			if walks, ok := tw.pool[params]; ok {
-				// Before ith Release we should have n-i treeWalk go-routines.
-				if treeWalkSameEntryLimit-i != len(walks) {
-					t.Error("There aren't as many walks as were Set")
-				}
-			}
-			tw.mu.Unlock()
-			tw.Release(params)
-		}
-	}
-}

cmd/fs-v1-helpers.go

@@ -1,481 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"io"
-	"os"
-	pathutil "path"
-	"runtime"
-	"strings"
-	"time"
-
-	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/lock"
-	"github.com/minio/minio/internal/logger"
-)
-
-// Removes only the file at given path does not remove
-// any parent directories, handles long paths for
-// windows automatically.
-func fsRemoveFile(ctx context.Context, filePath string) (err error) {
-	if filePath == "" {
-		logger.LogIf(ctx, errInvalidArgument)
-		return errInvalidArgument
-	}
-
-	if err = checkPathLength(filePath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if err = os.Remove(filePath); err != nil {
-		if err = osErrToFileErr(err); err != errFileNotFound {
-			logger.LogIf(ctx, err)
-		}
-	}
-
-	return err
-}
-
-// Removes all files and folders at a given path, handles
-// long paths for windows automatically.
-func fsRemoveAll(ctx context.Context, dirPath string) (err error) {
-	if dirPath == "" {
-		logger.LogIf(ctx, errInvalidArgument)
-		return errInvalidArgument
-	}
-
-	if err = checkPathLength(dirPath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if err = removeAll(dirPath); err != nil {
-		if osIsPermission(err) {
-			logger.LogIf(ctx, errVolumeAccessDenied)
-			return errVolumeAccessDenied
-		} else if isSysErrNotEmpty(err) {
-			logger.LogIf(ctx, errVolumeNotEmpty)
-			return errVolumeNotEmpty
-		}
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	return nil
-}
-
-// Removes a directory only if its empty, handles long
-// paths for windows automatically.
-func fsRemoveDir(ctx context.Context, dirPath string) (err error) {
-	if dirPath == "" {
-		logger.LogIf(ctx, errInvalidArgument)
-		return errInvalidArgument
-	}
-
-	if err = checkPathLength(dirPath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if err = os.Remove((dirPath)); err != nil {
-		if osIsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrNotEmpty(err) {
-			return errVolumeNotEmpty
-		}
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	return nil
-}
-
-// Creates a new directory, parent dir should exist
-// otherwise returns an error. If directory already
-// exists returns an error. Windows long paths
-// are handled automatically.
-func fsMkdir(ctx context.Context, dirPath string) (err error) {
-	if dirPath == "" {
-		logger.LogIf(ctx, errInvalidArgument)
-		return errInvalidArgument
-	}
-
-	if err = checkPathLength(dirPath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if err = os.Mkdir((dirPath), 0o777); err != nil {
-		switch {
-		case osIsExist(err):
-			return errVolumeExists
-		case osIsPermission(err):
-			logger.LogIf(ctx, errDiskAccessDenied)
-			return errDiskAccessDenied
-		case isSysErrNotDir(err):
-			// File path cannot be verified since
-			// one of the parents is a file.
-			logger.LogIf(ctx, errDiskAccessDenied)
-			return errDiskAccessDenied
-		case isSysErrPathNotFound(err):
-			// Add specific case for windows.
-			logger.LogIf(ctx, errDiskAccessDenied)
-			return errDiskAccessDenied
-		default:
-			logger.LogIf(ctx, err)
-			return err
-		}
-	}
-
-	return nil
-}
-
-// fsStat is a low level call which validates input arguments
-// and checks input length upto supported maximum. Does
-// not perform any higher layer interpretation of files v/s
-// directories. For higher level interpretation look at
-// fsStatFileDir, fsStatFile, fsStatDir.
-func fsStat(ctx context.Context, statLoc string) (os.FileInfo, error) {
-	if statLoc == "" {
-		logger.LogIf(ctx, errInvalidArgument)
-		return nil, errInvalidArgument
-	}
-	if err := checkPathLength(statLoc); err != nil {
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-	fi, err := os.Stat(statLoc)
-	if err != nil {
-		return nil, err
-	}
-
-	return fi, nil
-}
-
-// fsTouch updates a file access & modtime with current time
-func fsTouch(ctx context.Context, statLoc string) error {
-	if statLoc == "" {
-		logger.LogIf(ctx, errInvalidArgument)
-		return errInvalidArgument
-	}
-	if err := checkPathLength(statLoc); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	now := time.Now()
-	if err := os.Chtimes(statLoc, now, now); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	return nil
-}
-
-// Lookup if volume exists, returns volume attributes upon success.
-func fsStatVolume(ctx context.Context, volume string) (os.FileInfo, error) {
-	fi, err := fsStat(ctx, volume)
-	if err != nil {
-		if osIsNotExist(err) {
-			return nil, errVolumeNotFound
-		} else if osIsPermission(err) {
-			return nil, errVolumeAccessDenied
-		}
-		return nil, err
-	}
-
-	if !fi.IsDir() {
-		return nil, errVolumeAccessDenied
-	}
-
-	return fi, nil
-}
-
-// Lookup if directory exists, returns directory attributes upon success.
-func fsStatDir(ctx context.Context, statDir string) (os.FileInfo, error) {
-	fi, err := fsStat(ctx, statDir)
-	if err != nil {
-		err = osErrToFileErr(err)
-		if err != errFileNotFound {
-			logger.LogIf(ctx, err)
-		}
-		return nil, err
-	}
-	if !fi.IsDir() {
-		return nil, errFileNotFound
-	}
-	return fi, nil
-}
-
-// Lookup if file exists, returns file attributes upon success.
-func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) {
-	fi, err := fsStat(ctx, statFile)
-	if err != nil {
-		err = osErrToFileErr(err)
-		if err != errFileNotFound {
-			logger.LogIf(ctx, err)
-		}
-		return nil, err
-	}
-	if fi.IsDir() {
-		return nil, errFileNotFound
-	}
-	return fi, nil
-}
-
-// Returns if the filePath is a regular file.
-func fsIsFile(ctx context.Context, filePath string) bool {
-	fi, err := fsStat(ctx, filePath)
-	if err != nil {
-		return false
-	}
-	return fi.Mode().IsRegular()
-}
-
-// Opens the file at given path, optionally from an offset. Upon success returns
-// a readable stream and the size of the readable stream.
-func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadCloser, int64, error) {
-	if readPath == "" || offset < 0 {
-		logger.LogIf(ctx, errInvalidArgument)
-		return nil, 0, errInvalidArgument
-	}
-	if err := checkPathLength(readPath); err != nil {
-		logger.LogIf(ctx, err)
-		return nil, 0, err
-	}
-
-	fr, err := os.Open(readPath)
-	if err != nil {
-		return nil, 0, osErrToFileErr(err)
-	}
-
-	// Stat to get the size of the file at path.
-	st, err := fr.Stat()
-	if err != nil {
-		fr.Close()
-		err = osErrToFileErr(err)
-		if err != errFileNotFound {
-			logger.LogIf(ctx, err)
-		}
-		return nil, 0, err
-	}
-
-	// Verify if its not a regular file, since subsequent Seek is undefined.
-	if !st.Mode().IsRegular() {
-		fr.Close()
-		return nil, 0, errIsNotRegular
-	}
-
-	// Seek to the requested offset.
-	if offset > 0 {
-		_, err = fr.Seek(offset, io.SeekStart)
-		if err != nil {
-			fr.Close()
-			logger.LogIf(ctx, err)
-			return nil, 0, err
-		}
-	}
-
-	// Success.
-	return fr, st.Size(), nil
-}
-
-// Creates a file and copies data from incoming reader.
-func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, fallocSize int64) (int64, error) {
-	if filePath == "" || reader == nil {
-		logger.LogIf(ctx, errInvalidArgument)
-		return 0, errInvalidArgument
-	}
-
-	if err := checkPathLength(filePath); err != nil {
-		logger.LogIf(ctx, err)
-		return 0, err
-	}
-
-	if err := mkdirAll(pathutil.Dir(filePath), 0o777); err != nil {
-		switch {
-		case osIsPermission(err):
-			return 0, errFileAccessDenied
-		case osIsExist(err):
-			return 0, errFileAccessDenied
-		case isSysErrIO(err):
-			return 0, errFaultyDisk
-		case isSysErrInvalidArg(err):
-			return 0, errUnsupportedDisk
-		case isSysErrNoSpace(err):
-			return 0, errDiskFull
-		}
-		return 0, err
-	}
-
-	flags := os.O_CREATE | os.O_WRONLY
-	if globalFSOSync {
-		flags |= os.O_SYNC
-	}
-	writer, err := lock.Open(filePath, flags, 0o666)
-	if err != nil {
-		return 0, osErrToFileErr(err)
-	}
-	defer writer.Close()
-
-	bytesWritten, err := xioutil.Copy(writer, reader)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return 0, err
-	}
-
-	return bytesWritten, nil
-}
-
-// Renames source path to destination path, creates all the
-// missing parents if they don't exist.
-func fsRenameFile(ctx context.Context, sourcePath, destPath string) error {
-	if err := checkPathLength(sourcePath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	if err := checkPathLength(destPath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if err := renameAll(sourcePath, destPath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	return nil
-}
-
-func deleteFile(basePath, deletePath string, recursive bool) error {
-	if basePath == "" || deletePath == "" {
-		return nil
-	}
-	isObjectDir := HasSuffix(deletePath, SlashSeparator)
-	basePath = pathutil.Clean(basePath)
-	deletePath = pathutil.Clean(deletePath)
-	if !strings.HasPrefix(deletePath, basePath) || deletePath == basePath {
-		return nil
-	}
-
-	var err error
-	if recursive {
-		os.RemoveAll(deletePath)
-	} else {
-		err = os.Remove(deletePath)
-	}
-	if err != nil {
-		switch {
-		case isSysErrNotEmpty(err):
-			// if object is a directory, but if its not empty
-			// return FileNotFound to indicate its an empty prefix.
-			if isObjectDir {
-				return errFileNotFound
-			}
-			// Ignore errors if the directory is not empty. The server relies on
-			// this functionality, and sometimes uses recursion that should not
-			// error on parent directories.
-			return nil
-		case osIsNotExist(err):
-			return errFileNotFound
-		case osIsPermission(err):
-			return errFileAccessDenied
-		case isSysErrIO(err):
-			return errFaultyDisk
-		default:
-			return err
-		}
-	}
-
-	deletePath = pathutil.Dir(deletePath)
-
-	// Delete parent directory obviously not recursively. Errors for
-	// parent directories shouldn't trickle down.
-	deleteFile(basePath, deletePath, false)
-
-	return nil
-}
-
-// fsDeleteFile is a wrapper for deleteFile(), after checking the path length.
-func fsDeleteFile(ctx context.Context, basePath, deletePath string) error {
-	if err := checkPathLength(basePath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if err := checkPathLength(deletePath); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if err := deleteFile(basePath, deletePath, false); err != nil {
-		if err != errFileNotFound {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	return nil
-}
-
-// fsRemoveMeta safely removes a locked file and takes care of Windows special case
-func fsRemoveMeta(ctx context.Context, basePath, deletePath, tmpDir string) error {
-	// Special case for windows please read through.
-	if runtime.GOOS == globalWindowsOSName {
-		// Ordinarily windows does not permit deletion or renaming of files still
-		// in use, but if all open handles to that file were opened with FILE_SHARE_DELETE
-		// then it can permit renames and deletions of open files.
-		//
-		// There are however some gotchas with this, and it is worth listing them here.
-		// Firstly, Windows never allows you to really delete an open file, rather it is
-		// flagged as delete pending and its entry in its directory remains visible
-		// (though no new file handles may be opened to it) and when the very last
-		// open handle to the file in the system is closed, only then is it truly
-		// deleted. Well, actually only sort of truly deleted, because Windows only
-		// appears to remove the file entry from the directory, but in fact that
-		// entry is merely hidden and actually still exists and attempting to create
-		// a file with the same name will return an access denied error. How long it
-		// silently exists for depends on a range of factors, but put it this way:
-		// if your code loops creating and deleting the same file name as you might
-		// when operating a lock file, you're going to see lots of random spurious
-		// access denied errors and truly dismal lock file performance compared to POSIX.
-		//
-		// We work-around these un-POSIX file semantics by taking a dual step to
-		// deleting files. Firstly, it renames the file to tmp location into multipartTmpBucket
-		// We always open files with FILE_SHARE_DELETE permission enabled, with that
-		// flag Windows permits renaming and deletion, and because the name was changed
-		// to a very random name somewhere not in its origin directory before deletion,
-		// you don't see those unexpected random errors when creating files with the
-		// same name as a recently deleted file as you do anywhere else on Windows.
-		// Because the file is probably not in its original containing directory any more,
-		// deletions of that directory will not fail with "directory not empty" as they
-		// otherwise normally would either.
-
-		tmpPath := pathJoin(tmpDir, mustGetUUID())
-
-		fsRenameFile(ctx, deletePath, tmpPath)
-
-		// Proceed to deleting the directory if empty
-		fsDeleteFile(ctx, basePath, pathutil.Dir(deletePath))
-
-		// Finally delete the renamed file.
-		return fsDeleteFile(ctx, tmpDir, tmpPath)
-	}
-	return fsDeleteFile(ctx, basePath, deletePath)
-}

cmd/fs-v1-helpers_test.go

@@ -1,549 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"io"
-	"os"
-	"path"
-	"testing"
-
-	"github.com/minio/minio/internal/lock"
-)
-
-func TestFSRenameFile(t *testing.T) {
-	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup(t)
-	if err != nil {
-		t.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-
-	if err = fsMkdir(GlobalContext, pathJoin(path, "testvolume1")); err != nil {
-		t.Fatal(err)
-	}
-	if err = fsRenameFile(GlobalContext, pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
-		t.Fatal(err)
-	}
-	if err = fsRenameFile(GlobalContext, pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != errFileNotFound {
-		t.Fatal(err)
-	}
-	if err = fsRenameFile(GlobalContext, pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); err != errFileNameTooLong {
-		t.Fatal("Unexpected error", err)
-	}
-	if err = fsRenameFile(GlobalContext, pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); err != errFileNameTooLong {
-		t.Fatal("Unexpected error", err)
-	}
-}
-
-func TestFSStats(t *testing.T) {
-	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup(t)
-	if err != nil {
-		t.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-
-	// Setup test environment.
-
-	if err = fsMkdir(GlobalContext, ""); err != errInvalidArgument {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if err = fsMkdir(GlobalContext, pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); err != errFileNameTooLong {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
-		t.Fatalf("Unable to create volume, %s", err)
-	}
-
-	reader := bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
-		t.Fatalf("Unable to create file, %s", err)
-	}
-	// Seek back.
-	reader.Seek(0, 0)
-
-	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "success-file")); err != errVolumeExists {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "path/to/success-file"), reader, 0); err != nil {
-		t.Fatalf("Unable to create file, %s", err)
-	}
-	// Seek back.
-	reader.Seek(0, 0)
-
-	testCases := []struct {
-		srcFSPath   string
-		srcVol      string
-		srcPath     string
-		expectedErr error
-	}{
-		// Test case - 1.
-		// Test case with valid inputs, expected to pass.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			expectedErr: nil,
-		},
-		// Test case - 2.
-		// Test case with valid inputs, expected to pass.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "path/to/success-file",
-			expectedErr: nil,
-		},
-		// Test case - 3.
-		// Test case with non-existent file.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "nonexistent-file",
-			expectedErr: errFileNotFound,
-		},
-		// Test case - 4.
-		// Test case with non-existent file path.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "path/2/success-file",
-			expectedErr: errFileNotFound,
-		},
-		// Test case - 5.
-		// Test case with path being a directory.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "path",
-			expectedErr: errFileNotFound,
-		},
-		// Test case - 6.
-		// Test case with src path segment > 255.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			expectedErr: errFileNameTooLong,
-		},
-		// Test case - 7.
-		// Test case validate only srcVol exists.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			expectedErr: nil,
-		},
-		// Test case - 8.
-		// Test case validate only srcVol doesn't exist.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol-non-existent",
-			expectedErr: errVolumeNotFound,
-		},
-		// Test case - 9.
-		// Test case validate invalid argument.
-		{
-			expectedErr: errInvalidArgument,
-		},
-	}
-
-	for i, testCase := range testCases {
-		if testCase.srcPath != "" {
-			if _, err := fsStatFile(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol,
-				testCase.srcPath)); err != testCase.expectedErr {
-				t.Fatalf("TestErasureStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
-			}
-		} else {
-			if _, err := fsStatVolume(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol)); err != testCase.expectedErr {
-				t.Fatalf("TestFS case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
-			}
-		}
-	}
-}
-
-func TestFSCreateAndOpen(t *testing.T) {
-	// Setup test environment.
-	_, path, err := newXLStorageTestSetup(t)
-	if err != nil {
-		t.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-
-	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
-		t.Fatalf("Unable to create directory, %s", err)
-	}
-
-	if _, err = fsCreateFile(GlobalContext, "", nil, 0); err != errInvalidArgument {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if _, _, err = fsOpenFile(GlobalContext, "", -1); err != errInvalidArgument {
-		t.Fatal("Unexpected error", err)
-	}
-
-	reader := bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
-		t.Fatalf("Unable to create file, %s", err)
-	}
-	// Seek back.
-	reader.Seek(0, 0)
-
-	testCases := []struct {
-		srcVol      string
-		srcPath     string
-		expectedErr error
-	}{
-		// Test case - 1.
-		// Test case with segment of the volume name > 255.
-		{
-			srcVol:      "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			srcPath:     "success-file",
-			expectedErr: errFileNameTooLong,
-		},
-		// Test case - 2.
-		// Test case with src path segment > 255.
-		{
-			srcVol:      "success-vol",
-			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			expectedErr: errFileNameTooLong,
-		},
-	}
-
-	for i, testCase := range testCases {
-		_, err = fsCreateFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), reader, 0)
-		if err != testCase.expectedErr {
-			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
-		}
-		_, _, err = fsOpenFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), 0)
-		if err != testCase.expectedErr {
-			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
-		}
-	}
-
-	// Attempt to open a directory.
-	if _, _, err = fsOpenFile(GlobalContext, pathJoin(path), 0); err != errIsNotRegular {
-		t.Fatal("Unexpected error", err)
-	}
-}
-
-func TestFSDeletes(t *testing.T) {
-	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup(t)
-	if err != nil {
-		t.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-
-	// Setup test environment.
-	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
-		t.Fatalf("Unable to create directory, %s", err)
-	}
-
-	reader := bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, reader.Size()); err != nil {
-		t.Fatalf("Unable to create file, %s", err)
-	}
-	// Seek back.
-	reader.Seek(0, io.SeekStart)
-
-	// folder is not empty
-	err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "not-empty"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = os.WriteFile(pathJoin(path, "success-vol", "not-empty", "file"), []byte("data"), 0o777)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// recursive
-	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "parent")); err != nil {
-		t.Fatal(err)
-	}
-	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "parent", "dir")); err != nil {
-		t.Fatal(err)
-	}
-
-	testCases := []struct {
-		basePath    string
-		srcVol      string
-		srcPath     string
-		expectedErr error
-	}{
-		// valid case with existing volume and file to delete.
-		{
-			basePath:    path,
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			expectedErr: nil,
-		},
-		// The file was deleted in the last case, so Delete should fail.
-		{
-			basePath:    path,
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			expectedErr: errFileNotFound,
-		},
-		// Test case with segment of the volume name > 255.
-		{
-			basePath:    path,
-			srcVol:      "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			srcPath:     "success-file",
-			expectedErr: errFileNameTooLong,
-		},
-		// Test case with src path segment > 255.
-		{
-			basePath:    path,
-			srcVol:      "success-vol",
-			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			expectedErr: errFileNameTooLong,
-		},
-		// Base path is way too long.
-		{
-			basePath:    "path03333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333",
-			srcVol:      "success-vol",
-			srcPath:     "object",
-			expectedErr: errFileNameTooLong,
-		},
-		// Directory is not empty. Should give nil, but won't delete.
-		{
-			basePath:    path,
-			srcVol:      "success-vol",
-			srcPath:     "not-empty",
-			expectedErr: nil,
-		},
-		// Should delete recursively.
-		{
-			basePath:    path,
-			srcVol:      "success-vol",
-			srcPath:     pathJoin("parent", "dir"),
-			expectedErr: nil,
-		},
-	}
-
-	for i, testCase := range testCases {
-		if err = fsDeleteFile(GlobalContext, testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
-			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
-		}
-	}
-}
-
-func BenchmarkFSDeleteFile(b *testing.B) {
-	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup(b)
-	if err != nil {
-		b.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-
-	// Setup test environment.
-	if err = fsMkdir(GlobalContext, pathJoin(path, "benchmark")); err != nil {
-		b.Fatalf("Unable to create directory, %s", err)
-	}
-
-	benchDir := pathJoin(path, "benchmark")
-	filename := pathJoin(benchDir, "file.txt")
-
-	b.ResetTimer()
-	// We need to create and delete the file sequentially inside the benchmark.
-	for i := 0; i < b.N; i++ {
-		b.StopTimer()
-		err = os.WriteFile(filename, []byte("data"), 0o777)
-		if err != nil {
-			b.Fatal(err)
-		}
-		b.StartTimer()
-
-		err = fsDeleteFile(GlobalContext, benchDir, filename)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-// Tests fs removes.
-func TestFSRemoves(t *testing.T) {
-	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup(t)
-	if err != nil {
-		t.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-
-	// Setup test environment.
-	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
-		t.Fatalf("Unable to create directory, %s", err)
-	}
-
-	reader := bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
-		t.Fatalf("Unable to create file, %s", err)
-	}
-	// Seek back.
-	reader.Seek(0, 0)
-
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file-new"), reader, 0); err != nil {
-		t.Fatalf("Unable to create file, %s", err)
-	}
-	// Seek back.
-	reader.Seek(0, 0)
-
-	testCases := []struct {
-		srcFSPath   string
-		srcVol      string
-		srcPath     string
-		expectedErr error
-	}{
-		// Test case - 1.
-		// valid case with existing volume and file to delete.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			expectedErr: nil,
-		},
-		// Test case - 2.
-		// The file was deleted in the last case, so Delete should fail.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			expectedErr: errFileNotFound,
-		},
-		// Test case - 3.
-		// Test case with segment of the volume name > 255.
-		{
-			srcFSPath:   path,
-			srcVol:      "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			srcPath:     "success-file",
-			expectedErr: errFileNameTooLong,
-		},
-		// Test case - 4.
-		// Test case with src path segment > 255.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			expectedErr: errFileNameTooLong,
-		},
-		// Test case - 5.
-		// Test case with src path empty.
-		{
-			srcFSPath:   path,
-			srcVol:      "success-vol",
-			expectedErr: errVolumeNotEmpty,
-		},
-		// Test case - 6.
-		// Test case with src path empty.
-		{
-			srcFSPath:   path,
-			srcVol:      "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			expectedErr: errFileNameTooLong,
-		},
-		// Test case - 7.
-		// Test case with src path empty.
-		{
-			srcFSPath:   path,
-			srcVol:      "non-existent",
-			expectedErr: errVolumeNotFound,
-		},
-		// Test case - 8.
-		// Test case with src and volume path empty.
-		{
-			expectedErr: errInvalidArgument,
-		},
-	}
-
-	for i, testCase := range testCases {
-		if testCase.srcPath != "" {
-			if err = fsRemoveFile(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
-				t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
-			}
-		} else {
-			if err = fsRemoveDir(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
-				t.Error(err)
-			}
-		}
-	}
-
-	if err = fsRemoveAll(GlobalContext, pathJoin(path, "success-vol")); err != nil {
-		t.Fatal(err)
-	}
-
-	if err = fsRemoveAll(GlobalContext, ""); err != errInvalidArgument {
-		t.Fatal(err)
-	}
-
-	if err = fsRemoveAll(GlobalContext, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); err != errFileNameTooLong {
-		t.Fatal(err)
-	}
-}
-
-func TestFSRemoveMeta(t *testing.T) {
-	// create xlStorage test setup
-	_, fsPath, err := newXLStorageTestSetup(t)
-	if err != nil {
-		t.Fatalf("Unable to create xlStorage test setup, %s", err)
-	}
-
-	// Setup test environment.
-	if err = fsMkdir(GlobalContext, pathJoin(fsPath, "success-vol")); err != nil {
-		t.Fatalf("Unable to create directory, %s", err)
-	}
-
-	filePath := pathJoin(fsPath, "success-vol", "success-file")
-
-	reader := bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, filePath, reader, 0); err != nil {
-		t.Fatalf("Unable to create file, %s", err)
-	}
-
-	rwPool := &fsIOPool{
-		readersMap: make(map[string]*lock.RLockedFile),
-	}
-
-	if _, err := rwPool.Open(filePath); err != nil {
-		t.Fatalf("Unable to lock file %s", filePath)
-	}
-
-	defer rwPool.Close(filePath)
-
-	tmpDir := t.TempDir()
-
-	if err := fsRemoveMeta(GlobalContext, fsPath, filePath, tmpDir); err != nil {
-		t.Fatalf("Unable to remove file, %s", err)
-	}
-
-	if _, err := os.Stat((filePath)); !osIsNotExist(err) {
-		t.Fatalf("`%s` file found though it should have been deleted.", filePath)
-	}
-
-	if _, err := os.Stat((path.Dir(filePath))); !osIsNotExist(err) {
-		t.Fatalf("`%s` parent directory found though it should have been deleted.", filePath)
-	}
-}
-
-func TestFSIsFile(t *testing.T) {
-	filePath := pathJoin(t.TempDir(), "tmpfile")
-
-	if err := os.WriteFile(filePath, nil, 0o777); err != nil {
-		t.Fatalf("Unable to create file %s", filePath)
-	}
-
-	if !fsIsFile(GlobalContext, filePath) {
-		t.Fatalf("Expected %s to be a file", filePath)
-	}
-}