update docs for Decommission

concurrent loading of erasure sets can now expose a
situation in a distributed setup that might return
diskID as empty, treat such disks as offline.

.github/ISSUE_TEMPLATE.md

@@ -7,6 +7,9 @@ assignees: ''
 
 ---
 
+## NOTE
+If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.
+
 <!--- Provide a general summary of the issue in the Title above -->
 
 ## Expected Behavior

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,9 @@ assignees: ''
 
 ---
 
+## NOTE
+If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.
+
 <!--- Provide a general summary of the issue in the Title above -->
 
 ## Expected Behavior

.github/lock.yml

@@ -1,39 +0,0 @@
-# Configuration for Lock Threads - https://github.com/dessant/lock-threads-app
-
-# Number of days of inactivity before a closed issue or pull request is locked
-daysUntilLock: 365
-
-# Skip issues and pull requests created before a given timestamp. Timestamp must
-# follow ISO 8601 (`YYYY-MM-DD`). Set to `false` to disable
-skipCreatedBefore: false
-
-# Issues and pull requests with these labels will be ignored. Set to `[]` to disable
-exemptLabels: []
-
-# Label to add before locking, such as `outdated`. Set to `false` to disable
-lockLabel: true
-
-# Comment to post before locking. Set to `false` to disable
-lockComment: >-
-
-  This thread has been automatically locked since there has not been
-  any recent activity after it was closed. Please open a new issue for
-  related bugs.
-
-# Assign `resolved` as the reason for locking. Set to `false` to disable
-setLockReason: true
-
-# Limit to only `issues` or `pulls`
-only: issues
-
-# Optionally, specify configuration settings just for `issues` or `pulls`
-# issues:
-#   exemptLabels:
-#     - help-wanted
-#   lockLabel: outdated
-
-# pulls:
-#   daysUntilLock: 30
-
-# Repository to extend settings from
-# _extends: repo

.github/stale.yml

@@ -14,7 +14,7 @@ onlyLabels: []
 exemptLabels:
   - "security"
   - "pending discussion"
-  - "do not close"
+  - "do-not-close"
 
 # Set to true to ignore issues in a project (defaults to false)
 exemptProjects: false

.github/workflows/iam-integrations.yaml

@@ -88,4 +88,8 @@ jobs:
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |
-          make test-site-replication
+          make test-site-replication-ldap
+      - name: Test OIDC for automatic site replication
+        if: matrix.openid == 'http://127.0.0.1:5556/dex'
+        run: |
+          make test-site-replication-oidc

.github/workflows/lock.yml

@@ -0,0 +1,25 @@
+name: 'Lock Threads'
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+
+concurrency:
+  group: lock
+
+jobs:
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/lock-threads@v3
+        with:
+          github-token: ${{ github.token }}
+          issue-inactive-days: '730'
+          exclude-any-issue-labels: 'do-not-close'
+          issue-lock-reason: 'resolved'
+          process-only: 'issues'
+          log-output: false

.github/workflows/replication.yaml

@@ -38,3 +38,9 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-replication
+      - name: Test MinIO IDP for automatic site replication
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-site-replication-minio
+

Makefile

@@ -59,9 +59,17 @@ test-replication: install ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
 
-test-site-replication: install ## verify automatic site replication
-	@echo "Running tests for automatic site replication of IAM"
-	@(env bash $(PWD)/docs/site-replication/run-multi-site.sh)
+test-site-replication-ldap: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with LDAP)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
+
+test-site-replication-oidc: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with OIDC)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
+
+test-site-replication-minio: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"

README.md

@@ -227,21 +227,25 @@ For example: `export MINIO_SERVER_URL="https://minio.example.net"`
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
 
 # Upgrading MinIO
-MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster simultaneously and restart them, as shown in the following command from the MinIO client (mc):
+Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all transactions on MinIO are atomic. So upgrading all the servers simultaneously is the recommended way to upgrade MinIO.
 
+> NOTE: requires internet access to update directly from https://dl.min.io, optionally you can host any mirrors at https://my-artifactory.example.com/minio/
+
+- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/minio/baremetal/reference/minio-mc-admin/mc-admin-update.html)
 ```
 mc admin update <minio alias, e.g., myminio>
 ```
 
-> NOTE: some releases might not allow rolling upgrades, this is always called out in the release notes and it is generally advised to read release notes before upgrading. In such a situation `mc admin update` is the recommended upgrading mechanism to upgrade all servers at once.
+- For deployments without external internet access (e.g. airgapped environments), download the binary from https://dl.min.io and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and do `mc admin service restart alias/`.
 
-## Important things to remember during MinIO upgrades
+- For RPM/DEB installations, upgrade packages **parallelly** on all servers. Once upgraded, perform `systemctl restart minio` across all nodes in **parallel**. RPM/DEB based installations are usually automated using [`ansible`](https://github.com/minio/ansible-minio).
 
-- Container environments are advised to update the container images instead of updating binaries inside the container.
-- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
-- `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
-- `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
-- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
+## Upgrade Checklist
+- Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
+- Read the release notes for the targeted MinIO release *before* performing any installation, there is no forced requirement to upgrade to latest releases every week. If it has a bug fix you are looking for then yes, else avoid actively upgrading a running production system.
+- Make sure MinIO process has write access to `/opt/bin` if you plan to use `mc admin update`. This is needed for MinIO to download the latest binary from https://dl.min.io and save it locally for upgrades.
+- `mc admin update` is not supported in kubernetes/container environments, container environments provide their own mechanisms for container updates.
+- **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
 
 # Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)

buildscripts/race.sh

@@ -2,6 +2,7 @@
 
 set -e
 
-for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=1 go test -v -tags kqueue -race --timeout 100m "$d"
+## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
+for d in $(go list ./... | grep -v dsync); do
+    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done

cmd/admin-bucket-handlers.go

@@ -65,7 +65,8 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	if _, err = parseBucketQuota(bucket, data); err != nil {
+	quotaConfig, err := parseBucketQuota(bucket, data)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -75,6 +76,21 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
+	bucketMeta := madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeQuotaConfig,
+		Bucket: bucket,
+		Quota:  data,
+	}
+	if quotaConfig.Quota == 0 {
+		bucketMeta.Quota = nil
+	}
+
+	// Call site replication hook.
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/admin-handler-utils.go

@@ -88,8 +88,26 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 		}
 	case SRError:
 		apiErr = errorCodes.ToAPIErrWithErr(e.Code, e.Cause)
+	case decomError:
+		apiErr = APIError{
+			Code:           "XMinioDecommissionNotAllowed",
+			Description:    e.Err,
+			HTTPStatusCode: http.StatusBadRequest,
+		}
 	default:
 		switch {
+		case errors.Is(err, errDecommissionAlreadyRunning):
+			apiErr = APIError{
+				Code:           "XMinioDecommissionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, errDecommissionComplete):
+			apiErr = APIError{
+				Code:           "XMinioDecommissionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errConfigNotFound):
 			apiErr = APIError{
 				Code:           "XMinioConfigError",
@@ -102,6 +120,18 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusForbidden,
 			}
+		case errors.Is(err, errIAMServiceAccount):
+			apiErr = APIError{
+				Code:           "XMinioIAMServiceAccount",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, errIAMServiceAccountUsed):
+			apiErr = APIError{
+				Code:           "XMinioIAMServiceAccountUsed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errIAMNotInitialized):
 			apiErr = APIError{
 				Code:           "XMinioIAMNotInitialized",

cmd/admin-handlers-pools.go

@@ -0,0 +1,179 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "StartDecommission")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	v := vars["pool"]
+
+	idx := globalEndpoints.GetPoolIdx(v)
+	if idx == -1 {
+		// We didn't find any matching pools, invalid input
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	if err := pools.Decommission(r.Context(), idx); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "CancelDecommission")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	v := vars["pool"]
+
+	idx := globalEndpoints.GetPoolIdx(v)
+	if idx == -1 {
+		// We didn't find any matching pools, invalid input
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	if err := pools.DecommissionCancel(ctx, idx); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "StatusPool")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	v := vars["pool"]
+
+	idx := globalEndpoints.GetPoolIdx(v)
+	if idx == -1 {
+		// We didn't find any matching pools, invalid input
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	status, err := pools.Status(r.Context(), idx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(&status))
+}
+
+func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListPools")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	poolsStatus := make([]PoolStatus, len(globalEndpoints))
+	for idx := range globalEndpoints {
+		status, err := pools.Status(r.Context(), idx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		poolsStatus[idx] = status
+	}
+
+	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
+}

cmd/admin-handlers-site-replication.go

@@ -45,8 +45,7 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	}
 
 	var sites []madmin.PeerSite
-	err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey)
-	if err != nil {
+	if err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -226,6 +225,20 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
 			}
 		}
+	case madmin.SRBucketMetaTypeQuotaConfig:
+		if item.Quota == nil {
+			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil)
+		} else {
+			quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig); err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+		}
 	case madmin.SRBucketMetaTypeTags:
 		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
 	case madmin.SRBucketMetaTypeObjectLockConfig:
@@ -360,3 +373,58 @@ func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http
 		return
 	}
 }
+
+// SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit
+func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationEdit")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+	var site madmin.PeerInfo
+	err := parseJSONBody(ctx, r.Body, &site, cred.SecretKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	body, err := json.Marshal(status)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
+//
+// used internally to tell current cluster to update endpoint for peer
+func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerEdit")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var pi madmin.PeerInfo
+	if err := parseJSONBody(ctx, r.Body, &pi, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}

cmd/admin-handlers-users.go

@@ -75,7 +75,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// ListUsers - GET /minio/admin/v3/list-users?bucket={bucket}
+// ListBucketUsers - GET /minio/admin/v3/list-users?bucket={bucket}
 func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBucketUsers")
 
@@ -330,6 +330,20 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemGroupInfo,
+		GroupInfo: &madmin.SRGroupInfo{
+			UpdateReq: madmin.GroupAddRemove{
+				Group:    group,
+				Status:   madmin.GroupStatus(status),
+				IsRemove: false,
+			},
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
@@ -347,7 +361,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
 
-	// This API is not allowed to lookup accessKey user status
+	// This API is not allowed to lookup master access key user status
 	if accessKey == globalActiveCred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
@@ -357,6 +371,20 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemIAMUser,
+		IAMUser: &madmin.SRIAMUser{
+			AccessKey:   accessKey,
+			IsDeleteReq: false,
+			UserReq: &madmin.AddOrUpdateUserReq{
+				Status: madmin.AccountStatus(status),
+			},
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
@@ -743,14 +771,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Call site replication hook. Only LDAP accounts are supported for
-	// replication operations.
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+	// Call site replication hook - non-root user accounts are replicated.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -767,6 +789,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 			return
 		}
 	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -821,18 +844,15 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 
 	var svcAccountPolicy iampolicy.Policy
 
-	impliedPolicy := policy == nil
-
-	// If policy is empty, check for policy of the parent user
-	if !impliedPolicy {
-		svcAccountPolicy = svcAccountPolicy.Merge(*policy)
+	if policy != nil {
+		svcAccountPolicy = *policy
 	} else {
 		policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, false)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		svcAccountPolicy = svcAccountPolicy.Merge(globalIAMSys.GetCombinedPolicy(policiesNames...))
+		svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
 	}
 
 	policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
@@ -844,7 +864,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
 		AccountStatus: svcAccount.Status,
-		ImpliedPolicy: impliedPolicy,
+		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
 	}
 
@@ -990,22 +1010,14 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	// Save svc acc claims before deletion (for site replication hook).
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, serviceAccount)
-	if err != nil && err != errNoSuchServiceAccount {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Call site replication hook. Only LDAP accounts are supported for
-	// replication operations.
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+	// Call site replication hook - non-root user accounts are replicated.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -1085,11 +1097,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	var err error
 	if !globalIsGateway {
 		// Load the latest calculated data usage
-		dataUsageInfo, err = loadDataUsageFromBackend(ctx, objectAPI)
-		if err != nil {
-			// log the error, continue with the accounting response
-			logger.LogIf(ctx, err)
-		}
+		dataUsageInfo, _ = loadDataUsageFromBackend(ctx, objectAPI)
 	}
 
 	// If etcd, dns federation configured list buckets from etcd.
@@ -1158,11 +1166,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
 			if enablePrefixUsage {
-				if pu, err := loadPrefixUsageFromBackend(ctx, objectAPI, bucket.Name); err == nil {
-					prefixUsage = pu
-				} else {
-					logger.LogIf(ctx, err)
-				}
+				prefixUsage, _ = loadPrefixUsageFromBackend(ctx, objectAPI, bucket.Name)
 			}
 
 			lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)

cmd/admin-router.go

@@ -74,8 +74,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
-
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(gz(httpTraceAll(adminAPI.BackgroundHealStatusHandler)))
+
+			// Pool operations
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/pools/list").HandlerFunc(gz(httpTraceAll(adminAPI.ListPools)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/pools/status").HandlerFunc(gz(httpTraceAll(adminAPI.StatusPool))).Queries("pool", "{pool:.*}")
+
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
 		}
 
 		// Profiling operations
@@ -201,6 +207,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
 		}
 
 		if globalIsDistErasure {

cmd/background-newdisks-heal-ops.go

@@ -289,20 +289,20 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
-	for _, ep := range globalEndpoints {
-		for _, endpoint := range ep.Endpoints {
-			if !endpoint.IsLocal {
-				continue
-			}
-			// Try to connect to the current endpoint
-			// and reformat if the current disk is not formatted
-			disk, _, err := connectEndpoint(endpoint)
-			if errors.Is(err, errUnformattedDisk) {
-				disksToHeal = append(disksToHeal, endpoint)
-			} else if err == nil && disk != nil && disk.Healing() != nil {
-				disksToHeal = append(disksToHeal, disk.Endpoint())
-			}
+	for _, disk := range globalLocalDrives {
+		_, err := disk.GetDiskID()
+		if errors.Is(err, errUnformattedDisk) {
+			disksToHeal = append(disksToHeal, disk.Endpoint())
+			continue
 		}
+		if disk.Healing() != nil {
+			disksToHeal = append(disksToHeal, disk.Endpoint())
+		}
+	}
+	if len(disksToHeal) == globalEndpoints.NEndpoints() {
+		// When all disks == all command line endpoints
+		// this is a fresh setup, no need to trigger healing.
+		return Endpoints{}
 	}
 	return disksToHeal
 }

cmd/bitrot.go

@@ -119,8 +119,10 @@ func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath strin
 // Close all the readers.
 func closeBitrotReaders(rs []io.ReaderAt) {
 	for _, r := range rs {
-		if br, ok := r.(io.Closer); ok {
-			br.Close()
+		if r != nil {
+			if br, ok := r.(io.Closer); ok {
+				br.Close()
+			}
 		}
 	}
 }
@@ -128,8 +130,10 @@ func closeBitrotReaders(rs []io.ReaderAt) {
 // Close all the writers.
 func closeBitrotWriters(ws []io.Writer) {
 	for _, w := range ws {
-		if bw, ok := w.(io.Closer); ok {
-			bw.Close()
+		if w != nil {
+			if bw, ok := w.(io.Closer); ok {
+				bw.Close()
+			}
 		}
 	}
 }

cmd/bootstrap-peer-server.go

@@ -108,6 +108,7 @@ var skipEnvs = map[string]struct{}{
 	"MINIO_OPTS":         {},
 	"MINIO_CERT_PASSWD":  {},
 	"MINIO_SERVER_DEBUG": {},
+	"MINIO_DSYNC_TRACE":  {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {

cmd/bucket-policy-handlers_test.go

@@ -37,60 +37,72 @@ import (
 func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
-			policy.NewResourceSet(policy.NewResource(bucketName, "")),
-			condition.NewFunctions(),
-		)},
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
+				policy.NewResourceSet(policy.NewResource(bucketName, "")),
+				condition.NewFunctions(),
+			),
+		},
 	}
 }
 
 func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(
-				policy.GetBucketLocationAction,
-				policy.ListBucketMultipartUploadsAction,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.GetBucketLocationAction,
+					policy.ListBucketMultipartUploadsAction,
+				),
+				policy.NewResourceSet(policy.NewResource(bucketName, "")),
+				condition.NewFunctions(),
 			),
-			policy.NewResourceSet(policy.NewResource(bucketName, "")),
-			condition.NewFunctions(),
-		)},
+		},
 	}
 }
 
 func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(policy.GetObjectAction),
-			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
-			condition.NewFunctions(),
-		)},
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetObjectAction),
+				policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+				condition.NewFunctions(),
+			),
+		},
 	}
 }
 
 func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(
-				policy.AbortMultipartUploadAction,
-				policy.DeleteObjectAction,
-				policy.ListMultipartUploadPartsAction,
-				policy.PutObjectAction,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.AbortMultipartUploadAction,
+					policy.DeleteObjectAction,
+					policy.ListMultipartUploadPartsAction,
+					policy.PutObjectAction,
+				),
+				policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+				condition.NewFunctions(),
 			),
-			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
-			condition.NewFunctions(),
-		)},
+		},
 	}
 }
 

cmd/bucket-replication.go

@@ -28,6 +28,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
@@ -769,7 +770,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 	}
 
 	t, _ := tags.ParseObjectTags(oi1.UserTags)
-	if !reflect.DeepEqual(oi2.UserTags, t.ToMap()) {
+	if !reflect.DeepEqual(oi2.UserTags, t.ToMap()) || (oi2.UserTagCount != len(t.ToMap())) {
 		return replicateMetadata
 	}
 
@@ -860,8 +861,9 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		return
 	}
 	tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
-		Name: object,
-		SSEC: crypto.SSEC.IsEncrypted(objInfo.UserDefined),
+		Name:     object,
+		SSEC:     crypto.SSEC.IsEncrypted(objInfo.UserDefined),
+		UserTags: objInfo.UserTags,
 	})
 	// Lock the object name before starting replication.
 	// Use separate lock that doesn't collide with regular objects.
@@ -1184,9 +1186,17 @@ func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket,
 	defer func() {
 		if err != nil {
 			// block and abort remote upload upon failure.
-			if aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID); aerr != nil {
-				aerr = fmt.Errorf("Unable to cleanup failed multipart replication %s on remote %s/%s: %w", uploadID, bucket, object, aerr)
-				logger.LogIf(ctx, aerr)
+			attempts := 1
+			for attempts <= 3 {
+				aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID)
+				if aerr == nil {
+					return
+				}
+				logger.LogIf(ctx,
+					fmt.Errorf("Trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
+						humanize.Ordinal(attempts), uploadID, bucket, object, aerr))
+				attempts++
+				time.Sleep(time.Second)
 			}
 		}
 	}()

cmd/common-main.go

@@ -40,6 +40,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	fcolor "github.com/fatih/color"
 	"github.com/go-openapi/loads"
 	"github.com/inconshreveable/mousetrap"
@@ -652,6 +653,16 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable")
 	}
 
+	if rootDiskSize := env.Get(config.EnvRootDiskThresholdSize, ""); rootDiskSize != "" {
+		size, err := humanize.ParseBytes(rootDiskSize)
+		if err != nil {
+			logger.Fatal(err, fmt.Sprintf("Invalid %s value in environment variable", config.EnvRootDiskThresholdSize))
+		}
+		globalRootDiskThreshold = size
+	}
+
+	globalIsCICD = env.Get("MINIO_CI_CD", "") != ""
+
 	domains := env.Get(config.EnvDomain, "")
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
@@ -678,7 +689,7 @@ func handleCommonEnvVars() {
 		for _, endpoint := range minioEndpoints {
 			if net.ParseIP(endpoint) == nil {
 				// Checking if the IP is a DNS entry.
-				addrs, err := net.LookupHost(endpoint)
+				addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint)
 				if err != nil {
 					logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
 				}

cmd/config.go

@@ -35,8 +35,8 @@ import (
 
 const (
 	minioConfigPrefix = "config"
-
-	kvPrefix = ".kv"
+	minioConfigBucket = minioMetaBucket + SlashSeparator + minioConfigPrefix
+	kvPrefix          = ".kv"
 
 	// Captures all the previous SetKV operations and allows rollback.
 	minioConfigHistoryPrefix = minioConfigPrefix + "/history"

cmd/data-update-tracker.go

@@ -204,14 +204,14 @@ func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
 // All of these will exit when the context is canceled.
 func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
 	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No drives specified"))
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No local drives specified"))
 		return
 	}
 	d.load(ctx, drives...)
 	go d.startCollector(ctx)
 	// startSaver will unlock.
 	d.mu.Lock()
-	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives)
+	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives...)
 }
 
 // load will attempt to load data tracking information from the supplied drives.
@@ -221,7 +221,7 @@ func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
 // If object is shared the caller should lock it.
 func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No drives specified"))
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No local drives specified"))
 		return
 	}
 	for _, drive := range drives {
@@ -246,7 +246,11 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 // startSaver will start a saver that will write d to all supplied drives at specific intervals.
 // 'd' must be write locked when started and will be unlocked.
 // The saver will save and exit when supplied context is closed.
-func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives []string) {
+func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives ...string) {
+	if len(drives) == 0 {
+		return
+	}
+
 	saveNow := d.save
 	exited := make(chan struct{})
 	d.saveExited = exited

cmd/data-usage.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
@@ -90,7 +91,7 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	buf, err := readConfig(ctx, objAPI, dataUsageObjNamePath)
 	if err != nil {
-		if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
+		if errors.Is(err, errConfigNotFound) {
 			return DataUsageInfo{}, nil
 		}
 		return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)

cmd/endpoint-ellipses.go

@@ -353,9 +353,11 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			return nil, -1, err
 		}
 		endpointServerPools = append(endpointServerPools, PoolEndpoints{
+			Legacy:       true,
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
+			CmdLine:      strings.Join(args, " "),
 		})
 		setupType = newSetupType
 		return endpointServerPools, setupType, nil
@@ -376,6 +378,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
+			CmdLine:      arg,
 		}); err != nil {
 			return nil, -1, err
 		}

cmd/endpoint.go

@@ -197,14 +197,28 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 // PoolEndpoints represent endpoints in a given pool
 // along with its setCount and setDriveCount.
 type PoolEndpoints struct {
+	// indicates if endpoints are provided in non-ellipses style
+	Legacy       bool
 	SetCount     int
 	DrivesPerSet int
 	Endpoints    Endpoints
+	CmdLine      string
 }
 
 // EndpointServerPools - list of list of endpoints
 type EndpointServerPools []PoolEndpoints
 
+// GetPoolIdx return pool index
+func (l EndpointServerPools) GetPoolIdx(pool string) int {
+	for id, ep := range globalEndpoints {
+		if ep.CmdLine != pool {
+			continue
+		}
+		return id
+	}
+	return -1
+}
+
 // GetLocalPoolIdx returns the pool which endpoint belongs to locally.
 // if ep is remote this code will return -1 poolIndex
 func (l EndpointServerPools) GetLocalPoolIdx(ep Endpoint) int {
@@ -220,6 +234,13 @@ func (l EndpointServerPools) GetLocalPoolIdx(ep Endpoint) int {
 	return -1
 }
 
+// Legacy returns 'true' if the MinIO server commandline was
+// provided with no ellipses pattern, those are considered
+// legacy deployments.
+func (l EndpointServerPools) Legacy() bool {
+	return len(l) == 1 && l[0].Legacy
+}
+
 // Add add pool endpoints
 func (l *EndpointServerPools) Add(zeps PoolEndpoints) error {
 	existSet := set.NewStringSet()
@@ -622,20 +643,33 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		}
 	}
 
-	// Check whether same path is not used in endpoints of a host on different port.
-	{
-		pathIPMap := make(map[string]set.StringSet)
-		for _, endpoint := range endpoints {
-			host := endpoint.Hostname()
-			hostIPSet, _ := getHostIP(host)
-			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
-				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					return endpoints, setupType,
-						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
+	orchestrated := IsKubernetes() || IsDocker()
+	if !orchestrated {
+		// Check whether same path is not used in endpoints of a host on different port.
+		// Only verify this on baremetal setups, DNS is not available in orchestrated
+		// environments so we can't do much here.
+		{
+			pathIPMap := make(map[string]set.StringSet)
+			hostIPCache := make(map[string]set.StringSet)
+			for _, endpoint := range endpoints {
+				host := endpoint.Hostname()
+				hostIPSet, ok := hostIPCache[host]
+				if !ok {
+					hostIPSet, err = getHostIP(host)
+					if err != nil {
+						return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+					}
+					hostIPCache[host] = hostIPSet
+				}
+				if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+					if !IPSet.Intersection(hostIPSet).IsEmpty() {
+						return endpoints, setupType,
+							config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
+					}
+					pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+				} else {
+					pathIPMap[endpoint.Path] = hostIPSet
 				}
-				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
-			} else {
-				pathIPMap[endpoint.Path] = hostIPSet
 			}
 		}
 	}

cmd/erasure-common.go

@@ -22,6 +22,38 @@ import (
 	"sync"
 )
 
+func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
+	disks := er.getDisks()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if disks[i-1] == nil {
+				return
+			}
+			di, err := disks[i-1].DiskInfo(context.Background())
+			if err != nil || di.Healing {
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				// - Do not consume disks which are being healed
+				//
+				// - Future: skip busy disks
+				return
+			}
+
+			mu.Lock()
+			newDisks = append(newDisks, disks[i-1])
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+	return newDisks
+}
+
 func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	// Based on the random shuffling return back randomized disks.

cmd/erasure-decode.go

@@ -322,8 +322,8 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 			errs:        make([]error, len(writers)),
 		}
 
-		err = w.Write(ctx, bufs)
-		if err != nil {
+		if err = w.Write(ctx, bufs); err != nil {
+			logger.LogIf(ctx, err)
 			return err
 		}
 	}

cmd/erasure-encode.go

@@ -52,7 +52,10 @@ func (p *parallelWriter) Write(ctx context.Context, blocks [][]byte) error {
 			if p.errs[i] == nil {
 				if n != len(blocks[i]) {
 					p.errs[i] = io.ErrShortWrite
+					p.writers[i] = nil
 				}
+			} else {
+				p.writers[i] = nil
 			}
 		}(i)
 	}
@@ -93,6 +96,7 @@ func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer
 		blocks, err = e.EncodeData(ctx, buf[:n])
 		if err != nil {
 			logger.LogIf(ctx, err)
+
 			return 0, err
 		}
 

cmd/erasure-healing.go

@@ -246,7 +246,7 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
-	case errors.Is(erErr, errCorruptedFormat):
+	case errors.Is(erErr, errFileCorrupt):
 		return true
 	}
 	if erErr == nil {
@@ -312,13 +312,13 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
-		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
+		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, nil, opts)
 	}
 
 	result.ParityBlocks = result.DiskCount - readQuorum
 	result.DataBlocks = readQuorum
 
-	// List of disks having latest version of the object er.meta
+	// List of disks having latest version of the object xl.meta
 	// (by modtime).
 	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
@@ -356,10 +356,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			// If data is sane on any one disk, we can
 			// extract the correct object size.
 			result.ObjectSize = partsMetadata[i].Size
-			if partsMetadata[i].Erasure.ParityBlocks > 0 && partsMetadata[i].Erasure.DataBlocks > 0 {
-				result.ParityBlocks = partsMetadata[i].Erasure.ParityBlocks
-				result.DataBlocks = partsMetadata[i].Erasure.DataBlocks
-			}
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
 		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
@@ -406,7 +402,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
-	if numAvailableDisks < result.DataBlocks {
+	if numAvailableDisks < readQuorum {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, dataErrs, opts)
 	}
 
@@ -466,9 +462,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	copyPartsMetadata = shufflePartsMetadata(copyPartsMetadata, latestMeta.Erasure.Distribution)
 
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		result.DataBlocks = latestMeta.Erasure.DataBlocks
-		result.ParityBlocks = latestMeta.Erasure.ParityBlocks
-
 		// Heal each part. erasureHealFile() will write the healed
 		// part to .minio/tmp/uuid/ which needs to be renamed later to
 		// the final location.
@@ -547,14 +540,14 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 			// If all disks are having errors, we give up.
 			if disksToHealCount == 0 {
-				return result, fmt.Errorf("all disks had write errors, unable to heal")
+				return result, fmt.Errorf("all disks had write errors, unable to heal %s/%s", bucket, object)
 			}
 
 		}
 
 	}
 
-	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpID, len(storageDisks)/2+1)
+	defer er.renameAll(context.Background(), minioMetaTmpBucket, tmpID)
 
 	// Rename from tmp location to the actual location.
 	for i, disk := range outDatedDisks {
@@ -819,21 +812,20 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 	// remove we simply delete it from namespace.
 	m, ok := isObjectDangling(metaArr, errs, dataErrs)
 	if ok {
-		parityBlocks := m.Erasure.ParityBlocks
-		if m.Erasure.ParityBlocks == 0 {
-			parityBlocks = er.defaultParityCount
-		}
-		dataBlocks := m.Erasure.DataBlocks
-		if m.Erasure.DataBlocks == 0 {
-			dataBlocks = len(storageDisks) - parityBlocks
+		parityBlocks := er.defaultParityCount
+		dataBlocks := len(storageDisks) - parityBlocks
+		if m.IsValid() {
+			parityBlocks = m.Erasure.ParityBlocks
+			dataBlocks = m.Erasure.DataBlocks
 		}
+
 		writeQuorum := dataBlocks
 		if dataBlocks == parityBlocks {
 			writeQuorum++
 		}
+
 		var err error
-		var returnNotFound bool
-		if !opts.DryRun && opts.Remove {
+		if opts.Remove {
 			err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{
 				VersionID: versionID,
 			}, false)
@@ -849,19 +841,10 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 				m.Size = 0
 			}
 
-			// Delete successfully purged dangling content, return ObjectNotFound/VersionNotFound instead.
-			if countErrs(errs, nil) == len(errs) {
-				returnNotFound = true
-			}
-		}
-		if returnNotFound {
-			err = toObjectErr(errFileNotFound, bucket, object)
-			if versionID != "" {
-				err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-			}
-			return er.defaultHealResult(m, storageDisks, storageEndpoints,
-				errs, bucket, object, versionID), err
+			return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+				errs, bucket, object, versionID), nil
 		}
+
 		return er.defaultHealResult(m, storageDisks, storageEndpoints,
 			errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID)
 	}
@@ -878,54 +861,64 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 // files is lesser than number of data blocks.
 func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (validMeta FileInfo, ok bool) {
 	// We can consider an object data not reliable
-	// when er.meta is not found in read quorum disks.
-	// or when er.meta is not readable in read quorum disks.
-	var notFoundErasureMeta, corruptedErasureMeta int
-	for _, readErr := range errs {
-		if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
-			notFoundErasureMeta++
-		} else if errors.Is(readErr, errCorruptedFormat) {
-			corruptedErasureMeta++
-		}
-	}
-	var notFoundParts int
-	for i := range dataErrs {
-		// Only count part errors, if the error is not
-		// same as er.meta error. This is to avoid
-		// double counting when both parts and er.meta
-		// are not available.
-		if errs[i] != dataErrs[i] {
-			if IsErr(dataErrs[i], []error{
-				errFileNotFound,
-				errFileVersionNotFound,
-			}...) {
-				notFoundParts++
+	// when xl.meta is not found in read quorum disks.
+	// or when xl.meta is not readable in read quorum disks.
+	danglingErrsCount := func(cerrs []error) (int, int) {
+		var (
+			notFoundCount  int
+			corruptedCount int
+		)
+		for _, readErr := range cerrs {
+			if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
+				notFoundCount++
+			} else if errors.Is(readErr, errFileCorrupt) {
+				corruptedCount++
 			}
 		}
+		return notFoundCount, corruptedCount
 	}
 
+	ndataErrs := make([]error, len(dataErrs))
+	for i := range dataErrs {
+		if errs[i] != dataErrs[i] {
+			// Only count part errors, if the error is not
+			// same as xl.meta error. This is to avoid
+			// double counting when both parts and xl.meta
+			// are not available.
+			ndataErrs[i] = dataErrs[i]
+		}
+	}
+
+	notFoundMetaErrs, corruptedMetaErrs := danglingErrsCount(errs)
+	notFoundPartsErrs, corruptedPartsErrs := danglingErrsCount(ndataErrs)
+
 	for _, m := range metaArr {
-		if !m.IsValid() {
-			continue
+		if m.IsValid() {
+			validMeta = m
+			break
 		}
-		validMeta = m
-		break
 	}
 
-	if validMeta.Deleted || validMeta.IsRemote() {
-		// notFoundParts is ignored since a
+	if !validMeta.IsValid() {
+		// We have no idea what this file is, leave it as is.
+		return validMeta, false
+	}
+
+	if validMeta.Deleted {
+		// notFoundPartsErrs is ignored since
 		// - delete marker does not have any parts
-		// - transition status of complete has no parts
-		return validMeta, corruptedErasureMeta+notFoundErasureMeta > len(errs)/2
+		return validMeta, corruptedMetaErrs+notFoundMetaErrs > len(errs)/2
 	}
 
-	// We couldn't find any valid meta we are indeed corrupted, return true right away.
-	if validMeta.Erasure.DataBlocks == 0 {
-		return validMeta, true
+	totalErrs := notFoundMetaErrs + corruptedMetaErrs + notFoundPartsErrs + corruptedPartsErrs
+	if validMeta.IsRemote() {
+		// notFoundPartsErrs is ignored since
+		// - transition status of complete has no parts
+		totalErrs = notFoundMetaErrs + corruptedMetaErrs
 	}
 
 	// We have valid meta, now verify if we have enough files with parity blocks.
-	return validMeta, corruptedErasureMeta+notFoundErasureMeta+notFoundParts > validMeta.Erasure.ParityBlocks
+	return validMeta, totalErrs > validMeta.Erasure.ParityBlocks
 }
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.

cmd/erasure-healing_test.go

@@ -34,6 +34,192 @@ import (
 	"github.com/minio/madmin-go"
 )
 
+// Tests isObjectDangling function
+func TestIsObjectDangling(t *testing.T) {
+	fi := newFileInfo("test-object", 2, 2)
+	fi.Erasure.Index = 1
+
+	testCases := []struct {
+		name             string
+		metaArr          []FileInfo
+		errs             []error
+		dataErrs         []error
+		expectedMeta     FileInfo
+		expectedDangling bool
+	}{
+		{
+			name: "FileInfoExists-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				fi,
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				nil,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoExists-case2",
+			metaArr: []FileInfo{
+				{},
+				{},
+				fi,
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoUndecided-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				errDiskNotFound,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name:    "FileInfoUndecided-case2",
+			metaArr: []FileInfo{},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				errDiskNotFound,
+				errFileNotFound,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{},
+			expectedDangling: false,
+		},
+		{
+			name:    "FileInfoUndecided-case3(file deleted)",
+			metaArr: []FileInfo{},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{},
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoDecided-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2-delete-marker",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				{Deleted: true},
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{Deleted: true},
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2-(duplicate data errors)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				{Deleted: true},
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				nil,
+				nil,
+			},
+			expectedMeta:     FileInfo{Deleted: true},
+			expectedDangling: true,
+		},
+		// Add new cases as seen
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.name, func(t *testing.T) {
+			gotMeta, dangling := isObjectDangling(testCase.metaArr, testCase.errs, testCase.dataErrs)
+			if !gotMeta.Equals(testCase.expectedMeta) {
+				t.Errorf("Expected %#v, got %#v", testCase.expectedMeta, gotMeta)
+			}
+			if dangling != testCase.expectedDangling {
+				t.Errorf("Expected dangling %t, got %t", testCase.expectedDangling, dangling)
+			}
+		})
+	}
+}
+
 // Tests both object and bucket healing.
 func TestHealing(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())

cmd/erasure-multipart.go

@@ -29,6 +29,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -392,6 +393,53 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 	return partInfo, nil
 }
 
+func undoRenamePart(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, errs []error) {
+	// Undo rename object on disks where RenameFile succeeded.
+	g := errgroup.WithNErrs(len(disks))
+	for index, disk := range disks {
+		if disk == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			if errs[index] == nil {
+				_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
+			}
+			return nil
+		}, index)
+	}
+	g.Wait()
+}
+
+// renamePart - renames multipart part to its relevant location under uploadID.
+func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
+	g := errgroup.WithNErrs(len(disks))
+
+	// Rename file on all underlying storage disks.
+	for index := range disks {
+		index := index
+		g.Go(func() error {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			return disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry)
+		}, index)
+	}
+
+	// Wait for all renames to finish.
+	errs := g.Wait()
+
+	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
+	// otherwise return failure. Cleanup successful renames.
+	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	if err == errErasureWriteQuorum {
+		// Undo all the partial rename operations.
+		undoRenamePart(disks, srcBucket, srcEntry, dstBucket, dstEntry, errs)
+	}
+
+	return evalDisks(disks, errs), err
+}
+
 // PutObjectPart - reads incoming stream and internally erasure codes
 // them. This call is similar to single put operation but it is part
 // of the multipart transaction.
@@ -481,7 +529,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.deleteObject(context.Background(), minioMetaTmpBucket, tmpPart, writeQuorum)
+			er.renameAll(context.Background(), minioMetaTmpBucket, tmpPart)
 		}
 	}()
 
@@ -521,7 +569,22 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
-	n, err := erasure.Encode(pctx, data, writers, buffer, writeQuorum)
+	toEncode := io.Reader(data)
+	if data.Size() > bigFileThreshold {
+		// Add input readahead.
+		// We use 2 buffers, so we always have a full buffer of input.
+		bufA := er.bp.Get()
+		bufB := er.bp.Get()
+		defer er.bp.Put(bufA)
+		defer er.bp.Put(bufB)
+		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
+		if err == nil {
+			toEncode = ra
+			defer ra.Close()
+		}
+	}
+
+	n, err := erasure.Encode(pctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
@@ -555,7 +618,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	// Rename temporary part file to its final location.
 	partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix)
-	onlineDisks, err = rename(wctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, false, writeQuorum, nil)
+	onlineDisks, err = renamePart(wctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
@@ -978,21 +1041,8 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 		return toObjectErr(err, bucket, object, uploadID)
 	}
 
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-
-	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "", false)
-
-	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
-	if err != nil {
-		return toObjectErr(err, bucket, object, uploadID)
-	}
-
 	// Cleanup all uploaded parts.
-	if err = er.deleteObject(ctx, minioMetaMultipartBucket, uploadIDPath, writeQuorum); err != nil {
-		return toObjectErr(err, bucket, object, uploadID)
-	}
+	er.renameAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
 
 	// Successfully purged.
 	return nil

cmd/erasure-object.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/klauspost/readahead"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -408,7 +409,8 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}
 
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
-		if reducedErr == errErasureReadQuorum && bucket != minioMetaBucket {
+		if errors.Is(reducedErr, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+			// Skip buckets that live at `.minio.sys` bucket.
 			if _, ok := isObjectDangling(metaArr, errs, nil); ok {
 				reducedErr = errFileNotFound
 				if opts.VersionID != "" {
@@ -522,31 +524,6 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 	return objInfo, wquorum, nil
 }
 
-func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, errs []error) {
-	// Undo rename object on disks where RenameFile succeeded.
-
-	// If srcEntry/dstEntry are objects then add a trailing slash to copy
-	// over all the parts inside the object directory
-	if isDir {
-		srcEntry = retainSlash(srcEntry)
-		dstEntry = retainSlash(dstEntry)
-	}
-	g := errgroup.WithNErrs(len(disks))
-	for index, disk := range disks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			if errs[index] == nil {
-				_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
-			}
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
 func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
 	defer NSUpdated(dstBucket, dstEntry)
@@ -587,46 +564,6 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 	return evalDisks(disks, errs), err
 }
 
-// rename - common function that renamePart and renameObject use to rename
-// the respective underlying storage layer representations.
-func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
-	if isDir {
-		dstEntry = retainSlash(dstEntry)
-		srcEntry = retainSlash(srcEntry)
-	}
-	defer NSUpdated(dstBucket, dstEntry)
-
-	g := errgroup.WithNErrs(len(disks))
-
-	// Rename file on all underlying storage disks.
-	for index := range disks {
-		index := index
-		g.Go(func() error {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			if err := disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
-				if !IsErrIgnored(err, ignoredErr...) {
-					return err
-				}
-			}
-			return nil
-		}, index)
-	}
-
-	// Wait for all renames to finish.
-	errs := g.Wait()
-
-	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
-	// otherwise return failure. Cleanup successful renames.
-	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if err == errErasureWriteQuorum {
-		// Undo all the partial rename operations.
-		undoRename(disks, srcBucket, srcEntry, dstBucket, dstEntry, isDir, errs)
-	}
-	return evalDisks(disks, errs), err
-}
-
 func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	data := r.Reader
 
@@ -700,9 +637,10 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		if disk == nil {
 			continue
 		}
-
-		inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
-		writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
+		if disk.IsOnline() {
+			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
+			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
+		}
 	}
 
 	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
@@ -719,6 +657,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 
 	for i, w := range writers {
 		if w == nil {
+			// Make sure to avoid writing to disks which we couldn't complete in erasure.Encode()
 			onlineDisks[i] = nil
 			continue
 		}
@@ -901,7 +840,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
+			er.renameAll(context.Background(), minioMetaTmpBucket, tempObj)
 		}
 	}()
 
@@ -929,6 +868,10 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			continue
 		}
 
+		if !disk.IsOnline() {
+			continue
+		}
+
 		if len(inlineBuffers) > 0 {
 			sz := shardFileSize
 			if sz < 0 {
@@ -938,10 +881,25 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 			continue
 		}
+
 		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
-	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
+	toEncode := io.Reader(data)
+	if data.Size() > bigFileThreshold {
+		// We use 2 buffers, so we always have a full buffer of input.
+		bufA := er.bp.Get()
+		bufB := er.bp.Get()
+		defer er.bp.Put(bufA)
+		defer er.bp.Put(bufB)
+		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
+		if err == nil {
+			toEncode = ra
+			defer ra.Close()
+		}
+		logger.LogIf(ctx, err)
+	}
+	n, erasureErr := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if erasureErr != nil {
 		return ObjectInfo{}, toObjectErr(erasureErr, minioMetaTmpBucket, tempErasureObj)
@@ -1011,6 +969,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// Rename the successfully written temporary object to final location.
 	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum); err != nil {
+		if errors.Is(err, errFileNotFound) {
+			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
+		}
 		logger.LogIf(ctx, err)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -1059,42 +1020,6 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
 }
 
-// deleteObject - wrapper for delete object, deletes an object from
-// all the disks in parallel, including `xl.meta` associated with the
-// object.
-func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string, writeQuorum int) error {
-	var err error
-
-	disks := er.getDisks()
-	tmpObj := mustGetUUID()
-	if bucket == minioMetaTmpBucket {
-		tmpObj = object
-	} else {
-		// Rename the current object while requiring write quorum, but also consider
-		// that a non found object in a given disk as a success since it already
-		// confirms that the object doesn't have a part in that disk (already removed)
-		disks, err = rename(ctx, disks, bucket, object, minioMetaTmpBucket, tmpObj, true, writeQuorum,
-			[]error{errFileNotFound})
-		if err != nil {
-			return toObjectErr(err, bucket, object)
-		}
-	}
-
-	g := errgroup.WithNErrs(len(disks))
-	for index := range disks {
-		index := index
-		g.Go(func() error {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			return disks[index].Delete(ctx, minioMetaTmpBucket, tmpObj, true)
-		}, index)
-	}
-
-	// return errors if any during deletion
-	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
-}
-
 // DeleteObjects deletes objects/versions in bulk, this function will still automatically split objects list
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
@@ -1572,6 +1497,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
 
 	fi.Metadata[xhttp.AmzObjectTagging] = tags
+	fi.ReplicationState = opts.PutReplicationState()
 	for k, v := range opts.UserDefined {
 		fi.Metadata[k] = v
 	}

cmd/erasure-server-pool-decom.go

@@ -0,0 +1,996 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/console"
+)
+
+// PoolDecommissionInfo currently decommissioning information
+type PoolDecommissionInfo struct {
+	StartTime   time.Time `json:"startTime" msg:"st"`
+	StartSize   int64     `json:"startSize" msg:"ss"`
+	TotalSize   int64     `json:"totalSize" msg:"ts"`
+	CurrentSize int64     `json:"currentSize" msg:"cs"`
+
+	Complete bool `json:"complete" msg:"cmp"`
+	Failed   bool `json:"failed" msg:"fl"`
+	Canceled bool `json:"canceled" msg:"cnl"`
+
+	// Internal information.
+	QueuedBuckets         []string `json:"-" msg:"bkts"`
+	DecommissionedBuckets []string `json:"-" msg:"dbkts"`
+
+	// Last bucket/object decommissioned.
+	Bucket string `json:"-" msg:"bkt"`
+	Object string `json:"-" msg:"obj"`
+
+	// Verbose information
+	ItemsDecommissioned     int64 `json:"-" msg:"id"`
+	ItemsDecommissionFailed int64 `json:"-" msg:"idf"`
+	BytesDone               int64 `json:"-" msg:"bd"`
+	BytesFailed             int64 `json:"-" msg:"bf"`
+}
+
+// bucketPop should be called when a bucket is done decommissioning.
+// Adds the bucket to the list of decommissioned buckets and updates resume numbers.
+func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
+	pd.DecommissionedBuckets = append(pd.DecommissionedBuckets, bucket)
+	for i, b := range pd.QueuedBuckets {
+		if b == bucket {
+			// Bucket is done.
+			pd.QueuedBuckets = append(pd.QueuedBuckets[:i], pd.QueuedBuckets[i+1:]...)
+			// Clear tracker info.
+			if pd.Bucket == bucket {
+				pd.Bucket = "" // empty this out for next bucket
+				pd.Object = "" // empty this out for next object
+			}
+			return
+		}
+	}
+}
+
+func (pd *PoolDecommissionInfo) bucketsToDecommission() []string {
+	queuedBuckets := make([]string, len(pd.QueuedBuckets))
+	copy(queuedBuckets, pd.QueuedBuckets)
+	return queuedBuckets
+}
+
+func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
+	for _, b := range pd.DecommissionedBuckets {
+		if b == bucket {
+			return true
+		}
+	}
+	return false
+}
+
+func (pd *PoolDecommissionInfo) bucketPush(bucket string) {
+	for _, b := range pd.QueuedBuckets {
+		if pd.isBucketDecommissioned(b) {
+			return
+		}
+		if b == bucket {
+			return
+		}
+	}
+	pd.QueuedBuckets = append(pd.QueuedBuckets, bucket)
+	pd.Bucket = bucket
+}
+
+// PoolStatus captures current pool status
+type PoolStatus struct {
+	ID           int                   `json:"id" msg:"id"`
+	CmdLine      string                `json:"cmdline" msg:"cl"`
+	LastUpdate   time.Time             `json:"lastUpdate" msg:"lu"`
+	Decommission *PoolDecommissionInfo `json:"decommissionInfo,omitempty" msg:"dec"`
+}
+
+//go:generate msgp -file $GOFILE -unexported
+type poolMeta struct {
+	Version int          `msg:"v"`
+	Pools   []PoolStatus `msg:"pls"`
+}
+
+// A decommission resumable tells us if decommission is worth
+// resuming upon restart of a cluster.
+func (p *poolMeta) returnResumablePools(n int) []PoolStatus {
+	var newPools []PoolStatus
+	for _, pool := range p.Pools {
+		if pool.Decommission == nil {
+			continue
+		}
+		if pool.Decommission.Complete || pool.Decommission.Canceled {
+			// Do not resume decommission upon startup for
+			// - decommission complete
+			// - decommission canceled
+			continue
+		} // In all other situations we need to resume
+		newPools = append(newPools, pool)
+		if n > 0 && len(newPools) == n {
+			return newPools
+		}
+	}
+	return nil
+}
+
+func (p *poolMeta) DecommissionComplete(idx int) bool {
+	if p.Pools[idx].Decommission != nil {
+		p.Pools[idx].LastUpdate = UTCNow()
+		p.Pools[idx].Decommission.Complete = true
+		p.Pools[idx].Decommission.Failed = false
+		p.Pools[idx].Decommission.Canceled = false
+		return true
+	}
+	return false
+}
+
+func (p *poolMeta) DecommissionFailed(idx int) bool {
+	if p.Pools[idx].Decommission != nil {
+		p.Pools[idx].LastUpdate = UTCNow()
+		p.Pools[idx].Decommission.StartTime = time.Time{}
+		p.Pools[idx].Decommission.Complete = false
+		p.Pools[idx].Decommission.Failed = true
+		p.Pools[idx].Decommission.Canceled = false
+		return true
+	}
+	return false
+}
+
+func (p *poolMeta) DecommissionCancel(idx int) bool {
+	if p.Pools[idx].Decommission != nil {
+		p.Pools[idx].LastUpdate = UTCNow()
+		p.Pools[idx].Decommission.StartTime = time.Time{}
+		p.Pools[idx].Decommission.Complete = false
+		p.Pools[idx].Decommission.Failed = false
+		p.Pools[idx].Decommission.Canceled = true
+		return true
+	}
+	return false
+}
+
+func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool {
+	return p.Pools[idx].Decommission.isBucketDecommissioned(bucket)
+}
+
+func (p *poolMeta) BucketDone(idx int, bucket string) {
+	if p.Pools[idx].Decommission == nil {
+		// Decommission not in progress.
+		return
+	}
+	p.Pools[idx].Decommission.bucketPop(bucket)
+}
+
+func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) {
+	if p.Pools[idx].Decommission != nil {
+		bucket = p.Pools[idx].Decommission.Bucket
+		object = p.Pools[idx].Decommission.Object
+	}
+	return
+}
+
+func (p *poolMeta) TrackCurrentBucketObject(idx int, bucket string, object string) {
+	if p.Pools[idx].Decommission == nil {
+		// Decommission not in progress.
+		return
+	}
+	p.Pools[idx].Decommission.Bucket = bucket
+	p.Pools[idx].Decommission.Object = object
+}
+
+func (p *poolMeta) PendingBuckets(idx int) []string {
+	if p.Pools[idx].Decommission == nil {
+		// Decommission not in progress.
+		return nil
+	}
+
+	return p.Pools[idx].Decommission.bucketsToDecommission()
+}
+
+func (p *poolMeta) QueueBuckets(idx int, buckets []BucketInfo) {
+	// add new queued buckets
+	for _, bucket := range buckets {
+		p.Pools[idx].Decommission.bucketPush(bucket.Name)
+	}
+}
+
+var (
+	errDecommissionAlreadyRunning = errors.New("decommission is already in progress")
+	errDecommissionComplete       = errors.New("decommission is complete, please remove the servers from command-line")
+)
+
+func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
+	for i, pool := range p.Pools {
+		if idx == i {
+			continue
+		}
+		if pool.Decommission != nil {
+			// Do not allow multiple decommissions at the same time.
+			// We shall for now only allow one pool decommission at
+			// a time.
+			return fmt.Errorf("%w at index: %d", errDecommissionAlreadyRunning, i)
+		}
+	}
+
+	now := UTCNow()
+	if p.Pools[idx].Decommission == nil {
+		p.Pools[idx].LastUpdate = now
+		p.Pools[idx].Decommission = &PoolDecommissionInfo{
+			StartTime:   now,
+			StartSize:   pi.Free,
+			CurrentSize: pi.Free,
+			TotalSize:   pi.Total,
+		}
+		return nil
+	}
+
+	// Completed pool doesn't need to be decommissioned again.
+	if p.Pools[idx].Decommission.Complete {
+		return errDecommissionComplete
+	}
+
+	// Canceled or Failed decommission can be triggered again.
+	if p.Pools[idx].Decommission.StartTime.IsZero() {
+		if p.Pools[idx].Decommission.Canceled || p.Pools[idx].Decommission.Failed {
+			p.Pools[idx].LastUpdate = now
+			p.Pools[idx].Decommission = &PoolDecommissionInfo{
+				StartTime:   now,
+				StartSize:   pi.Free,
+				CurrentSize: pi.Free,
+				TotalSize:   pi.Total,
+			}
+			return nil
+		}
+	} // In-progress pool doesn't need to be decommissioned again.
+
+	// In all other scenarios an active decommissioning is in progress.
+	return errDecommissionAlreadyRunning
+}
+
+func (p poolMeta) IsSuspended(idx int) bool {
+	return p.Pools[idx].Decommission != nil
+}
+
+func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
+	type poolInfo struct {
+		position     int
+		completed    bool
+		decomStarted bool // started but not finished yet
+	}
+
+	rememberedPools := make(map[string]poolInfo)
+	for idx, pool := range p.Pools {
+		complete := false
+		decomStarted := false
+		if pool.Decommission != nil {
+			if pool.Decommission.Complete {
+				complete = true
+			}
+			decomStarted = true
+		}
+		rememberedPools[pool.CmdLine] = poolInfo{
+			position:     idx,
+			completed:    complete,
+			decomStarted: decomStarted,
+		}
+	}
+
+	specifiedPools := make(map[string]int)
+	for idx, pool := range pools {
+		specifiedPools[pool.endpoints.CmdLine] = idx
+	}
+
+	// Check if specified pools need to remove decommissioned pool.
+	for k := range specifiedPools {
+		pi, ok := rememberedPools[k]
+		if ok && pi.completed {
+			return false, fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k)
+		}
+	}
+
+	// check if remembered pools are in right position or missing from command line.
+	for k, pi := range rememberedPools {
+		if pi.completed {
+			continue
+		}
+		_, ok := specifiedPools[k]
+		if !ok {
+			return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
+		}
+	}
+
+	// check when remembered pools and specified pools are same they are at the expected position
+	if len(rememberedPools) == len(specifiedPools) {
+		for k, pi := range rememberedPools {
+			pos, ok := specifiedPools[k]
+			if !ok {
+				return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
+			}
+			if pos != pi.position {
+				return false, fmt.Errorf("pool order change detected for %s, expected position is (%s) but found (%s)", k, humanize.Ordinal(pi.position+1), humanize.Ordinal(pos+1))
+			}
+		}
+	}
+
+	update := len(rememberedPools) != len(specifiedPools)
+	if update {
+		for k, pi := range rememberedPools {
+			if pi.decomStarted && !pi.completed {
+				return false, fmt.Errorf("pool(%s) = %s is being decommissioned, No changes should be made to the command line arguments. Please complete the decommission in progress", humanize.Ordinal(pi.position+1), k)
+			}
+		}
+	}
+	return update, nil
+}
+
+func (p *poolMeta) load(ctx context.Context, pool *erasureSets, pools []*erasureSets) error {
+	data, err := readConfig(ctx, pool, poolMetaName)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	if len(data) == 0 {
+		// Seems to be empty create a new poolMeta object.
+		return nil
+	}
+	if len(data) <= 4 {
+		return fmt.Errorf("poolMeta: no data")
+	}
+	// Read header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case poolMetaFormat:
+	default:
+		return fmt.Errorf("poolMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case poolMetaVersion:
+	default:
+		return fmt.Errorf("poolMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+
+	// OK, parse data.
+	if _, err = p.UnmarshalMsg(data[4:]); err != nil {
+		return err
+	}
+
+	switch p.Version {
+	case poolMetaVersionV1:
+	default:
+		return fmt.Errorf("unexpected pool meta version: %d", p.Version)
+	}
+
+	return nil
+}
+
+func (p *poolMeta) CountItem(idx int, size int64, failed bool) {
+	pd := p.Pools[idx].Decommission
+	if pd != nil {
+		if failed {
+			pd.ItemsDecommissionFailed++
+			pd.BytesFailed += size
+		} else {
+			pd.ItemsDecommissioned++
+			pd.BytesDone += size
+		}
+		p.Pools[idx].Decommission = pd
+	}
+}
+
+func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSets, duration time.Duration) (bool, error) {
+	if p.Pools[idx].Decommission == nil {
+		return false, errInvalidArgument
+	}
+	now := UTCNow()
+	if now.Sub(p.Pools[idx].LastUpdate) >= duration {
+		if serverDebugLog {
+			console.Debugf("decommission: persisting poolMeta on disk: threshold:%s, poolMeta:%#v\n", now.Sub(p.Pools[idx].LastUpdate), p.Pools[idx])
+		}
+		p.Pools[idx].LastUpdate = now
+		if err := p.save(ctx, pools); err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
+	data := make([]byte, 4, p.Msgsize()+4)
+
+	// Initialize the header.
+	binary.LittleEndian.PutUint16(data[0:2], poolMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], poolMetaVersion)
+
+	buf, err := p.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	// Saves on all pools to make sure decommissioning of first pool is allowed.
+	for _, eset := range pools {
+		if err = saveConfig(ctx, eset, poolMetaName, buf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+const (
+	poolMetaName      = "pool.bin"
+	poolMetaFormat    = 1
+	poolMetaVersionV1 = 1
+	poolMetaVersion   = poolMetaVersionV1
+)
+
+// Init() initializes pools and saves additional information about them
+// in 'pool.bin', this is eventually used for decommissioning the pool.
+func (z *erasureServerPools) Init(ctx context.Context) error {
+	meta := poolMeta{}
+
+	if err := meta.load(ctx, z.serverPools[0], z.serverPools); err != nil {
+		return err
+	}
+
+	update, err := meta.validate(z.serverPools)
+	if err != nil {
+		return err
+	}
+
+	// if no update is needed return right away.
+	if !update {
+		// We are only supporting single pool decommission at this time
+		// so it makes sense to only resume single pools at any given
+		// time, in future meta.returnResumablePools() might take
+		// '-1' as argument to decommission multiple pools at a time
+		// but this is not a priority at the moment.
+		for _, pool := range meta.returnResumablePools(1) {
+			err := z.Decommission(ctx, pool.ID)
+			switch err {
+			case errDecommissionAlreadyRunning:
+				fallthrough
+			case nil:
+				go z.doDecommissionInRoutine(ctx, pool.ID)
+			}
+		}
+		z.poolMeta = meta
+
+		return nil
+	}
+
+	meta = poolMeta{} // to update write poolMeta fresh.
+	// looks like new pool was added we need to update,
+	// or this is a fresh installation (or an existing
+	// installation with pool removed)
+	meta.Version = poolMetaVersion
+	for idx, pool := range z.serverPools {
+		meta.Pools = append(meta.Pools, PoolStatus{
+			CmdLine:    pool.endpoints.CmdLine,
+			ID:         idx,
+			LastUpdate: UTCNow(),
+		})
+	}
+	if err = meta.save(ctx, z.serverPools); err != nil {
+		return err
+	}
+	z.poolMeta = meta
+	return nil
+}
+
+func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
+	defer gr.Close()
+	objInfo := gr.ObjInfo
+	if objInfo.isMultipart() {
+		uploadID, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
+			VersionID:   objInfo.VersionID,
+			MTime:       objInfo.ModTime,
+			UserDefined: objInfo.UserDefined,
+		})
+		if err != nil {
+			return err
+		}
+		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, uploadID, ObjectOptions{})
+		parts := make([]CompletePart, len(objInfo.Parts))
+		for i, part := range objInfo.Parts {
+			hr, err := hash.NewReader(gr, part.Size, "", "", part.Size)
+			if err != nil {
+				return err
+			}
+			pi, err := z.PutObjectPart(ctx, bucket, objInfo.Name, uploadID,
+				part.Number,
+				NewPutObjReader(hr),
+				ObjectOptions{})
+			if err != nil {
+				return err
+			}
+			parts[i] = CompletePart{
+				ETag:       pi.ETag,
+				PartNumber: pi.PartNumber,
+			}
+		}
+		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, uploadID, parts, ObjectOptions{
+			MTime: objInfo.ModTime,
+		})
+		return err
+	}
+	hr, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size)
+	if err != nil {
+		return err
+	}
+	_, err = z.PutObject(ctx,
+		bucket,
+		objInfo.Name,
+		NewPutObjReader(hr),
+		ObjectOptions{
+			VersionID:   objInfo.VersionID,
+			MTime:       objInfo.ModTime,
+			UserDefined: objInfo.UserDefined,
+		})
+	return err
+}
+
+// versionsSorter sorts FileInfo slices by version.
+//msgp:ignore versionsSorter
+type versionsSorter []FileInfo
+
+func (v versionsSorter) reverse() {
+	sort.Slice(v, func(i, j int) bool {
+		return v[i].ModTime.Before(v[j].ModTime)
+	})
+}
+
+func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bName string) error {
+	var forwardTo string
+	// If we resume to the same bucket, forward to last known item.
+	rbucket, robject := z.poolMeta.ResumeBucketObject(idx)
+	if rbucket != "" && rbucket == bName {
+		forwardTo = robject
+	}
+
+	versioned := globalBucketVersioningSys.Enabled(bName)
+	for _, set := range pool.sets {
+		disks := set.getOnlineDisks()
+		if len(disks) == 0 {
+			logger.LogIf(GlobalContext, fmt.Errorf("no online disks found for set with endpoints %s",
+				set.getEndpoints()))
+			continue
+		}
+
+		decommissionEntry := func(entry metaCacheEntry) {
+			if entry.isDir() {
+				return
+			}
+
+			fivs, err := entry.fileInfoVersions(bName)
+			if err != nil {
+				return
+			}
+
+			// We need a reversed order for Decommissioning,
+			// to create the appropriate stack.
+			versionsSorter(fivs.Versions).reverse()
+
+			for _, version := range fivs.Versions {
+				// TODO: Skip transitioned objects for now.
+				if version.IsRemote() {
+					continue
+				}
+				// We will skip decommissioning delete markers
+				// with single version, its as good as there
+				// is no data associated with the object.
+				if version.Deleted && len(fivs.Versions) == 1 {
+					continue
+				}
+				if version.Deleted {
+					_, err := z.DeleteObject(ctx,
+						bName,
+						version.Name,
+						ObjectOptions{
+							Versioned:         versioned,
+							VersionID:         version.VersionID,
+							MTime:             version.ModTime,
+							DeleteReplication: version.ReplicationState,
+						})
+					if err != nil {
+						logger.LogIf(ctx, err)
+						z.poolMetaMutex.Lock()
+						z.poolMeta.CountItem(idx, 0, true)
+						z.poolMetaMutex.Unlock()
+					} else {
+						set.DeleteObject(ctx,
+							bName,
+							version.Name,
+							ObjectOptions{
+								VersionID: version.VersionID,
+							})
+						z.poolMetaMutex.Lock()
+						z.poolMeta.CountItem(idx, 0, false)
+						z.poolMetaMutex.Unlock()
+					}
+					continue
+				}
+				gr, err := set.GetObjectNInfo(ctx,
+					bName,
+					version.Name,
+					nil,
+					http.Header{},
+					noLock, // all mutations are blocked reads are safe without locks.
+					ObjectOptions{
+						VersionID: version.VersionID,
+					})
+				if err != nil {
+					logger.LogIf(ctx, err)
+					z.poolMetaMutex.Lock()
+					z.poolMeta.CountItem(idx, version.Size, true)
+					z.poolMetaMutex.Unlock()
+					continue
+				}
+				// gr.Close() is ensured by decommissionObject().
+				if err = z.decommissionObject(ctx, bName, gr); err != nil {
+					logger.LogIf(ctx, err)
+					z.poolMetaMutex.Lock()
+					z.poolMeta.CountItem(idx, version.Size, true)
+					z.poolMetaMutex.Unlock()
+					continue
+				}
+				set.DeleteObject(ctx,
+					bName,
+					version.Name,
+					ObjectOptions{
+						VersionID: version.VersionID,
+					})
+				z.poolMetaMutex.Lock()
+				z.poolMeta.CountItem(idx, version.Size, false)
+				z.poolMetaMutex.Unlock()
+			}
+			z.poolMetaMutex.Lock()
+			z.poolMeta.TrackCurrentBucketObject(idx, bName, entry.name)
+			ok, err := z.poolMeta.updateAfter(ctx, idx, z.serverPools, 30*time.Second)
+			logger.LogIf(ctx, err)
+			if ok {
+				globalNotificationSys.ReloadPoolMeta(ctx)
+			}
+			z.poolMetaMutex.Unlock()
+		}
+
+		// How to resolve partial results.
+		resolver := metadataResolutionParams{
+			dirQuorum: set.defaultRQuorum(),
+			objQuorum: set.defaultRQuorum(),
+			bucket:    bName,
+		}
+
+		if err := listPathRaw(ctx, listPathRawOptions{
+			disks:          disks,
+			bucket:         bName,
+			recursive:      true,
+			forwardTo:      forwardTo,
+			minDisks:       len(disks),
+			reportNotFound: false,
+			agreed:         decommissionEntry,
+			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+				entry, ok := entries.resolve(&resolver)
+				if ok {
+					decommissionEntry(*entry)
+				}
+			},
+			finished: nil,
+		}); err != nil {
+			// Decommissioning failed and won't continue
+			return err
+		}
+	}
+	return nil
+}
+
+func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
+	pool := z.serverPools[idx]
+	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
+		if z.poolMeta.isBucketDecommissioned(idx, bucket) {
+			if serverDebugLog {
+				console.Debugln("decommission: already done, moving on", bucket)
+			}
+
+			z.poolMetaMutex.Lock()
+			z.poolMeta.BucketDone(idx, bucket) // remove from pendingBuckets and persist.
+			z.poolMeta.save(ctx, z.serverPools)
+			z.poolMetaMutex.Unlock()
+			continue
+		}
+		if serverDebugLog {
+			console.Debugln("decommission: currently on bucket", bucket)
+		}
+		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
+			return err
+		}
+		z.poolMetaMutex.Lock()
+		z.poolMeta.BucketDone(idx, bucket)
+		z.poolMeta.save(ctx, z.serverPools)
+		z.poolMetaMutex.Unlock()
+	}
+	return nil
+}
+
+func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx int) {
+	z.poolMetaMutex.Lock()
+	var dctx context.Context
+	dctx, z.decommissionCancelers[idx] = context.WithCancel(GlobalContext)
+	z.poolMetaMutex.Unlock()
+
+	if err := z.decommissionInBackground(dctx, idx); err != nil {
+		logger.LogIf(GlobalContext, err)
+		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
+		return
+	}
+	// Complete the decommission..
+	logger.LogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
+}
+
+func (z *erasureServerPools) IsSuspended(idx int) bool {
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+	return z.poolMeta.IsSuspended(idx)
+}
+
+// Decommission - start decommission session.
+func (z *erasureServerPools) Decommission(ctx context.Context, idx int) error {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	// Make pool unwritable before decommissioning.
+	if err := z.StartDecommission(ctx, idx); err != nil {
+		return err
+	}
+
+	go z.doDecommissionInRoutine(ctx, idx)
+
+	// Successfully started decommissioning.
+	return nil
+}
+
+type decomError struct {
+	Err string
+}
+
+func (d decomError) Error() string {
+	return d.Err
+}
+
+type poolSpaceInfo struct {
+	Free  int64
+	Total int64
+	Used  int64
+}
+
+func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpaceInfo, err error) {
+	if idx < 0 {
+		return pi, errInvalidArgument
+	}
+	if idx+1 > len(z.serverPools) {
+		return pi, errInvalidArgument
+	}
+	info, errs := z.serverPools[idx].StorageInfo(context.Background())
+	for _, err := range errs {
+		if err != nil {
+			return pi, errInvalidArgument
+		}
+	}
+	info.Backend = z.BackendInfo()
+	for _, disk := range info.Disks {
+		if disk.Healing {
+			return pi, decomError{
+				Err: fmt.Sprintf("%s drive is healing, decommission will not be started", disk.Endpoint),
+			}
+		}
+	}
+
+	usableTotal := int64(GetTotalUsableCapacity(info.Disks, info))
+	usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info))
+	return poolSpaceInfo{
+		Total: usableTotal,
+		Free:  usableFree,
+		Used:  usableTotal - usableFree,
+	}, nil
+}
+
+func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, error) {
+	if idx < 0 {
+		return PoolStatus{}, errInvalidArgument
+	}
+
+	z.poolMetaMutex.RLock()
+	defer z.poolMetaMutex.RUnlock()
+
+	pi, err := z.getDecommissionPoolSpaceInfo(idx)
+	if err != nil {
+		return PoolStatus{}, errInvalidArgument
+	}
+
+	poolInfo := z.poolMeta.Pools[idx]
+	if poolInfo.Decommission != nil {
+		poolInfo.Decommission.TotalSize = pi.Total
+		poolInfo.Decommission.CurrentSize = poolInfo.Decommission.StartSize + poolInfo.Decommission.BytesDone
+	} else {
+		poolInfo.Decommission = &PoolDecommissionInfo{
+			TotalSize:   pi.Total,
+			CurrentSize: pi.Free,
+		}
+	}
+	return poolInfo, nil
+}
+
+func (z *erasureServerPools) ReloadPoolMeta(ctx context.Context) (err error) {
+	meta := poolMeta{}
+
+	if err = meta.load(ctx, z.serverPools[0], z.serverPools); err != nil {
+		return err
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	z.poolMeta = meta
+	return nil
+}
+
+func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if z.poolMeta.DecommissionCancel(idx) {
+		z.decommissionCancelers[idx]() // cancel any active thread.
+		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+			return err
+		}
+		globalNotificationSys.ReloadPoolMeta(ctx)
+	}
+	return nil
+}
+
+func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if z.poolMeta.DecommissionFailed(idx) {
+		z.decommissionCancelers[idx]() // cancel any active thread.
+		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+			return err
+		}
+		globalNotificationSys.ReloadPoolMeta(ctx)
+	}
+	return nil
+}
+
+func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if z.poolMeta.DecommissionComplete(idx) {
+		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+			return err
+		}
+		globalNotificationSys.ReloadPoolMeta(ctx)
+	}
+	return nil
+}
+
+func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	buckets, err := z.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+
+	// TODO: Support decommissioning transition tiers.
+	for _, bucket := range buckets {
+		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
+			if lc.HasTransition() {
+				return decomError{
+					Err: fmt.Sprintf("Bucket is part of transitioned tier %s: decommission is not allowed in Tier'd setups", bucket.Name),
+				}
+			}
+		}
+	}
+
+	// Buckets data are dispersed in multiple zones/sets, make
+	// sure to decommission the necessary metadata.
+	buckets = append(buckets, BucketInfo{
+		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+	})
+	buckets = append(buckets, BucketInfo{
+		Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+	})
+
+	var pool *erasureSets
+	for pidx := range z.serverPools {
+		if pidx == idx {
+			pool = z.serverPools[idx]
+			break
+		}
+	}
+
+	if pool == nil {
+		return errInvalidArgument
+	}
+
+	pi, err := z.getDecommissionPoolSpaceInfo(idx)
+	if err != nil {
+		return err
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if err = z.poolMeta.Decommission(idx, pi); err != nil {
+		return err
+	}
+	z.poolMeta.QueueBuckets(idx, buckets)
+	if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+		return err
+	}
+	globalNotificationSys.ReloadPoolMeta(ctx)
+	return nil
+}

cmd/erasure-server-pool-decom_gen_test.go

@@ -0,0 +1,575 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalPoolDecommissionInfo(t *testing.T) {
+	v := PoolDecommissionInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgPoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgPoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalPoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodePoolDecommissionInfo(t *testing.T) {
+	v := PoolDecommissionInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodePoolDecommissionInfo Msgsize() is inaccurate")
+	}
+
+	vn := PoolDecommissionInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodePoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodePoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalPoolStatus(t *testing.T) {
+	v := PoolStatus{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgPoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgPoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalPoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodePoolStatus(t *testing.T) {
+	v := PoolStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodePoolStatus Msgsize() is inaccurate")
+	}
+
+	vn := PoolStatus{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodePoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodePoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshaldecomError(t *testing.T) {
+	v := decomError{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdecomError(b *testing.B) {
+	v := decomError{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdecomError(b *testing.B) {
+	v := decomError{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldecomError(b *testing.B) {
+	v := decomError{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedecomError(t *testing.T) {
+	v := decomError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedecomError Msgsize() is inaccurate")
+	}
+
+	vn := decomError{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedecomError(b *testing.B) {
+	v := decomError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedecomError(b *testing.B) {
+	v := decomError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalpoolMeta(t *testing.T) {
+	v := poolMeta{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgpoolMeta(b *testing.B) {
+	v := poolMeta{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgpoolMeta(b *testing.B) {
+	v := poolMeta{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalpoolMeta(b *testing.B) {
+	v := poolMeta{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodepoolMeta(t *testing.T) {
+	v := poolMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodepoolMeta Msgsize() is inaccurate")
+	}
+
+	vn := poolMeta{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodepoolMeta(b *testing.B) {
+	v := poolMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodepoolMeta(b *testing.B) {
+	v := poolMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalpoolSpaceInfo(t *testing.T) {
+	v := poolSpaceInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgpoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgpoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalpoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodepoolSpaceInfo(t *testing.T) {
+	v := poolSpaceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodepoolSpaceInfo Msgsize() is inaccurate")
+	}
+
+	vn := poolSpaceInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodepoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodepoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/erasure-server-pool-decom_test.go

@@ -0,0 +1,196 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"testing"
+)
+
+func prepareErasurePools() (ObjectLayer, []string, error) {
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, err := newErasureServerPools(context.Background(), pools)
+	if err != nil {
+		return nil, nil, err
+	}
+	return objLayer, fsDirs, nil
+}
+
+func TestPoolMetaValidate(t *testing.T) {
+	objLayer1, fsDirs, err := prepareErasurePools()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeRoots(fsDirs)
+
+	meta := objLayer1.(*erasureServerPools).poolMeta
+	pools := objLayer1.(*erasureServerPools).serverPools
+
+	objLayer2, fsDirs, err := prepareErasurePools()
+	if err != nil {
+		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
+	}
+	defer removeRoots(fsDirs)
+
+	newPools := objLayer2.(*erasureServerPools).serverPools
+	reducedPools := pools[1:]
+	orderChangePools := []*erasureSets{
+		pools[1],
+		pools[0],
+	}
+
+	var nmeta1 poolMeta
+	nmeta1.Version = poolMetaVersion
+	nmeta1.Pools = append(nmeta1.Pools, meta.Pools...)
+	for i, pool := range nmeta1.Pools {
+		if i == 0 {
+			nmeta1.Pools[i] = PoolStatus{
+				CmdLine:    pool.CmdLine,
+				ID:         i,
+				LastUpdate: UTCNow(),
+				Decommission: &PoolDecommissionInfo{
+					Complete: true,
+				},
+			}
+		}
+	}
+
+	var nmeta2 poolMeta
+	nmeta2.Version = poolMetaVersion
+	nmeta2.Pools = append(nmeta2.Pools, meta.Pools...)
+	for i, pool := range nmeta2.Pools {
+		if i == 0 {
+			nmeta2.Pools[i] = PoolStatus{
+				CmdLine:    pool.CmdLine,
+				ID:         i,
+				LastUpdate: UTCNow(),
+				Decommission: &PoolDecommissionInfo{
+					Complete: false,
+				},
+			}
+		}
+	}
+
+	testCases := []struct {
+		meta           poolMeta
+		pools          []*erasureSets
+		expectedUpdate bool
+		expectedErr    bool
+		name           string
+	}{
+		{
+			meta:           meta,
+			pools:          pools,
+			name:           "Correct",
+			expectedErr:    false,
+			expectedUpdate: false,
+		},
+		{
+			meta:           meta,
+			pools:          newPools,
+			name:           "Invalid-Commandline",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           meta,
+			pools:          reducedPools,
+			name:           "Invalid-Reduced",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           meta,
+			pools:          orderChangePools,
+			name:           "Invalid-Orderchange",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta1,
+			pools:          pools,
+			name:           "Invalid-Completed-Pool-Not-Removed",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta2,
+			pools:          pools,
+			name:           "Correct-Decom-Pending",
+			expectedErr:    false,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta2,
+			pools:          reducedPools,
+			name:           "Invalid-Decom-Pending-Pool-Removal",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta1,
+			pools:          reducedPools,
+			name:           "Correct-Decom-Pool-Removed",
+			expectedErr:    false,
+			expectedUpdate: true,
+		},
+		{
+			meta:           poolMeta{}, // no-pool info available fresh setup.
+			pools:          pools,
+			name:           "Correct-Fresh-Setup",
+			expectedErr:    false,
+			expectedUpdate: true,
+		},
+		{
+			meta:           nmeta2,
+			pools:          orderChangePools,
+			name:           "Invalid-Orderchange-Decom",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+	}
+
+	t.Parallel()
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.name, func(t *testing.T) {
+			update, err := testCase.meta.validate(testCase.pools)
+			if testCase.expectedErr {
+				t.Log(err)
+			}
+			if err != nil && !testCase.expectedErr {
+				t.Errorf("Expected success, but found %s", err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error("Expected error, but got `nil`")
+			}
+			if update != testCase.expectedUpdate {
+				t.Errorf("Expected %t, got %t", testCase.expectedUpdate, update)
+			}
+		})
+	}
+}

cmd/erasure-server-pool.go

@@ -42,10 +42,15 @@ import (
 type erasureServerPools struct {
 	GatewayUnsupported
 
-	serverPools []*erasureSets
+	poolMetaMutex sync.RWMutex
+	poolMeta      poolMeta
+	serverPools   []*erasureSets
 
 	// Shut down async operations
 	shutdown context.CancelFunc
+
+	// Active decommission canceler
+	decommissionCancelers []context.CancelFunc
 }
 
 func (z *erasureServerPools) SinglePool() bool {
@@ -62,19 +67,14 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 		formats      = make([]*formatErasureV3, len(endpointServerPools))
 		storageDisks = make([][]StorageAPI, len(endpointServerPools))
-		z            = &erasureServerPools{serverPools: make([]*erasureSets, len(endpointServerPools))}
+		z            = &erasureServerPools{
+			serverPools: make([]*erasureSets, len(endpointServerPools)),
+		}
 	)
 
-	var localDrives []string
-
+	var localDrives []StorageAPI
 	local := endpointServerPools.FirstLocal()
 	for i, ep := range endpointServerPools {
-		for _, endpoint := range ep.Endpoints {
-			if endpoint.IsLocal {
-				localDrives = append(localDrives, endpoint.Path)
-			}
-		}
-
 		// If storage class is not set during startup, default values are used
 		// -- Default for Reduced Redundancy Storage class is, parity = 2
 		// -- Default for Standard Storage class is, parity = 2 - disks 4, 5
@@ -94,6 +94,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			return nil, err
 		}
 
+		for _, storageDisk := range storageDisks[i] {
+			if storageDisk != nil && storageDisk.IsLocal() {
+				localDrives = append(localDrives, storageDisk)
+			}
+		}
+
 		if deploymentID == "" {
 			// all zones should have same deployment ID
 			deploymentID = formats[i].ID
@@ -108,13 +114,34 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			return nil, fmt.Errorf("All serverPools should have same deployment ID expected %s, got %s", deploymentID, formats[i].ID)
 		}
 
-		z.serverPools[i], err = newErasureSets(ctx, ep.Endpoints, storageDisks[i], formats[i], commonParityDrives, i)
+		z.serverPools[i], err = newErasureSets(ctx, ep, storageDisks[i], formats[i], commonParityDrives, i)
 		if err != nil {
 			return nil, err
 		}
 	}
+
+	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for {
+		err := z.Init(ctx) // Initializes all pools.
+		if err != nil {
+			if !configRetriableErrors(err) {
+				logger.Fatal(err, "Unable to initialize backend")
+			}
+			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
+			continue
+		}
+		break
+	}
+
+	drives := make([]string, 0, len(localDrives))
+	for _, localDrive := range localDrives {
+		drives = append(drives, localDrive.Endpoint().Path)
+	}
+
+	globalLocalDrives = localDrives
 	ctx, z.shutdown = context.WithCancel(ctx)
-	go intDataUpdateTracker.start(ctx, localDrives...)
+	go intDataUpdateTracker.start(ctx, drives...)
 	return z, nil
 }
 
@@ -155,7 +182,7 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 	found := 0
 	for _, s := range z.serverPools {
 		for _, disks := range s.erasureDisks {
-			for i, disk := range disks {
+			for _, disk := range disks {
 				if disk == OfflineDisk {
 					continue
 				}
@@ -163,10 +190,6 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 				if err != nil {
 					continue
 				}
-				did, err := disk.GetDiskID()
-				if err != nil {
-					did = fmt.Sprintf("disk-%d", i)
-				}
 				for _, si := range stats {
 					found++
 					var r io.ReadCloser
@@ -178,7 +201,9 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 					} else {
 						r = io.NopCloser(bytes.NewBuffer([]byte{}))
 					}
-					err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si)
+					// Keep disk path instead of ID, to ensure that the downloaded zip file can be
+					// easily automated with `minio server hostname{1...n}/disk{1...m}`.
+					err = fn(r, disk.Hostname(), disk.Endpoint().Path, pathJoin(volume, si.Name), si)
 					r.Close()
 					if err != nil {
 						return err
@@ -249,6 +274,10 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 	g := errgroup.WithNErrs(len(z.serverPools))
 	for index := range z.serverPools {
 		index := index
+		// skip suspended pools for any new I/O.
+		if z.IsSuspended(index) {
+			continue
+		}
 		g.Go(func() error {
 			// Get the set where it would be placed.
 			storageInfos[index] = getDiskInfos(ctx, z.serverPools[index].getHashedSet(object).getDisks())
@@ -292,19 +321,24 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 	}
 
 	poolObjInfos := make([]poolObjInfo, len(z.serverPools))
+	poolOpts := make([]ObjectOptions, len(z.serverPools))
+	for i := range z.serverPools {
+		poolOpts[i] = opts
+	}
 
 	var wg sync.WaitGroup
 	for i, pool := range z.serverPools {
 		wg.Add(1)
-		go func(i int, pool *erasureSets) {
+		go func(i int, pool *erasureSets, opts ObjectOptions) {
 			defer wg.Done()
 			// remember the pool index, we may sort the slice original index might be lost.
 			pinfo := poolObjInfo{
 				PoolIndex: i,
 			}
+			opts.VersionID = "" // no need to check for specific versionId
 			pinfo.ObjInfo, pinfo.Err = pool.GetObjectInfo(ctx, bucket, object, opts)
 			poolObjInfos[i] = pinfo
-		}(i, pool)
+		}(i, pool, poolOpts[i])
 	}
 	wg.Wait()
 
@@ -322,6 +356,12 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
 			return -1, pinfo.Err
 		}
+
+		// skip all objects from suspended pools for mutating calls.
+		if z.IsSuspended(pinfo.PoolIndex) && opts.Mutate {
+			continue
+		}
+
 		if isErrObjectNotFound(pinfo.Err) {
 			// No object exists or its a delete marker,
 			// check objInfo to confirm.
@@ -333,23 +373,23 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 			// exist proceed to next pool.
 			continue
 		}
+
 		return pinfo.PoolIndex, nil
 	}
 
 	return -1, toObjectErr(errFileNotFound, bucket, object)
 }
 
-func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
-	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{NoLock: true})
-}
-
-// getPoolIdxExisting returns the (first) found object pool index containing an object.
+// getPoolIdxExistingNoLock returns the (first) found object pool index containing an object.
 // If the object exists, but the latest version is a delete marker, the index with it is still returned.
 // If the object does not exist ObjectNotFound error is returned.
 // If any other error is found, it is returned.
 // The check is skipped if there is only one zone, and 0, nil is always returned in that case.
-func (z *erasureServerPools) getPoolIdxExisting(ctx context.Context, bucket, object string) (idx int, err error) {
-	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{})
+func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
+	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{
+		NoLock: true,
+		Mutate: true,
+	})
 }
 
 func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
@@ -369,9 +409,10 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
 }
 
 // getPoolIdx returns the found previous object and its corresponding pool idx,
-// if none are found falls back to most available space pool.
+// if none are found falls back to most available space pool, this function is
+// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
 func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
-	idx, err = z.getPoolIdxExisting(ctx, bucket, object)
+	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{Mutate: true})
 	if err != nil && !isErrObjectNotFound(err) {
 		return idx, err
 	}
@@ -847,7 +888,11 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 }
 
 func (z *erasureServerPools) deletePrefix(ctx context.Context, bucket string, prefix string) error {
-	for _, zone := range z.serverPools {
+	for idx, zone := range z.serverPools {
+		if z.IsSuspended(idx) {
+			logger.LogIf(ctx, fmt.Errorf("pool %d is suspended, all writes are suspended", idx+1))
+			continue
+		}
 		_, err := zone.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
 		if err != nil {
 			return err
@@ -871,7 +916,8 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return objInfo, err
 	}
@@ -1785,7 +1831,10 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		defer close(errCh)
 
 		var wg sync.WaitGroup
-		for _, erasureSet := range z.serverPools {
+		for idx, erasureSet := range z.serverPools {
+			if z.IsSuspended(idx) {
+				continue
+			}
 			for _, set := range erasureSet.sets {
 				wg.Add(1)
 				go func(set *erasureObjects) {
@@ -1807,6 +1856,9 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 	results := make([]madmin.HealResultItem, len(z.serverPools))
 	var wg sync.WaitGroup
 	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		wg.Add(1)
 		go func(idx int, pool *erasureSets) {
 			defer wg.Done()
@@ -2006,7 +2058,8 @@ func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, obje
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -2022,7 +2075,8 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -2037,7 +2091,8 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -2052,7 +2107,8 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = false
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -2067,7 +2123,8 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 		return z.serverPools[0].TransitionObject(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
 	}
@@ -2082,7 +2139,8 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
 	}

cmd/erasure-sets.go

@@ -71,7 +71,7 @@ type erasureSets struct {
 	erasureLockOwner string
 
 	// List of endpoints provided on the command line.
-	endpoints Endpoints
+	endpoints PoolEndpoints
 
 	// String version of all the endpoints, an optimization
 	// to avoid url.String() conversion taking CPU on
@@ -106,7 +106,7 @@ func isEndpointConnectionStable(diskMap map[Endpoint]StorageAPI, endpoint Endpoi
 	if !disk.IsOnline() {
 		return false
 	}
-	if disk.LastConn().After(lastCheck) {
+	if !lastCheck.IsZero() && disk.LastConn().After(lastCheck) {
 		return false
 	}
 	return true
@@ -160,6 +160,9 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 //   - i'th position is the set index
 //   - j'th position is the disk index in the current set
 func findDiskIndexByDiskID(refFormat *formatErasureV3, diskID string) (int, int, error) {
+	if diskID == "" {
+		return -1, -1, errDiskNotFound
+	}
 	if diskID == offlineDiskUUID {
 		return -1, -1, fmt.Errorf("diskID: %s is offline", diskID)
 	}
@@ -208,7 +211,7 @@ func (s *erasureSets) connectDisks() {
 	var wg sync.WaitGroup
 	setsJustConnected := make([]bool, s.setCount)
 	diskMap := s.getDiskMap()
-	for _, endpoint := range s.endpoints {
+	for _, endpoint := range s.endpoints.Endpoints {
 		if isEndpointConnectionStable(diskMap, endpoint, s.lastConnectDisksOpTime) {
 			continue
 		}
@@ -328,7 +331,7 @@ func (s *erasureSets) GetEndpoints(setIndex int) func() []Endpoint {
 
 		eps := make([]Endpoint, s.setDriveCount)
 		for i := 0; i < s.setDriveCount; i++ {
-			eps[i] = s.endpoints[setIndex*s.setDriveCount+i]
+			eps[i] = s.endpoints.Endpoints[setIndex*s.setDriveCount+i]
 		}
 		return eps
 	}
@@ -350,12 +353,12 @@ func (s *erasureSets) GetDisks(setIndex int) func() []StorageAPI {
 const defaultMonitorConnectEndpointInterval = defaultMonitorNewDiskInterval + time.Second*5
 
 // Initialize new set of erasure coded sets.
-func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageAPI, format *formatErasureV3, defaultParityCount, poolIdx int) (*erasureSets, error) {
+func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks []StorageAPI, format *formatErasureV3, defaultParityCount, poolIdx int) (*erasureSets, error) {
 	setCount := len(format.Erasure.Sets)
 	setDriveCount := len(format.Erasure.Sets[0])
 
-	endpointStrings := make([]string, len(endpoints))
-	for i, endpoint := range endpoints {
+	endpointStrings := make([]string, len(endpoints.Endpoints))
+	for i, endpoint := range endpoints.Endpoints {
 		endpointStrings[i] = endpoint.String()
 	}
 
@@ -399,7 +402,7 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	}
 
 	erasureLockers := map[string]dsync.NetLocker{}
-	for _, endpoint := range endpoints {
+	for _, endpoint := range endpoints.Endpoints {
 		if _, ok := erasureLockers[endpoint.Host]; !ok {
 			erasureLockers[endpoint.Host] = newLockAPI(endpoint)
 		}
@@ -408,50 +411,76 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	for i := 0; i < setCount; i++ {
 		lockerEpSet := set.NewStringSet()
 		for j := 0; j < setDriveCount; j++ {
-			endpoint := endpoints[i*setDriveCount+j]
-			// Only add lockers per endpoint.
+			endpoint := endpoints.Endpoints[i*setDriveCount+j]
+			// Only add lockers only one per endpoint and per erasure set.
 			if locker, ok := erasureLockers[endpoint.Host]; ok && !lockerEpSet.Contains(endpoint.Host) {
 				lockerEpSet.Add(endpoint.Host)
 				s.erasureLockers[i] = append(s.erasureLockers[i], locker)
 			}
-			disk := storageDisks[i*setDriveCount+j]
-			if disk == nil {
-				continue
-			}
-			diskID, derr := disk.GetDiskID()
-			if derr != nil {
-				continue
-			}
-			m, n, err := findDiskIndexByDiskID(format, diskID)
-			if err != nil {
-				continue
-			}
-			if m != i || n != j {
-				logger.LogIf(GlobalContext, fmt.Errorf("Detected unexpected disk ordering refusing to use the disk - poolID: %s, found disk mounted at (set=%s, disk=%s) expected mount at (set=%s, disk=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
-				s.erasureDisks[i][j] = &unrecognizedDisk{storage: disk}
-				continue
-			}
-			disk.SetDiskLoc(s.poolIndex, m, n)
-			s.endpointStrings[m*setDriveCount+n] = disk.String()
-			s.erasureDisks[m][n] = disk
-		}
-
-		// Initialize erasure objects for a given set.
-		s.sets[i] = &erasureObjects{
-			setIndex:              i,
-			poolIndex:             poolIdx,
-			setDriveCount:         setDriveCount,
-			defaultParityCount:    defaultParityCount,
-			getDisks:              s.GetDisks(i),
-			getLockers:            s.GetLockers(i),
-			getEndpoints:          s.GetEndpoints(i),
-			deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
-			nsMutex:               mutex,
-			bp:                    bp,
-			bpOld:                 bpOld,
 		}
 	}
 
+	var wg sync.WaitGroup
+	for i := 0; i < setCount; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+
+			var innerWg sync.WaitGroup
+			for j := 0; j < setDriveCount; j++ {
+				disk := storageDisks[i*setDriveCount+j]
+				if disk == nil {
+					continue
+				}
+				innerWg.Add(1)
+				go func(disk StorageAPI, i, j int) {
+					defer innerWg.Done()
+					diskID, err := disk.GetDiskID()
+					if err != nil {
+						if !errors.Is(err, errUnformattedDisk) {
+							logger.LogIf(ctx, err)
+						}
+						return
+					}
+					if diskID == "" {
+						return
+					}
+					m, n, err := findDiskIndexByDiskID(format, diskID)
+					if err != nil {
+						logger.LogIf(ctx, err)
+						return
+					}
+					if m != i || n != j {
+						logger.LogIf(ctx, fmt.Errorf("Detected unexpected disk ordering refusing to use the disk - poolID: %s, found disk mounted at (set=%s, disk=%s) expected mount at (set=%s, disk=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
+						s.erasureDisks[i][j] = &unrecognizedDisk{storage: disk}
+						return
+					}
+					disk.SetDiskLoc(s.poolIndex, m, n)
+					s.endpointStrings[m*setDriveCount+n] = disk.String()
+					s.erasureDisks[m][n] = disk
+				}(disk, i, j)
+			}
+			innerWg.Wait()
+
+			// Initialize erasure objects for a given set.
+			s.sets[i] = &erasureObjects{
+				setIndex:              i,
+				poolIndex:             poolIdx,
+				setDriveCount:         setDriveCount,
+				defaultParityCount:    defaultParityCount,
+				getDisks:              s.GetDisks(i),
+				getLockers:            s.GetLockers(i),
+				getEndpoints:          s.GetEndpoints(i),
+				deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
+				nsMutex:               mutex,
+				bp:                    bp,
+				bpOld:                 bpOld,
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
 	// start cleanup stale uploads go-routine.
 	go s.cleanupStaleUploads(ctx)
 
@@ -1234,7 +1263,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
-	storageDisks, _ := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints)
+	storageDisks, _ := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints.Endpoints)
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {
@@ -1264,7 +1293,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	}
 
 	// Fetch all the drive info status.
-	beforeDrives := formatsToDrivesInfo(s.endpoints, formats, sErrs)
+	beforeDrives := formatsToDrivesInfo(s.endpoints.Endpoints, formats, sErrs)
 
 	res.After.Drives = make([]madmin.HealDriveInfo, len(beforeDrives))
 	res.Before.Drives = make([]madmin.HealDriveInfo, len(beforeDrives))
@@ -1314,6 +1343,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 			m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This)
 			if err != nil {
+				logger.LogIf(ctx, err)
 				continue
 			}
 

cmd/erasure-sets_test.go

@@ -190,7 +190,8 @@ func TestNewErasureSets(t *testing.T) {
 		t.Fatalf("Unable to format disks for erasure, %s", err)
 	}
 
-	if _, err := newErasureSets(ctx, endpoints, storageDisks, format, ecDrivesNoConfig(16), 0); err != nil {
+	ep := PoolEndpoints{Endpoints: endpoints}
+	if _, err := newErasureSets(ctx, ep, storageDisks, format, ecDrivesNoConfig(16), 0); err != nil {
 		t.Fatalf("Unable to initialize erasure")
 	}
 }

cmd/erasure.go

@@ -214,7 +214,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				}
 			}
 			di.Metrics = &madmin.DiskMetrics{
-				APILatencies: make(map[string]string),
+				APILatencies: make(map[string]interface{}),
 				APICalls:     make(map[string]uint64),
 			}
 			for k, v := range info.Metrics.APILatencies {
@@ -517,6 +517,11 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 					} else {
 						logger.LogIf(ctx, err)
 					}
+					// This ensures that we don't close
+					// bucketResults channel while the
+					// updates-collector goroutine still
+					// holds a reference to this.
+					wg.Wait()
 					continue
 				}
 

cmd/format-erasure.go

@@ -24,6 +24,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"io/ioutil"
 	"reflect"
 	"sync"
@@ -156,13 +157,9 @@ func newFormatErasureV3(numSets int, setLen int) *formatErasureV3 {
 
 // Returns format Erasure version after reading `format.json`, returns
 // successfully the version only if the backend is Erasure.
-func formatGetBackendErasureVersion(formatPath string) (string, error) {
+func formatGetBackendErasureVersion(b []byte) (string, error) {
 	meta := &formatMetaV1{}
-	b, err := xioutil.ReadFile(formatPath)
-	if err != nil {
-		return "", err
-	}
-	if err = json.Unmarshal(b, meta); err != nil {
+	if err := json.Unmarshal(b, meta); err != nil {
 		return "", err
 	}
 	if meta.Version != formatMetaVersionV1 {
@@ -173,7 +170,7 @@ func formatGetBackendErasureVersion(formatPath string) (string, error) {
 	}
 	// Erasure backend found, proceed to detect version.
 	format := &formatErasureVersionDetect{}
-	if err = json.Unmarshal(b, format); err != nil {
+	if err := json.Unmarshal(b, format); err != nil {
 		return "", err
 	}
 	return format.Erasure.Version, nil
@@ -182,50 +179,63 @@ func formatGetBackendErasureVersion(formatPath string) (string, error) {
 // Migrates all previous versions to latest version of `format.json`,
 // this code calls migration in sequence, such as V1 is migrated to V2
 // first before it V2 migrates to V3.n
-func formatErasureMigrate(export string) error {
+func formatErasureMigrate(export string) ([]byte, fs.FileInfo, error) {
 	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
-	version, err := formatGetBackendErasureVersion(formatPath)
+	formatData, formatFi, err := xioutil.ReadFileWithFileInfo(formatPath)
 	if err != nil {
-		return fmt.Errorf("Disk %s: %w", export, err)
+		return nil, nil, err
 	}
+
+	version, err := formatGetBackendErasureVersion(formatData)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
+	}
+
+	migrate := func(formatPath string, formatData []byte) ([]byte, fs.FileInfo, error) {
+		if err = ioutil.WriteFile(formatPath, formatData, 0o666); err != nil {
+			return nil, nil, err
+		}
+		formatFi, err := Lstat(formatPath)
+		if err != nil {
+			return nil, nil, err
+		}
+		return formatData, formatFi, nil
+	}
+
 	switch version {
 	case formatErasureVersionV1:
-		if err = formatErasureMigrateV1ToV2(export, version); err != nil {
-			return fmt.Errorf("Disk %s: %w", export, err)
+		formatData, err = formatErasureMigrateV1ToV2(formatData, version)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
 		}
 		// Migrate successful v1 => v2, proceed to v2 => v3
 		version = formatErasureVersionV2
 		fallthrough
 	case formatErasureVersionV2:
-		if err = formatErasureMigrateV2ToV3(export, version); err != nil {
-			return fmt.Errorf("Disk %s: %w", export, err)
+		formatData, err = formatErasureMigrateV2ToV3(formatData, export, version)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
 		}
 		// Migrate successful v2 => v3, v3 is latest
 		// version = formatXLVersionV3
-		fallthrough
+		return migrate(formatPath, formatData)
 	case formatErasureVersionV3:
 		// v3 is the latest version, return.
-		return nil
+		return formatData, formatFi, nil
 	}
-	return fmt.Errorf(`Disk %s: unknown format version %s`, export, version)
+	return nil, nil, fmt.Errorf(`Disk %s: unknown format version %s`, export, version)
 }
 
 // Migrates version V1 of format.json to version V2 of format.json,
 // migration fails upon any error.
-func formatErasureMigrateV1ToV2(export, version string) error {
+func formatErasureMigrateV1ToV2(data []byte, version string) ([]byte, error) {
 	if version != formatErasureVersionV1 {
-		return fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV1, version)
+		return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV1, version)
 	}
 
-	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
-
 	formatV1 := &formatErasureV1{}
-	b, err := xioutil.ReadFile(formatPath)
-	if err != nil {
-		return err
-	}
-	if err = json.Unmarshal(b, formatV1); err != nil {
-		return err
+	if err := json.Unmarshal(data, formatV1); err != nil {
+		return nil, err
 	}
 
 	formatV2 := &formatErasureV2{}
@@ -238,53 +248,38 @@ func formatErasureMigrateV1ToV2(export, version string) error {
 	formatV2.Erasure.Sets[0] = make([]string, len(formatV1.Erasure.JBOD))
 	copy(formatV2.Erasure.Sets[0], formatV1.Erasure.JBOD)
 
-	b, err = json.Marshal(formatV2)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(formatPath, b, 0o666)
+	return json.Marshal(formatV2)
 }
 
 // Migrates V2 for format.json to V3 (Flat hierarchy for multipart)
-func formatErasureMigrateV2ToV3(export, version string) error {
+func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, error) {
 	if version != formatErasureVersionV2 {
-		return fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV2, version)
+		return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV2, version)
 	}
 
-	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
 	formatV2 := &formatErasureV2{}
-	b, err := xioutil.ReadFile(formatPath)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(b, formatV2)
-	if err != nil {
-		return err
+	if err := json.Unmarshal(data, formatV2); err != nil {
+		return nil, err
 	}
 
-	if err = removeAll(pathJoin(export, minioMetaMultipartBucket)); err != nil {
-		return err
-	}
-
-	if err = mkdirAll(pathJoin(export, minioMetaMultipartBucket), 0o755); err != nil {
-		return err
+	tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID())
+	if err := renameAll(pathJoin(export, minioMetaMultipartBucket),
+		tmpOld); err != nil && err != errFileNotFound {
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
+			pathJoin(export, minioMetaMultipartBucket),
+			tmpOld,
+			osErrToFileErr(err)))
 	}
 
 	// format-V2 struct is exactly same as format-V1 except that version is "3"
 	// which indicates the simplified multipart backend.
 	formatV3 := formatErasureV3{}
-
 	formatV3.Version = formatV2.Version
 	formatV3.Format = formatV2.Format
 	formatV3.Erasure = formatV2.Erasure
-
 	formatV3.Erasure.Version = formatErasureVersionV3
 
-	b, err = json.Marshal(formatV3)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(formatPath, b, 0o666)
+	return json.Marshal(formatV3)
 }
 
 // countErrs - count a specific error.
@@ -382,25 +377,6 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	return nil
 }
 
-var ignoredHiddenDirectories = map[string]struct{}{
-	minioMetaBucket:             {}, // metabucket '.minio.sys'
-	".minio":                    {}, // users may choose to double down the backend as the config folder for certs
-	".snapshot":                 {}, // .snapshot for ignoring NetApp based persistent volumes WAFL snapshot
-	"lost+found":                {}, // 'lost+found' directory default on ext4 filesystems
-	"$RECYCLE.BIN":              {}, // windows specific directory for each drive (hidden)
-	"System Volume Information": {}, // windows specific directory for each drive (hidden)
-}
-
-func isHiddenDirectories(vols ...VolInfo) bool {
-	for _, vol := range vols {
-		if _, ok := ignoredHiddenDirectories[vol.Name]; ok {
-			continue
-		}
-		return false
-	}
-	return true
-}
-
 // loadFormatErasure - loads format.json from disk.
 func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
 	buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
@@ -408,17 +384,6 @@ func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
 		// 'file not found' and 'volume not found' as
 		// same. 'volume not found' usually means its a fresh disk.
 		if err == errFileNotFound || err == errVolumeNotFound {
-			var vols []VolInfo
-			vols, err = disk.ListVols(context.TODO())
-			if err != nil {
-				return nil, err
-			}
-			if !isHiddenDirectories(vols...) {
-				// 'format.json' not found, but we found user data, reject such disks.
-				return nil, fmt.Errorf("some unexpected files '%v' found on %s: %w",
-					vols, disk, errCorruptedFormat)
-			}
-			// No other data found, its a fresh disk.
 			return nil, errUnformattedDisk
 		}
 		return nil, err
@@ -718,12 +683,18 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 
 // relinquishes the underlying connection for all storage disks.
 func closeStorageDisks(storageDisks []StorageAPI) {
+	var wg sync.WaitGroup
 	for _, disk := range storageDisks {
 		if disk == nil {
 			continue
 		}
-		disk.Close()
+		wg.Add(1)
+		go func(disk StorageAPI) {
+			defer wg.Done()
+			disk.Close()
+		}(disk)
 	}
+	wg.Wait()
 }
 
 func initStorageDisksWithErrorsWithoutHealthCheck(endpoints Endpoints) ([]StorageAPI, []error) {
@@ -896,8 +867,14 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error {
 	if disk == nil {
 		return errDiskNotFound
 	}
+	volumes := []string{
+		minioMetaTmpDeletedBucket, // creates .minio.sys/tmp as well as .minio.sys/tmp/.trash
+		minioMetaMultipartBucket,  // creates .minio.sys/multipart
+		dataUsageBucket,           // creates .minio.sys/buckets
+		minioConfigBucket,         // creates .minio.sys/config
+	}
 	// Attempt to create MinIO internal buckets.
-	return disk.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaTmpDeletedBucket, dataUsageBucket, minioMetaTmpBucket+"-old")
+	return disk.MakeVolBulk(context.TODO(), volumes...)
 }
 
 // Initialize a new set of set formats which will be written to all disks.

cmd/format-erasure_test.go

@@ -132,11 +132,12 @@ func TestFormatErasureMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = formatErasureMigrate(rootPath); err != nil {
+	formatData, _, err := formatErasureMigrate(rootPath)
+	if err != nil {
 		t.Fatal(err)
 	}
 
-	migratedVersion, err := formatGetBackendErasureVersion(pathJoin(rootPath, minioMetaBucket, formatConfigFile))
+	migratedVersion, err := formatGetBackendErasureVersion(formatData)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -179,7 +180,7 @@ func TestFormatErasureMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = formatErasureMigrate(rootPath); err == nil {
+	if _, _, err = formatErasureMigrate(rootPath); err == nil {
 		t.Fatal("Expected to fail with unexpected backend format")
 	}
 
@@ -199,7 +200,7 @@ func TestFormatErasureMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = formatErasureMigrate(rootPath); err == nil {
+	if _, _, err = formatErasureMigrate(rootPath); err == nil {
 		t.Fatal("Expected to fail with unexpected backend format version number")
 	}
 }

cmd/gateway/azure/gateway-azure.go

@@ -1416,6 +1416,7 @@ func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (*pol
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
 			policy.NewStatement(
+				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(

cmd/gateway/gcs/gateway-gcs.go

@@ -1470,6 +1470,7 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
 			policy.NewStatement(
+				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				actionSet,

cmd/global-heal.go

@@ -219,7 +219,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 
 		healEntry := func(entry metaCacheEntry) {
+			if entry.name == "" && len(entry.metadata) == 0 {
+				// ignore entries that don't have metadata.
+				return
+			}
 			if entry.isDir() {
+				// ignore healing entry.name's with `/` suffix.
 				return
 			}
 			// We might land at .metacache, .trash, .multipart
@@ -246,7 +251,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}, madmin.HealItemObject)
 				if err != nil {
 					tracker.ItemsFailed++
-					logger.LogIf(ctx, err)
+					logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
 					tracker.ItemsHealed++
 				}
@@ -263,7 +268,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					// If not deleted, assume they failed.
 					tracker.ItemsFailed++
 					tracker.BytesFailed += uint64(version.Size)
-					logger.LogIf(ctx, err)
+					if version.VersionID != "" {
+						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
+					} else {
+						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
+					}
 				} else {
 					tracker.ItemsHealed++
 					tracker.BytesDone += uint64(version.Size)

cmd/globals.go

@@ -338,6 +338,13 @@ var (
 	globalServiceFreezeCnt int32
 	globalServiceFreezeMu  sync.Mutex // Updates.
 
+	// List of local drives to this node, this is only set during server startup.
+	globalLocalDrives []StorageAPI
+
+	// Is MINIO_CI_CD set?
+	globalIsCICD bool
+
+	globalRootDiskThreshold uint64
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -49,6 +49,7 @@ type apiConfig struct {
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
 	deleteCleanupInterval       time.Duration
+	disableODirect              bool
 }
 
 const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
@@ -150,6 +151,14 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
 	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
+	t.disableODirect = cfg.DisableODirect
+}
+
+func (t *apiConfig) isDisableODirect() bool {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.disableODirect
 }
 
 func (t *apiConfig) getListQuorum() int {

cmd/handler-utils.go

@@ -429,69 +429,8 @@ func extractAPIVersion(r *http.Request) string {
 	return "unknown"
 }
 
-func generateUnexpectedRPCMsg(rpcPath, subsystem string, expectedVersion, gotVersion string) string {
-	var reason string
-	switch {
-	case expectedVersion != gotVersion:
-		reason = fmt.Sprintf("Server expects '%s' API version '%s', instead found '%s'", subsystem, expectedVersion, gotVersion)
-	default:
-		reason = fmt.Sprintf("Unexpected RPC call at this path '%s'", rpcPath)
-	}
-
-	return fmt.Sprintf("%s - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", reason, ReleaseTag)
-}
-
 func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if r.Method == http.MethodOptions {
-			return
-		}
-		version := extractAPIVersion(r)
-		switch {
-		case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
-			desc := generateUnexpectedRPCMsg(r.URL.Path, "peer", peerRESTVersion, version)
-			writeErrorResponseString(r.Context(), w, APIError{
-				Code:           "XMinioPeerVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
-			desc := generateUnexpectedRPCMsg(r.URL.Path, "storage", storageRESTVersion, version)
-			writeErrorResponseString(r.Context(), w, APIError{
-				Code:           "XMinioStorageVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
-			desc := generateUnexpectedRPCMsg(r.URL.Path, "lock", lockRESTVersion, version)
-			writeErrorResponseString(r.Context(), w, APIError{
-				Code:           "XMinioLockVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		case strings.HasPrefix(r.URL.Path, adminPathPrefix):
-			var desc string
-			if version == "v1" {
-				desc = fmt.Sprintf("Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases", madmin.AdminAPIVersion, version)
-			} else if version == madmin.AdminAPIVersion {
-				desc = fmt.Sprintf("This 'admin' API is not supported by server in '%s'", getMinioMode())
-			} else {
-				desc = fmt.Sprintf("Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases", version, madmin.AdminAPIVersion)
-			}
-			writeErrorResponseJSON(r.Context(), w, APIError{
-				Code:           "XMinioAdminVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		default:
-			writeErrorResponse(r.Context(), w, APIError{
-				Code: "BadRequest",
-				Description: fmt.Sprintf("An error occurred when parsing the HTTP request %s at '%s'",
-					r.Method, r.URL.Path),
-				HTTPStatusCode: http.StatusBadRequest,
-			}, r.URL)
-		}
-	}
+	return errorResponseHandler
 }
 
 // If none of the http routes match respond with appropriate errors
@@ -499,24 +438,21 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method == http.MethodOptions {
 		return
 	}
-	version := extractAPIVersion(r)
+	desc := "Do not upgrade one server at a time - please follow the recommended guidelines mentioned here https://github.com/minio/minio#upgrading-minio for your environment"
 	switch {
 	case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
-		desc := fmt.Sprintf("Server expects 'peer' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", peerRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",
 			Description:    desc,
 			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
-		desc := fmt.Sprintf("Server expects 'storage' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", storageRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioStorageVersionMismatch",
 			Description:    desc,
 			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
-		desc := fmt.Sprintf("Server expects 'lock' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", lockRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioLockVersionMismatch",
 			Description:    desc,
@@ -524,11 +460,13 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, adminPathPrefix):
 		var desc string
-		if version == "v1" {
+		version := extractAPIVersion(r)
+		switch version {
+		case "v1", madmin.AdminAPIVersionV2:
 			desc = fmt.Sprintf("Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases", madmin.AdminAPIVersion, version)
-		} else if version == madmin.AdminAPIVersion {
+		case madmin.AdminAPIVersion:
 			desc = fmt.Sprintf("This 'admin' API is not supported by server in '%s'", getMinioMode())
-		} else {
+		default:
 			desc = fmt.Sprintf("Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases", version, madmin.AdminAPIVersion)
 		}
 		writeErrorResponseJSON(r.Context(), w, APIError{

cmd/iam-store.go

@@ -1084,11 +1084,12 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 			continue
 		}
 		p, found := cache.iamPolicyDocsMap[policy]
-		if found {
-			if bucketName == "" || p.Policy.MatchResource(bucketName) {
-				policies = append(policies, policy)
-				combinedPolicy = combinedPolicy.Merge(p.Policy)
-			}
+		if !found {
+			continue
+		}
+		if bucketName == "" || p.Policy.MatchResource(bucketName) {
+			policies = append(policies, policy)
+			combinedPolicy = combinedPolicy.Merge(p.Policy)
 		}
 	}
 	return strings.Join(policies, ","), combinedPolicy
@@ -1456,7 +1457,20 @@ func (store *IAMStoreSys) GetAllParentUsers() []string {
 	res := set.NewStringSet()
 	for _, cred := range cache.iamUsersMap {
 		if cred.IsServiceAccount() || cred.IsTemp() {
-			res.Add(cred.ParentUser)
+			parentUser := cred.ParentUser
+			if cred.SessionToken != "" {
+				claims, err := getClaimsFromToken(cred.SessionToken)
+				if err != nil {
+					continue
+				}
+				if v, ok := claims[subClaim]; ok {
+					subFromToken, ok := v.(string)
+					if ok {
+						parentUser = subFromToken
+					}
+				}
+			}
+			res.Add(parentUser)
 		}
 	}
 
@@ -1485,7 +1499,8 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
 		AccessKey: accessKey,
 		SecretKey: cred.SecretKey,
 		Status: func() string {
-			if status == madmin.AccountEnabled {
+			switch string(status) {
+			case string(madmin.AccountEnabled), string(auth.AccountOn):
 				return auth.AccountOn
 			}
 			return auth.AccountOff
@@ -1511,13 +1526,16 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
 	// Found newly requested service account, to be an existing account -
 	// reject such operation (updates to the service account are handled in
 	// a different API).
-	if _, found := cache.iamUsersMap[accessKey]; found {
-		return errIAMActionNotAllowed
+	if scred, found := cache.iamUsersMap[accessKey]; found {
+		if scred.ParentUser != parentUser {
+			return errIAMServiceAccountUsed
+		}
+		return errIAMServiceAccount
 	}
 
 	// Parent user must not be a service account.
 	if cr, found := cache.iamUsersMap[parentUser]; found && cr.IsServiceAccount() {
-		return errIAMActionNotAllowed
+		return errIAMServiceAccount
 	}
 
 	u := newUserIdentity(cred)
@@ -1551,6 +1569,10 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 	switch opts.status {
 	// The caller did not ask to update status account, do nothing
 	case "":
+	case string(madmin.AccountEnabled):
+		cr.Status = auth.AccountOn
+	case string(madmin.AccountDisabled):
+		cr.Status = auth.AccountOff
 	// Update account status
 	case auth.AccountOn, auth.AccountOff:
 		cr.Status = opts.status
@@ -1629,7 +1651,8 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
 		AccessKey: accessKey,
 		SecretKey: ureq.SecretKey,
 		Status: func() string {
-			if ureq.Status == madmin.AccountEnabled {
+			switch string(ureq.Status) {
+			case string(madmin.AccountEnabled), string(auth.AccountOn):
 				return auth.AccountOn
 			}
 			return auth.AccountOff

cmd/iam.go

@@ -883,18 +883,19 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut
 	var embeddedPolicy *iampolicy.Policy
 
 	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
-	if err == nil {
-		pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
-		sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
-		if ptok && spok && pt == "embedded-policy" {
-			policyBytes, err := base64.StdEncoding.DecodeString(sp)
-			if err == nil {
-				p, err := iampolicy.ParseConfig(bytes.NewReader(policyBytes))
-				if err == nil {
-					policy := iampolicy.Policy{}.Merge(*p)
-					embeddedPolicy = &policy
-				}
-			}
+	if err != nil {
+		return auth.Credentials{}, nil, err
+	}
+	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
+	sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
+	if ptok && spok && pt == "embedded-policy" {
+		policyBytes, err := base64.StdEncoding.DecodeString(sp)
+		if err != nil {
+			return auth.Credentials{}, nil, err
+		}
+		embeddedPolicy, err = iampolicy.ParseConfig(bytes.NewReader(policyBytes))
+		if err != nil {
+			return auth.Credentials{}, nil, err
 		}
 	}
 
@@ -1005,11 +1006,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var expiredUsers []string
 	for _, parentUser := range parentUsers {
-		userid, err := parseOpenIDParentUser(parentUser)
-		if err == errSkipFile {
-			continue
-		}
-		u, err := globalOpenIDConfig.LookupUser(userid)
+		u, err := globalOpenIDConfig.LookupUser(parentUser)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 			continue

cmd/metrics-v2.go

@@ -226,10 +226,31 @@ func (g *MetricsGroup) RegisterRead(read func(ctx context.Context) []Metric) {
 	})
 }
 
+func (m *Metric) copyMetric() Metric {
+	metric := Metric{
+		Description:          m.Description,
+		Value:                m.Value,
+		HistogramBucketLabel: m.HistogramBucketLabel,
+		StaticLabels:         make(map[string]string),
+		VariableLabels:       make(map[string]string),
+		Histogram:            make(map[string]uint64),
+	}
+	for k, v := range m.StaticLabels {
+		metric.StaticLabels[k] = v
+	}
+	for k, v := range m.VariableLabels {
+		metric.VariableLabels[k] = v
+	}
+	for k, v := range m.Histogram {
+		metric.Histogram[k] = v
+	}
+	return metric
+}
+
 // Get - returns cached value always upton the configured TTL,
 // once the TTL expires "read()" registered function is called
 // to return the new values and updated.
-func (g *MetricsGroup) Get() []Metric {
+func (g *MetricsGroup) Get() (metrics []Metric) {
 	var c interface{}
 	var err error
 	if g.cacheInterval <= 0 {
@@ -247,7 +268,11 @@ func (g *MetricsGroup) Get() []Metric {
 		return []Metric{}
 	}
 
-	return m
+	metrics = make([]Metric, 0, len(m))
+	for i := range m {
+		metrics = append(metrics, m[i].copyMetric())
+	}
+	return metrics
 }
 
 func getClusterCapacityTotalBytesMD() MetricDescription {

cmd/net.go

@@ -102,15 +102,14 @@ func mustGetLocalIP6() (ipList set.StringSet) {
 
 // getHostIP returns IP address of given host.
 func getHostIP(host string) (ipList set.StringSet, err error) {
-	var ips []net.IP
-
-	if ips, err = net.LookupIP(host); err != nil {
+	addrs, err := globalDNSCache.LookupHost(GlobalContext, host)
+	if err != nil {
 		return ipList, err
 	}
 
 	ipList = set.NewStringSet()
-	for _, ip := range ips {
-		ipList.Add(ip.String())
+	for _, addr := range addrs {
+		ipList.Add(addr)
 	}
 
 	return ipList, err

cmd/notification.go

@@ -616,6 +616,26 @@ func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketNam
 	return bucketStats
 }
 
+// ReloadPoolMeta reloads on disk updates on pool metadata
+func (sys *NotificationSys) ReloadPoolMeta(ctx context.Context) {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(ctx, func() error {
+			return client.ReloadPoolMeta(ctx)
+		}, idx, *client.host)
+	}
+	for _, nErr := range ng.Wait() {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
+		if nErr.Err != nil {
+			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+		}
+	}
+}
+
 // LoadTransitionTierConfig notifies remote peers to load their remote tier
 // configs from config store.
 func (sys *NotificationSys) LoadTransitionTierConfig(ctx context.Context) {

cmd/object-api-common.go

@@ -164,6 +164,8 @@ func listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter
 		result.IsTruncated = true
 		if len(objInfos) > 0 {
 			result.NextMarker = objInfos[len(objInfos)-1].Name
+		} else if len(result.Prefixes) > 0 {
+			result.NextMarker = result.Prefixes[len(result.Prefixes)-1]
 		}
 	}
 
@@ -399,6 +401,8 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 		result.IsTruncated = true
 		if len(objInfos) > 0 {
 			result.NextMarker = objInfos[len(objInfos)-1].Name
+		} else if len(result.Prefixes) > 0 {
+			result.NextMarker = result.Prefixes[len(result.Prefixes)-1]
 		}
 	}
 

cmd/object-api-interface.go

@@ -71,6 +71,9 @@ type ObjectOptions struct {
 
 	// Use the maximum parity (N/2), used when saving server configuration files
 	MaxParity bool
+
+	// Mutate set to 'true' if the call is namespace mutation call
+	Mutate bool
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.

cmd/peer-rest-client.go

@@ -767,6 +767,16 @@ func (client *peerRESTClient) UpdateMetacacheListing(ctx context.Context, m meta
 	return resp, msgp.Decode(respBody, &resp)
 }
 
+func (client *peerRESTClient) ReloadPoolMeta(ctx context.Context) error {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodReloadPoolMeta, nil, nil, 0)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
 func (client *peerRESTClient) LoadTransitionTierConfig(ctx context.Context) error {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodLoadTransitionTierConfig, nil, nil, 0)
 	if err != nil {

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v17" // Add "storage-class" option for SpeedTest
+	peerRESTVersion       = "v18" // Add LoadPoolMeta
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -67,6 +67,7 @@ const (
 	peerRESTMethodLoadTransitionTierConfig    = "/loadtransitiontierconfig"
 	peerRESTMethodSpeedtest                   = "/speedtest"
 	peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
+	peerRESTMethodReloadPoolMeta              = "/reloadpoolmeta"
 )
 
 const (

cmd/peer-rest-server.go

@@ -1023,6 +1023,27 @@ func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *h
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(state))
 }
 
+func (s *peerRESTServer) ReloadPoolMetaHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	pools, ok := objAPI.(*erasureServerPools)
+	if !ok {
+		return
+	}
+	if err := pools.ReloadPoolMeta(r.Context()); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+}
+
 func (s *peerRESTServer) LoadTransitionTierConfigHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
@@ -1353,4 +1374,5 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))
 }

cmd/policy_test.go

@@ -31,14 +31,14 @@ func TestPolicySysIsAllowed(t *testing.T) {
 	p := &policy.Policy{
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetBucketLocationAction),
 				policy.NewResourceSet(policy.NewResource("mybucket", "")),
 				condition.NewFunctions(),
 			),
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.PutObjectAction),
@@ -164,14 +164,14 @@ func TestPolicyToBucketAccessPolicy(t *testing.T) {
 	case1Policy := &policy.Policy{
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
 				policy.NewResourceSet(policy.NewResource("mybucket", "")),
 				condition.NewFunctions(),
 			),
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetObjectAction),
@@ -199,7 +199,7 @@ func TestPolicyToBucketAccessPolicy(t *testing.T) {
 	case3Policy := &policy.Policy{
 		Version: "12-10-2012",
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.PutObjectAction),
@@ -244,14 +244,14 @@ func TestBucketAccessPolicyToPolicy(t *testing.T) {
 	case1Result := &policy.Policy{
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
 				policy.NewResourceSet(policy.NewResource("mybucket", "")),
 				condition.NewFunctions(),
 			),
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetObjectAction),

cmd/prepare-storage.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -70,7 +71,7 @@ var printEndpointError = func() func(Endpoint, error, bool) {
 }()
 
 // Cleans up tmp directory of the local disk.
-func formatErasureCleanupTmp(diskPath string) error {
+func formatErasureCleanupTmp(diskPath string) {
 	// Need to move temporary objects left behind from previous run of minio
 	// server to a unique directory under `minioMetaTmpBucket-old` to clean
 	// up `minioMetaTmpBucket` for the current run.
@@ -81,9 +82,23 @@ func formatErasureCleanupTmp(diskPath string) error {
 	//
 	// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
 	// temporary objects from one of the previous runs of minio server.
-	tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", mustGetUUID())
+	tmpID := mustGetUUID()
+	tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", tmpID)
 	if err := renameAll(pathJoin(diskPath, minioMetaTmpBucket),
-		tmpOld); err != nil && err != errFileNotFound {
+		tmpOld); err != nil && !errors.Is(err, errFileNotFound) {
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
+			pathJoin(diskPath, minioMetaTmpBucket),
+			tmpOld,
+			osErrToFileErr(err)))
+	}
+
+	if err := mkdirAll(pathJoin(diskPath, minioMetaTmpDeletedBucket), 0o777); err != nil {
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate",
+			pathJoin(diskPath, minioMetaTmpBucket),
+			err))
+	}
+
+	if err := renameAll(tmpOld, pathJoin(diskPath, minioMetaTmpDeletedBucket, tmpID)); err != nil && !errors.Is(err, errFileNotFound) {
 		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
 			pathJoin(diskPath, minioMetaTmpBucket),
 			tmpOld,
@@ -92,16 +107,6 @@ func formatErasureCleanupTmp(diskPath string) error {
 
 	// Renames and schedules for purging all bucket metacache.
 	renameAllBucketMetacache(diskPath)
-
-	// Removal of tmp-old folder is backgrounded completely.
-	go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old"))
-
-	if err := mkdirAll(pathJoin(diskPath, minioMetaTmpDeletedBucket), 0o777); err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate",
-			pathJoin(diskPath, minioMetaTmpBucket),
-			err))
-	}
-	return nil
 }
 
 // Following error message is added to fix a regression in release
@@ -178,13 +183,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		}
 	}(storageDisks)
 
-	// Sanitize all local disks during server startup.
-	for _, disk := range storageDisks {
-		if disk != nil && disk.IsLocal() {
-			disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize()
-		}
-	}
-
 	for i, err := range errs {
 		if err != nil {
 			if err == errDiskNotFound && retryCount >= 5 {
@@ -239,13 +237,14 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 
 	// Return error when quorum unformatted disks - indicating we are
 	// waiting for first server to be online.
-	if quorumUnformattedDisks(sErrs) && !firstDisk {
+	unformattedDisks := quorumUnformattedDisks(sErrs)
+	if unformattedDisks && !firstDisk {
 		return nil, nil, errNotFirstDisk
 	}
 
 	// Return error when quorum unformatted disks but waiting for rest
 	// of the servers to be online.
-	if quorumUnformattedDisks(sErrs) && firstDisk {
+	if unformattedDisks && firstDisk {
 		return nil, nil, errFirstDiskWait
 	}
 
@@ -291,10 +290,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		return nil, nil, err
 	}
 
-	// The will always recreate some directories inside .minio.sys of
-	// the local disk such as tmp, multipart and background-ops
-	initErasureMetaVolumesInLocalDisks(storageDisks, formatConfigs)
-
 	return storageDisks, format, nil
 }
 
@@ -311,10 +306,18 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou
 		return time.Now().Round(time.Second).Sub(formatStartTime).String()
 	}
 
-	// Wait on each try for an update.
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
 	var tries int
+	storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo)
+	if err == nil {
+		return storageDisks, format, nil
+	}
+
+	tries++ // tried already once
+
+	// Wait on each try for an update.
+	ticker := time.NewTicker(250 * time.Millisecond)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-ticker.C:

cmd/server-main.go

@@ -51,23 +51,27 @@ import (
 // ServerFlags - server command specific flags
 var ServerFlags = []cli.Flag{
 	cli.StringFlag{
-		Name:  "address",
-		Value: ":" + GlobalMinioDefaultPort,
-		Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
+		Name:   "address",
+		Value:  ":" + GlobalMinioDefaultPort,
+		Usage:  "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
+		EnvVar: "MINIO_ADDRESS",
 	},
 	cli.IntFlag{
-		Name:  "listeners",
-		Value: 1,
-		Usage: "bind N number of listeners per ADDRESS:PORT",
+		Name:   "listeners",
+		Value:  1,
+		Usage:  "bind N number of listeners per ADDRESS:PORT",
+		EnvVar: "MINIO_LISTENERS",
 	},
 	cli.StringFlag{
-		Name:  "console-address",
-		Usage: "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname",
+		Name:   "console-address",
+		Usage:  "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname",
+		EnvVar: "MINIO_CONSOLE_ADDRESS",
 	},
 	cli.DurationFlag{
 		Name:   "shutdown-timeout",
 		Value:  xhttp.DefaultShutdownTimeout,
 		Usage:  "shutdown timeout to gracefully shutdown server",
+		EnvVar: "MINIO_SHUTDOWN_TIMEOUT",
 		Hidden: true,
 	},
 }
@@ -291,6 +295,7 @@ func configRetriableErrors(err error) bool {
 		errors.Is(err, io.ErrUnexpectedEOF) ||
 		errors.As(err, &rquorum) ||
 		errors.As(err, &wquorum) ||
+		isErrObjectNotFound(err) ||
 		isErrBucketNotFound(err) ||
 		errors.Is(err, os.ErrDeadlineExceeded)
 }
@@ -575,22 +580,23 @@ func serverMain(ctx *cli.Context) {
 	// Initialize users credentials and policies in background right after config has initialized.
 	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
 
-	// Initialize transition tier configuration manager
-	if globalIsErasure {
-		if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
-			logger.LogIf(GlobalContext, err)
-		}
-	}
-
 	initDataScanner(GlobalContext, newObject)
 
-	if globalIsErasure { // to be done after config init
+	// Initialize transition tier configuration manager
+	if globalIsErasure {
 		initBackgroundReplication(GlobalContext, newObject)
 		initBackgroundTransition(GlobalContext, newObject)
-		globalTierJournal, err = initTierDeletionJournal(GlobalContext)
-		if err != nil {
-			logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
-		}
+
+		go func() {
+			if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
+				logger.LogIf(GlobalContext, err)
+			}
+
+			globalTierJournal, err = initTierDeletionJournal(GlobalContext)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
+			}
+		}()
 	}
 
 	// initialize the new disk cache objects.
@@ -604,7 +610,7 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	// Prints the formatted startup message, if err is not nil then it prints additional information as well.
-	printStartupMessage(getAPIEndpoints(), err)
+	go printStartupMessage(getAPIEndpoints(), err)
 
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
 		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", globalActiveCred)
@@ -631,7 +637,17 @@ func serverMain(ctx *cli.Context) {
 	if serverDebugLog {
 		logger.Info("== DEBUG Mode enabled ==")
 		logger.Info("Currently set environment settings:")
+		ks := []string{
+			config.EnvAccessKey,
+			config.EnvSecretKey,
+			config.EnvRootUser,
+			config.EnvRootPassword,
+		}
 		for _, v := range os.Environ() {
+			// Do not print sensitive creds in debug.
+			if contains(ks, strings.Split(v, "=")[0]) {
+				continue
+			}
 			logger.Info(v)
 		}
 		logger.Info("======")

cmd/site-replication.go

@@ -275,57 +275,43 @@ type PeerSiteInfo struct {
 }
 
 // getSiteStatuses gathers more info on the sites being added
-func (c *SiteReplicationSys) getSiteStatuses(ctx context.Context, sites []madmin.PeerSite) (psi []PeerSiteInfo, err SRError) {
+func (c *SiteReplicationSys) getSiteStatuses(ctx context.Context, sites ...madmin.PeerSite) (psi []PeerSiteInfo, err error) {
+	psi = make([]PeerSiteInfo, 0, len(sites))
 	for _, v := range sites {
 		admClient, err := getAdminClient(v.Endpoint, v.AccessKey, v.SecretKey)
 		if err != nil {
 			return psi, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
 		}
+
 		info, err := admClient.ServerInfo(ctx)
 		if err != nil {
 			return psi, errSRPeerResp(fmt.Errorf("unable to fetch server info for %s: %w", v.Name, err))
 		}
 
-		deploymentID := info.DeploymentID
-		pi := PeerSiteInfo{
-			PeerSite:     v,
-			DeploymentID: deploymentID,
-			Empty:        true,
+		s3Client, err := getS3Client(v)
+		if err != nil {
+			return psi, errSRPeerResp(fmt.Errorf("unable to create s3 client for %s: %w", v.Name, err))
 		}
 
-		if deploymentID == globalDeploymentID {
-			objAPI := newObjectLayerFn()
-			if objAPI == nil {
-				return psi, errSRObjectLayerNotReady
-			}
-			res, err := objAPI.ListBuckets(ctx)
-			if err != nil {
-				return psi, errSRBackendIssue(err)
-			}
-			if len(res) > 0 {
-				pi.Empty = false
-			}
-			pi.self = true
-		} else {
-			s3Client, err := getS3Client(v)
-			if err != nil {
-				return psi, errSRPeerResp(fmt.Errorf("unable to create s3 client for %s: %w", v.Name, err))
-			}
-			buckets, err := s3Client.ListBuckets(ctx)
-			if err != nil {
-				return psi, errSRPeerResp(fmt.Errorf("unable to list buckets for %s: %v", v.Name, err))
-			}
-			pi.Empty = len(buckets) == 0
+		buckets, err := s3Client.ListBuckets(ctx)
+		if err != nil {
+			return psi, errSRPeerResp(fmt.Errorf("unable to list buckets for %s: %v", v.Name, err))
 		}
-		psi = append(psi, pi)
+
+		psi = append(psi, PeerSiteInfo{
+			PeerSite:     v,
+			DeploymentID: info.DeploymentID,
+			Empty:        len(buckets) == 0,
+			self:         info.DeploymentID == globalDeploymentID,
+		})
 	}
 	return
 }
 
 // AddPeerClusters - add cluster sites for replication configuration.
 func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmin.PeerSite) (madmin.ReplicateAddStatus, error) {
-	sites, serr := c.getSiteStatuses(ctx, psites)
-	if serr.Cause != nil {
+	sites, serr := c.getSiteStatuses(ctx, psites...)
+	if serr != nil {
 		return madmin.ReplicateAddStatus{}, serr
 	}
 	var (
@@ -372,9 +358,18 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 			return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("all existing replicated sites must be specified - missing %s", strings.Join(diffSlc, " ")))
 		}
 	}
+
+	// validate that all clusters are using the same IDP settings.
+	pass, err := c.validateIDPSettings(ctx, sites)
+	if err != nil {
+		return madmin.ReplicateAddStatus{}, err
+	}
+	if !pass {
+		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("all cluster sites must have the same IAM/IDP settings"))
+	}
+
 	// For this `add` API, either all clusters must be empty or the local
 	// cluster must be the only one having some buckets.
-
 	if localHasBuckets && nonLocalPeerWithBuckets != "" {
 		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("only one cluster may have data when configuring site replication"))
 	}
@@ -383,16 +378,6 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("please send your request to the cluster containing data/buckets: %s", nonLocalPeerWithBuckets))
 	}
 
-	// validate that all clusters are using the same (LDAP based)
-	// external IDP.
-	pass, err := c.validateIDPSettings(ctx, sites)
-	if err != nil {
-		return madmin.ReplicateAddStatus{}, err
-	}
-	if !pass {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("all cluster sites must have the same (LDAP) IDP settings"))
-	}
-
 	// FIXME: Ideally, we also need to check if there are any global IAM
 	// policies and any (LDAP user created) service accounts on the other
 	// peer clusters, and if so, reject the cluster replicate add request.
@@ -506,7 +491,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 		Status:  madmin.ReplicateAddStatusSuccess,
 	}
 
-	if err := c.syncLocalToPeers(ctx); err != nil {
+	if err := c.syncToAllPeers(ctx); err != nil {
 		result.InitialSyncErrorMessage = err.Error()
 	}
 
@@ -928,6 +913,8 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 		return err
 	}
 
+	c.RLock()
+	defer c.RUnlock()
 	errMap := make(map[string]error, len(c.state.Peers))
 	for d, peer := range c.state.Peers {
 		if d == globalDeploymentID {
@@ -1041,7 +1028,14 @@ func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, chang
 		if change.UserReq == nil {
 			return errSRInvalidRequest(errInvalidArgument)
 		}
-		err = globalIAMSys.CreateUser(ctx, change.AccessKey, *change.UserReq)
+		userReq := *change.UserReq
+		if userReq.Status != "" && userReq.SecretKey == "" {
+			// Status is set without secretKey updates means we are
+			// only changing the account status.
+			err = globalIAMSys.SetUserStatus(ctx, change.AccessKey, userReq.Status)
+		} else {
+			err = globalIAMSys.CreateUser(ctx, change.AccessKey, userReq)
+		}
 	}
 	if err != nil {
 		return wrapSRErr(err)
@@ -1059,7 +1053,11 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
 	if updReq.IsRemove {
 		err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
 	} else {
-		err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+		if updReq.Status != "" && len(updReq.Members) == 0 {
+			err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled)
+		} else {
+			err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+		}
 	}
 	if err != nil {
 		return wrapSRErr(err)
@@ -1297,6 +1295,30 @@ func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, buc
 	return nil
 }
 
+// PeerBucketQuotaConfigHandler - copies/deletes policy to local cluster.
+func (c *SiteReplicationSys) PeerBucketQuotaConfigHandler(ctx context.Context, bucket string, quota *madmin.BucketQuota) error {
+	if quota != nil {
+		quotaData, err := json.Marshal(quota)
+		if err != nil {
+			return wrapSRErr(err)
+		}
+
+		if err = globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, quotaData); err != nil {
+			return wrapSRErr(err)
+		}
+
+		return nil
+	}
+
+	// Delete the bucket policy
+	err := globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, nil)
+	if err != nil {
+		return wrapSRErr(err)
+	}
+
+	return nil
+}
+
 // getAdminClient - NOTE: ensure to take at least a read lock on SiteReplicationSys
 // before calling this.
 func (c *SiteReplicationSys) getAdminClient(ctx context.Context, deploymentID string) (*madmin.AdminClient, error) {
@@ -1313,6 +1335,20 @@ func (c *SiteReplicationSys) getAdminClient(ctx context.Context, deploymentID st
 	return getAdminClient(peer.Endpoint, creds.AccessKey, creds.SecretKey)
 }
 
+// getAdminClientWithEndpoint - NOTE: ensure to take at least a read lock on SiteReplicationSys
+// before calling this.
+func (c *SiteReplicationSys) getAdminClientWithEndpoint(ctx context.Context, deploymentID, endpoint string) (*madmin.AdminClient, error) {
+	creds, err := c.getPeerCreds()
+	if err != nil {
+		return nil, err
+	}
+
+	if _, ok := c.state.Peers[deploymentID]; !ok {
+		return nil, errSRPeerNotFound
+	}
+	return getAdminClient(endpoint, creds.AccessKey, creds.SecretKey)
+}
+
 func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) {
 	creds, ok := globalIAMSys.store.GetUser(c.state.ServiceAccountAccessKey)
 	if !ok {
@@ -1321,20 +1357,26 @@ func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) {
 	return &creds, nil
 }
 
-// syncLocalToPeers is used when initially configuring site replication, to
-// copy existing buckets, their settings, service accounts and policies to all
-// new peers.
-func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) error {
+// listBuckets returns a consistent common view of latest unique buckets across
+// sites, this is used for replication.
+func (c *SiteReplicationSys) listBuckets(ctx context.Context) ([]BucketInfo, error) {
 	// If local has buckets, enable versioning on them, create them on peers
 	// and setup replication rules.
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return errSRObjectLayerNotReady
+		return nil, errSRObjectLayerNotReady
 	}
-	buckets, err := objAPI.ListBuckets(ctx)
+	return objAPI.ListBuckets(ctx)
+}
+
+// syncToAllPeers is used for syncing local data to all remote peers, it is
+// called once during initial "AddPeerClusters" request.
+func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
+	buckets, err := c.listBuckets(ctx)
 	if err != nil {
-		return errSRBackendIssue(err)
+		return err
 	}
+
 	for _, bucketInfo := range buckets {
 		bucket := bucketInfo.Name
 
@@ -1453,8 +1495,34 @@ func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) error {
 				return errSRBucketMetaError(err)
 			}
 		}
+
+		quotaConfig, err := globalBucketMetadataSys.GetQuotaConfig(bucket)
+		found = true
+		if _, ok := err.(BucketQuotaConfigNotFound); ok {
+			found = false
+		} else if err != nil {
+			return errSRBackendIssue(err)
+		}
+		if found {
+			quotaConfigJSON, err := json.Marshal(quotaConfig)
+			if err != nil {
+				return wrapSRErr(err)
+			}
+			err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:   madmin.SRBucketMetaTypeQuotaConfig,
+				Bucket: bucket,
+				Quota:  quotaConfigJSON,
+			})
+			if err != nil {
+				return errSRBucketMetaError(err)
+			}
+		}
 	}
 
+	// Order matters from now on how the information is
+	// synced to remote sites.
+
+	// Policies should be synced first.
 	{
 		// Replicate IAM policies on local to all peers.
 		allPolicies, err := globalIAMSys.ListPolicies(ctx, "")
@@ -1478,20 +1546,131 @@ func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) error {
 		}
 	}
 
+	// Next should be userAccounts those are local users, OIDC and LDAP will not
+	// may not have any local users.
+	{
+		userAccounts := make(map[string]auth.Credentials)
+		globalIAMSys.store.rlock()
+		err := globalIAMSys.store.loadUsers(ctx, regUser, userAccounts)
+		globalIAMSys.store.runlock()
+		if err != nil {
+			return errSRBackendIssue(err)
+		}
+
+		for _, acc := range userAccounts {
+			if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
+				Type: madmin.SRIAMItemIAMUser,
+				IAMUser: &madmin.SRIAMUser{
+					AccessKey:   acc.AccessKey,
+					IsDeleteReq: false,
+					UserReq: &madmin.AddOrUpdateUserReq{
+						SecretKey: acc.SecretKey,
+						Status:    madmin.AccountStatus(acc.Status),
+					},
+				},
+			}); err != nil {
+				return errSRIAMError(err)
+			}
+		}
+	}
+
+	// Next should be Groups for some of these users, LDAP might have some Group
+	// DNs here
+	{
+		groups := make(map[string]GroupInfo)
+
+		globalIAMSys.store.rlock()
+		err := globalIAMSys.store.loadGroups(ctx, groups)
+		globalIAMSys.store.runlock()
+		if err != nil {
+			return errSRBackendIssue(err)
+		}
+
+		for gname, group := range groups {
+			if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
+				Type: madmin.SRIAMItemGroupInfo,
+				GroupInfo: &madmin.SRGroupInfo{
+					UpdateReq: madmin.GroupAddRemove{
+						Group:    gname,
+						Members:  group.Members,
+						Status:   madmin.GroupStatus(group.Status),
+						IsRemove: false,
+					},
+				},
+			}); err != nil {
+				return errSRIAMError(err)
+			}
+		}
+	}
+
+	// Service accounts are the static accounts that should be synced with
+	// valid claims.
+	{
+		serviceAccounts := make(map[string]auth.Credentials)
+		globalIAMSys.store.rlock()
+		err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts)
+		globalIAMSys.store.runlock()
+		if err != nil {
+			return errSRBackendIssue(err)
+		}
+
+		for user, acc := range serviceAccounts {
+			if user == siteReplicatorSvcAcc {
+				// skip the site replicate svc account as it is
+				// already replicated.
+				continue
+			}
+
+			claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.AccessKey)
+			if err != nil {
+				return errSRBackendIssue(err)
+			}
+
+			_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.AccessKey)
+			if err != nil {
+				return errSRBackendIssue(err)
+			}
+
+			var policyJSON []byte
+			if policy != nil {
+				policyJSON, err = json.Marshal(policy)
+				if err != nil {
+					return wrapSRErr(err)
+				}
+			}
+
+			err = c.IAMChangeHook(ctx, madmin.SRIAMItem{
+				Type: madmin.SRIAMItemSvcAcc,
+				SvcAccChange: &madmin.SRSvcAccChange{
+					Create: &madmin.SRSvcAccCreate{
+						Parent:        acc.ParentUser,
+						AccessKey:     user,
+						SecretKey:     acc.SecretKey,
+						Groups:        acc.Groups,
+						Claims:        claims,
+						SessionPolicy: json.RawMessage(policyJSON),
+						Status:        acc.Status,
+					},
+				},
+			})
+			if err != nil {
+				return errSRIAMError(err)
+			}
+		}
+	}
+
+	// Followed by policy mapping for the userAccounts we previously synced.
 	{
 		// Replicate policy mappings on local to all peers.
 		userPolicyMap := make(map[string]MappedPolicy)
 		groupPolicyMap := make(map[string]MappedPolicy)
 		globalIAMSys.store.rlock()
-		errU := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
-		errG := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
+		errU := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
+		errG := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
 		globalIAMSys.store.runlock()
 		if errU != nil {
 			return errSRBackendIssue(errU)
 		}
-		if errG != nil {
-			return errSRBackendIssue(errG)
-		}
 
 		for user, mp := range userPolicyMap {
 			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
@@ -1507,6 +1686,10 @@ func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) error {
 			}
 		}
 
+		if errG != nil {
+			return errSRBackendIssue(errG)
+		}
+
 		for group, mp := range groupPolicyMap {
 			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
 				Type: madmin.SRIAMItemPolicyMapping,
@@ -1522,54 +1705,44 @@ func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) error {
 		}
 	}
 
+	// and finally followed by policy mappings for for STS users.
 	{
-		// Check for service accounts and replicate them. Only LDAP user
-		// owned service accounts are supported for this operation.
-		serviceAccounts := make(map[string]auth.Credentials)
+		// Replicate policy mappings on local to all peers.
+		userPolicyMap := make(map[string]MappedPolicy)
+		groupPolicyMap := make(map[string]MappedPolicy)
 		globalIAMSys.store.rlock()
-		err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts)
+		errU := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
+		errG := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
 		globalIAMSys.store.runlock()
-		if err != nil {
-			return errSRBackendIssue(err)
+		if errU != nil {
+			return errSRBackendIssue(errU)
 		}
-		for user, acc := range serviceAccounts {
-			if user == siteReplicatorSvcAcc {
-				// skip the site replicate svc account as it is
-				// already replicated.
-				continue
-			}
-			claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.AccessKey)
+
+		for user, mp := range userPolicyMap {
+			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
+				Type: madmin.SRIAMItemPolicyMapping,
+				PolicyMapping: &madmin.SRPolicyMapping{
+					UserOrGroup: user,
+					IsGroup:     false,
+					Policy:      mp.Policies,
+				},
+			})
 			if err != nil {
-				return errSRBackendIssue(err)
+				return errSRIAMError(err)
 			}
-			if claims != nil {
-				if _, isLDAPAccount := claims[ldapUserN]; !isLDAPAccount {
-					continue
-				}
-			}
-			_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.AccessKey)
-			if err != nil {
-				return errSRBackendIssue(err)
-			}
-			var policyJSON []byte
-			if policy != nil {
-				policyJSON, err = json.Marshal(policy)
-				if err != nil {
-					return wrapSRErr(err)
-				}
-			}
-			err = c.IAMChangeHook(ctx, madmin.SRIAMItem{
-				Type: madmin.SRIAMItemSvcAcc,
-				SvcAccChange: &madmin.SRSvcAccChange{
-					Create: &madmin.SRSvcAccCreate{
-						Parent:        acc.ParentUser,
-						AccessKey:     user,
-						SecretKey:     acc.SecretKey,
-						Groups:        acc.Groups,
-						Claims:        claims,
-						SessionPolicy: json.RawMessage(policyJSON),
-						Status:        acc.Status,
-					},
+		}
+
+		if errG != nil {
+			return errSRBackendIssue(errG)
+		}
+
+		for group, mp := range groupPolicyMap {
+			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
+				Type: madmin.SRIAMItemPolicyMapping,
+				PolicyMapping: &madmin.SRPolicyMapping{
+					UserOrGroup: group,
+					IsGroup:     true,
+					Policy:      mp.Policies,
 				},
 			})
 			if err != nil {
@@ -1701,7 +1874,10 @@ func getAdminClient(endpoint, accessKey, secretKey string) (*madmin.AdminClient,
 }
 
 func getS3Client(pc madmin.PeerSite) (*minioClient.Client, error) {
-	ep, _ := url.Parse(pc.Endpoint)
+	ep, err := url.Parse(pc.Endpoint)
+	if err != nil {
+		return nil, err
+	}
 	return minioClient.New(ep.Host, &minioClient.Options{
 		Creds:     credentials.NewStaticV4(pc.AccessKey, pc.SecretKey, ""),
 		Secure:    ep.Scheme == "https",
@@ -2312,3 +2488,110 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 	}
 	return info, nil
 }
+
+// EditPeerCluster - edits replication configuration and updates peer endpoint.
+func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.PeerInfo) (madmin.ReplicateEditStatus, error) {
+	sites, err := c.GetClusterInfo(ctx)
+	if err != nil {
+		return madmin.ReplicateEditStatus{}, errSRBackendIssue(err)
+	}
+	if !sites.Enabled {
+		return madmin.ReplicateEditStatus{}, errSRNotEnabled
+	}
+
+	var (
+		found     bool
+		admClient *madmin.AdminClient
+	)
+
+	for _, v := range sites.Sites {
+		if peer.DeploymentID == v.DeploymentID {
+			found = true
+			if peer.Endpoint == v.Endpoint {
+				return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s entered for deployment id %s already configured in site replication", v.Endpoint, v.DeploymentID))
+			}
+			admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint)
+			if err != nil {
+				return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
+			}
+			// check if endpoint is reachable
+			if _, err = admClient.ServerInfo(ctx); err != nil {
+				return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("Endpoint %s not reachable: %w", peer.Endpoint, err))
+			}
+		}
+	}
+
+	if !found {
+		return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("%s not found in existing replicated sites", peer.DeploymentID))
+	}
+
+	errs := make(map[string]error, len(c.state.Peers))
+	var wg sync.WaitGroup
+
+	pi := c.state.Peers[peer.DeploymentID]
+	pi.Endpoint = peer.Endpoint
+
+	for i, v := range sites.Sites {
+		if v.DeploymentID == globalDeploymentID {
+			c.state.Peers[peer.DeploymentID] = pi
+			continue
+		}
+		wg.Add(1)
+		go func(pi madmin.PeerInfo, i int) {
+			defer wg.Done()
+			v := sites.Sites[i]
+			admClient, err := c.getAdminClient(ctx, v.DeploymentID)
+			if v.DeploymentID == peer.DeploymentID {
+				admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint)
+			}
+			if err != nil {
+				errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
+				return
+			}
+			if err = admClient.SRPeerEdit(ctx, pi); err != nil {
+				errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", v.Name, err))
+				return
+			}
+		}(pi, i)
+	}
+
+	wg.Wait()
+	for dID, err := range errs {
+		if err != nil {
+			return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", c.state.Peers[dID].Name, err))
+		}
+	}
+	// we can now save the cluster replication configuration state.
+	if err = c.saveToDisk(ctx, c.state); err != nil {
+		return madmin.ReplicateEditStatus{
+			Status:    madmin.ReplicateAddStatusPartial,
+			ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err),
+		}, nil
+	}
+
+	result := madmin.ReplicateEditStatus{
+		Success: true,
+		Status:  fmt.Sprintf("Cluster replication configuration updated with endpoint %s for peer %s successfully", peer.Endpoint, peer.Name),
+	}
+	return result, nil
+}
+
+// PeerEditReq - internal API handler to respond to a peer cluster's request
+// to edit endpoint.
+func (c *SiteReplicationSys) PeerEditReq(ctx context.Context, arg madmin.PeerInfo) error {
+	ourName := ""
+	for i := range c.state.Peers {
+		p := c.state.Peers[i]
+		if p.DeploymentID == arg.DeploymentID {
+			p.Endpoint = arg.Endpoint
+			c.state.Peers[arg.DeploymentID] = p
+		}
+		if p.DeploymentID == globalDeploymentID {
+			ourName = p.Name
+		}
+	}
+	if err := c.saveToDisk(ctx, c.state); err != nil {
+		return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to disk on %s: %v", ourName, err))
+	}
+	return nil
+}

cmd/storage-datatypes.go

@@ -72,11 +72,12 @@ type FilesInfo struct {
 	IsTruncated bool
 }
 
-// FilesInfoVersions represents a list of file versions,
-// additionally indicates if the list is last.
-type FilesInfoVersions struct {
-	FilesVersions []FileInfoVersions
-	IsTruncated   bool
+// Size returns size of all versions for the object 'Name'
+func (f FileInfoVersions) Size() (size int64) {
+	for _, v := range f.Versions {
+		size += v.Size
+	}
+	return size
 }
 
 // FileInfoVersions represent a list of versions for a given file.
@@ -235,7 +236,9 @@ func (fi *FileInfo) SetInlineData() {
 }
 
 // VersionPurgeStatusKey denotes purge status in metadata
-const VersionPurgeStatusKey = "purgestatus"
+const (
+	VersionPurgeStatusKey = ReservedMetadataPrefixLower + "purgestatus"
+)
 
 // newFileInfo - initializes new FileInfo, allocates a fresh erasure info.
 func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {

cmd/storage-datatypes_gen.go

@@ -1530,178 +1530,6 @@ func (z *FilesInfo) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *FilesInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "FilesVersions":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "FilesVersions")
-				return
-			}
-			if cap(z.FilesVersions) >= int(zb0002) {
-				z.FilesVersions = (z.FilesVersions)[:zb0002]
-			} else {
-				z.FilesVersions = make([]FileInfoVersions, zb0002)
-			}
-			for za0001 := range z.FilesVersions {
-				err = z.FilesVersions[za0001].DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, "FilesVersions", za0001)
-					return
-				}
-			}
-		case "IsTruncated":
-			z.IsTruncated, err = dc.ReadBool()
-			if err != nil {
-				err = msgp.WrapError(err, "IsTruncated")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *FilesInfoVersions) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
-	// write "FilesVersions"
-	err = en.Append(0x82, 0xad, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.FilesVersions)))
-	if err != nil {
-		err = msgp.WrapError(err, "FilesVersions")
-		return
-	}
-	for za0001 := range z.FilesVersions {
-		err = z.FilesVersions[za0001].EncodeMsg(en)
-		if err != nil {
-			err = msgp.WrapError(err, "FilesVersions", za0001)
-			return
-		}
-	}
-	// write "IsTruncated"
-	err = en.Append(0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.IsTruncated)
-	if err != nil {
-		err = msgp.WrapError(err, "IsTruncated")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *FilesInfoVersions) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
-	// string "FilesVersions"
-	o = append(o, 0x82, 0xad, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.FilesVersions)))
-	for za0001 := range z.FilesVersions {
-		o, err = z.FilesVersions[za0001].MarshalMsg(o)
-		if err != nil {
-			err = msgp.WrapError(err, "FilesVersions", za0001)
-			return
-		}
-	}
-	// string "IsTruncated"
-	o = append(o, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64)
-	o = msgp.AppendBool(o, z.IsTruncated)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *FilesInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "FilesVersions":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "FilesVersions")
-				return
-			}
-			if cap(z.FilesVersions) >= int(zb0002) {
-				z.FilesVersions = (z.FilesVersions)[:zb0002]
-			} else {
-				z.FilesVersions = make([]FileInfoVersions, zb0002)
-			}
-			for za0001 := range z.FilesVersions {
-				bts, err = z.FilesVersions[za0001].UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "FilesVersions", za0001)
-					return
-				}
-			}
-		case "IsTruncated":
-			z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "IsTruncated")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *FilesInfoVersions) Msgsize() (s int) {
-	s = 1 + 14 + msgp.ArrayHeaderSize
-	for za0001 := range z.FilesVersions {
-		s += z.FilesVersions[za0001].Msgsize()
-	}
-	s += 12 + msgp.BoolSize
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *VolInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 	var zb0001 uint32

cmd/storage-datatypes_gen_test.go

@@ -574,119 +574,6 @@ func BenchmarkDecodeFilesInfo(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalFilesInfoVersions(t *testing.T) {
-	v := FilesInfoVersions{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeFilesInfoVersions(t *testing.T) {
-	v := FilesInfoVersions{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeFilesInfoVersions Msgsize() is inaccurate")
-	}
-
-	vn := FilesInfoVersions{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
 func TestMarshalUnmarshalVolInfo(t *testing.T) {
 	v := VolInfo{}
 	bts, err := v.MarshalMsg(nil)

cmd/sts-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+/// Copyright (c) 2015-2021 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -80,16 +80,6 @@ const (
 	roleArnClaim = "roleArn"
 )
 
-func parseOpenIDParentUser(parentUser string) (userID string, err error) {
-	if strings.HasPrefix(parentUser, "openid:") {
-		tokens := strings.SplitN(strings.TrimPrefix(parentUser, "openid:"), ":", 2)
-		if len(tokens) == 2 {
-			return tokens[0], nil
-		}
-	}
-	return "", errSkipFile
-}
-
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
 type stsAPIHandlers struct{}
 

cmd/sts-handlers_test.go

@@ -20,19 +20,15 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"net/http"
-	"net/url"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/coreos/go-oidc"
 	"github.com/minio/madmin-go"
 	minio "github.com/minio/minio-go/v7"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"golang.org/x/oauth2"
 )
 
 func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
@@ -742,7 +738,7 @@ func (s *TestSuiteIAM) TestOpenIDSTS(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -817,7 +813,7 @@ func (s *TestSuiteIAM) TestOpenIDSTSAddUser(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -907,7 +903,7 @@ func (s *TestSuiteIAM) TestOpenIDServiceAcc(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -985,132 +981,11 @@ func (s *TestSuiteIAM) TestOpenIDServiceAcc(c *check) {
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket)
 }
 
-type providerParams struct {
-	clientID, clientSecret, providerURL, redirectURL string
-}
-
-var testProvider = providerParams{
-	clientID:     "minio-client-app",
-	clientSecret: "minio-client-app-secret",
-	providerURL:  "http://127.0.0.1:5556/dex",
-	redirectURL:  "http://127.0.0.1:10000/oauth_callback",
-}
-
-// mockTestUserInteraction - tries to login to dex using provided credentials.
-// It performs the user's browser interaction to login and retrieves the auth
-// code from dex and exchanges it for a JWT.
-func mockTestUserInteraction(ctx context.Context, pro providerParams, username, password string) (string, error) {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
-	provider, err := oidc.NewProvider(ctx, pro.providerURL)
-	if err != nil {
-		return "", fmt.Errorf("unable to create provider: %v", err)
-	}
-
-	// Configure an OpenID Connect aware OAuth2 client.
-	oauth2Config := oauth2.Config{
-		ClientID:     pro.clientID,
-		ClientSecret: pro.clientSecret,
-		RedirectURL:  pro.redirectURL,
-
-		// Discovery returns the OAuth2 endpoints.
-		Endpoint: provider.Endpoint(),
-
-		// "openid" is a required scope for OpenID Connect flows.
-		Scopes: []string{oidc.ScopeOpenID, "groups"},
-	}
-
-	state := "xxx"
-	authCodeURL := oauth2Config.AuthCodeURL(state)
-	// fmt.Printf("authcodeurl: %s\n", authCodeURL)
-
-	var lastReq *http.Request
-	checkRedirect := func(req *http.Request, via []*http.Request) error {
-		// fmt.Printf("CheckRedirect:\n")
-		// fmt.Printf("Upcoming: %s %#v\n", req.URL.String(), req)
-		// for _, c := range via {
-		// 	fmt.Printf("Sofar: %s %#v\n", c.URL.String(), c)
-		// }
-		// Save the last request in a redirect chain.
-		lastReq = req
-		// We do not follow redirect back to client application.
-		if req.URL.Path == "/oauth_callback" {
-			return http.ErrUseLastResponse
-		}
-		return nil
-	}
-
-	dexClient := http.Client{
-		CheckRedirect: checkRedirect,
-	}
-
-	u, err := url.Parse(authCodeURL)
-	if err != nil {
-		return "", fmt.Errorf("url parse err: %v", err)
-	}
-
-	// Start the user auth flow. This page would present the login with
-	// email or LDAP option.
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return "", fmt.Errorf("new request err: %v", err)
-	}
-	_, err = dexClient.Do(req)
-	// fmt.Printf("Do: %#v %#v\n", resp, err)
-	if err != nil {
-		return "", fmt.Errorf("auth url request err: %v", err)
-	}
-
-	// Modify u to choose the ldap option
-	u.Path += "/ldap"
-	// fmt.Println(u)
-
-	// Pick the LDAP login option. This would return a form page after
-	// following some redirects. `lastReq` would be the URL of the form
-	// page, where we need to POST (submit) the form.
-	req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return "", fmt.Errorf("new request err (/ldap): %v", err)
-	}
-	_, err = dexClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("request err: %v", err)
-	}
-
-	// Fill the login form with our test creds:
-	// fmt.Printf("login form url: %s\n", lastReq.URL.String())
-	formData := url.Values{}
-	formData.Set("login", username)
-	formData.Set("password", password)
-	req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode()))
-	if err != nil {
-		return "", fmt.Errorf("new request err (/login): %v", err)
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	_, err = dexClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("post form err: %v", err)
-	}
-	// fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header)
-	// fmt.Printf("lastReq: %#v\n", lastReq.URL.String())
-
-	// On form submission, the last redirect response contains the auth
-	// code, which we now have in `lastReq`. Exchange it for a JWT id_token.
-	q := lastReq.URL.Query()
-	code := q.Get("code")
-	oauth2Token, err := oauth2Config.Exchange(ctx, code)
-	if err != nil {
-		return "", fmt.Errorf("unable to exchange code for id token: %v", err)
-	}
-
-	rawIDToken, ok := oauth2Token.Extra("id_token").(string)
-	if !ok {
-		return "", fmt.Errorf("id_token not found!")
-	}
-
-	// fmt.Printf("TOKEN: %s\n", rawIDToken)
-	return rawIDToken, nil
+var testAppParams = OpenIDClientAppParams{
+	ClientID:     "minio-client-app",
+	ClientSecret: "minio-client-app-secret",
+	ProviderURL:  "http://127.0.0.1:5556/dex",
+	RedirectURL:  "http://127.0.0.1:10000/oauth_callback",
 }
 
 const (
@@ -1248,7 +1123,7 @@ func (s *TestSuiteIAM) TestOpenIDSTSWithRolePolicy(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -1294,7 +1169,7 @@ func (s *TestSuiteIAM) TestOpenIDServiceAccWithRolePolicy(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}

cmd/test-utils_test.go

@@ -2157,6 +2157,7 @@ func mustGetPoolEndpoints(args ...string) EndpointServerPools {
 		SetCount:     setCount,
 		DrivesPerSet: drivesPerSet,
 		Endpoints:    endpoints,
+		CmdLine:      strings.Join(args, " "),
 	}}
 }
 

cmd/tree-walk_test.go

@@ -141,6 +141,8 @@ func TestTreeWalk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create tmp directory: %s", err)
 	}
+	defer os.RemoveAll(fsDir)
+
 	endpoints := mustGetNewEndpoints(fsDir)
 	disk, err := newStorageAPI(endpoints[0])
 	if err != nil {
@@ -175,11 +177,6 @@ func TestTreeWalk(t *testing.T) {
 
 	// Simple test when marker is set.
 	testTreeWalkMarker(t, listDir, isLeaf, isLeafDir)
-
-	err = os.RemoveAll(fsDir)
-	if err != nil {
-		t.Fatal(err)
-	}
 }
 
 // Test if tree walk go-routine exits cleanly if tree walk is aborted because of timeout.
@@ -188,6 +185,7 @@ func TestTreeWalkTimeout(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create tmp directory: %s", err)
 	}
+	defer os.RemoveAll(fsDir)
 	endpoints := mustGetNewEndpoints(fsDir)
 	disk, err := newStorageAPI(endpoints[0])
 	if err != nil {
@@ -250,10 +248,6 @@ func TestTreeWalkTimeout(t *testing.T) {
 	if ok {
 		t.Error("Tree-walk go routine has not exited after timeout.")
 	}
-	err = os.RemoveAll(fsDir)
-	if err != nil {
-		t.Error(err)
-	}
 }
 
 // TestRecursiveWalk - tests if treeWalk returns entries correctly with and
@@ -264,6 +258,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create tmp directory: %s", err)
 	}
+	defer os.RemoveAll(fsDir1)
 
 	endpoints := mustGetNewEndpoints(fsDir1)
 	disk1, err := newStorageAPI(endpoints[0])
@@ -366,10 +361,6 @@ func TestRecursiveTreeWalk(t *testing.T) {
 			}
 		})
 	}
-	err = os.RemoveAll(fsDir1)
-	if err != nil {
-		t.Error(err)
-	}
 }
 
 func TestSortedness(t *testing.T) {
@@ -378,6 +369,7 @@ func TestSortedness(t *testing.T) {
 	if err != nil {
 		t.Errorf("Unable to create tmp directory: %s", err)
 	}
+	defer os.RemoveAll(fsDir1)
 
 	endpoints := mustGetNewEndpoints(fsDir1)
 	disk1, err := newStorageAPI(endpoints[0])
@@ -444,12 +436,6 @@ func TestSortedness(t *testing.T) {
 			t.Error(i+1, "Expected entries to be sort, but it wasn't")
 		}
 	}
-
-	// Remove directory created for testing
-	err = os.RemoveAll(fsDir1)
-	if err != nil {
-		t.Error(err)
-	}
 }
 
 func TestTreeWalkIsEnd(t *testing.T) {
@@ -458,6 +444,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
 	if err != nil {
 		t.Errorf("Unable to create tmp directory: %s", err)
 	}
+	defer os.RemoveAll(fsDir1)
 
 	endpoints := mustGetNewEndpoints(fsDir1)
 	disk1, err := newStorageAPI(endpoints[0])
@@ -526,10 +513,4 @@ func TestTreeWalkIsEnd(t *testing.T) {
 			t.Errorf("Test %d: Last entry %s, doesn't have EOF marker set", i, entry.entry)
 		}
 	}
-
-	// Remove directory created for testing
-	err = os.RemoveAll(fsDir1)
-	if err != nil {
-		t.Error(err)
-	}
 }

cmd/typed-errors.go

@@ -91,6 +91,12 @@ var errTooManyPolicies = errors.New("Only a single policy may be specified here.
 // error returned in IAM subsystem when an external users systems is configured.
 var errIAMActionNotAllowed = errors.New("Specified IAM action is not allowed")
 
+// error returned in IAM service account
+var errIAMServiceAccount = errors.New("Specified service account cannot be updated in this API call")
+
+// error returned in IAM service account is already used.
+var errIAMServiceAccountUsed = errors.New("Specified service account is used by another user")
+
 // error returned in IAM subsystem when IAM sub-system is still being initialized.
 var errIAMNotInitialized = errors.New("IAM sub-system is being initialized, please try again")
 

cmd/utils.go

@@ -43,6 +43,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/coreos/go-oidc"
 	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
@@ -58,6 +59,7 @@ import (
 	"github.com/minio/minio/internal/rest"
 	"github.com/minio/pkg/certs"
 	"github.com/minio/pkg/env"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -1160,3 +1162,142 @@ func newTLSConfig(getCert certs.GetCertificateFunc) *tls.Config {
 	}
 	return tlsConfig
 }
+
+/////////// Types and functions for OpenID IAM testing
+
+// OpenIDClientAppParams - contains openID client application params, used in
+// testing.
+type OpenIDClientAppParams struct {
+	ClientID, ClientSecret, ProviderURL, RedirectURL string
+}
+
+// MockOpenIDTestUserInteraction - tries to login to dex using provided credentials.
+// It performs the user's browser interaction to login and retrieves the auth
+// code from dex and exchanges it for a JWT.
+func MockOpenIDTestUserInteraction(ctx context.Context, pro OpenIDClientAppParams, username, password string) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	provider, err := oidc.NewProvider(ctx, pro.ProviderURL)
+	if err != nil {
+		return "", fmt.Errorf("unable to create provider: %v", err)
+	}
+
+	// Configure an OpenID Connect aware OAuth2 client.
+	oauth2Config := oauth2.Config{
+		ClientID:     pro.ClientID,
+		ClientSecret: pro.ClientSecret,
+		RedirectURL:  pro.RedirectURL,
+
+		// Discovery returns the OAuth2 endpoints.
+		Endpoint: provider.Endpoint(),
+
+		// "openid" is a required scope for OpenID Connect flows.
+		Scopes: []string{oidc.ScopeOpenID, "groups"},
+	}
+
+	state := fmt.Sprintf("x%dx", time.Now().Unix())
+	authCodeURL := oauth2Config.AuthCodeURL(state)
+	// fmt.Printf("authcodeurl: %s\n", authCodeURL)
+
+	var lastReq *http.Request
+	checkRedirect := func(req *http.Request, via []*http.Request) error {
+		// fmt.Printf("CheckRedirect:\n")
+		// fmt.Printf("Upcoming: %s %s\n", req.Method, req.URL.String())
+		// for i, c := range via {
+		// 	fmt.Printf("Sofar %d: %s %s\n", i, c.Method, c.URL.String())
+		// }
+		// Save the last request in a redirect chain.
+		lastReq = req
+		// We do not follow redirect back to client application.
+		if req.URL.Path == "/oauth_callback" {
+			return http.ErrUseLastResponse
+		}
+		return nil
+	}
+
+	dexClient := http.Client{
+		CheckRedirect: checkRedirect,
+	}
+
+	u, err := url.Parse(authCodeURL)
+	if err != nil {
+		return "", fmt.Errorf("url parse err: %v", err)
+	}
+
+	// Start the user auth flow. This page would present the login with
+	// email or LDAP option.
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return "", fmt.Errorf("new request err: %v", err)
+	}
+	_, err = dexClient.Do(req)
+	// fmt.Printf("Do: %#v %#v\n", resp, err)
+	if err != nil {
+		return "", fmt.Errorf("auth url request err: %v", err)
+	}
+
+	// Modify u to choose the ldap option
+	u.Path += "/ldap"
+	// fmt.Println(u)
+
+	// Pick the LDAP login option. This would return a form page after
+	// following some redirects. `lastReq` would be the URL of the form
+	// page, where we need to POST (submit) the form.
+	req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return "", fmt.Errorf("new request err (/ldap): %v", err)
+	}
+	_, err = dexClient.Do(req)
+	// fmt.Printf("Fetch LDAP login page: %#v %#v\n", resp, err)
+	if err != nil {
+		return "", fmt.Errorf("request err: %v", err)
+	}
+	// {
+	// 	bodyBuf, err := ioutil.ReadAll(resp.Body)
+	// 	if err != nil {
+	// 		return "", fmt.Errorf("Error reading body: %v", err)
+	// 	}
+	// 	fmt.Printf("bodyBuf (for LDAP login page): %s\n", string(bodyBuf))
+	// }
+
+	// Fill the login form with our test creds:
+	// fmt.Printf("login form url: %s\n", lastReq.URL.String())
+	formData := url.Values{}
+	formData.Set("login", username)
+	formData.Set("password", password)
+	req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode()))
+	if err != nil {
+		return "", fmt.Errorf("new request err (/login): %v", err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	_, err = dexClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("post form err: %v", err)
+	}
+	// fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header)
+	// bodyBuf, err := ioutil.ReadAll(resp.Body)
+	// if err != nil {
+	// 	return "", fmt.Errorf("Error reading body: %v", err)
+	// }
+	// fmt.Printf("resp body: %s\n", string(bodyBuf))
+	// fmt.Printf("lastReq: %#v\n", lastReq.URL.String())
+
+	// On form submission, the last redirect response contains the auth
+	// code, which we now have in `lastReq`. Exchange it for a JWT id_token.
+	q := lastReq.URL.Query()
+	// fmt.Printf("lastReq.URL: %#v q: %#v\n", lastReq.URL, q)
+	code := q.Get("code")
+	oauth2Token, err := oauth2Config.Exchange(ctx, code)
+	if err != nil {
+		return "", fmt.Errorf("unable to exchange code for id token: %v", err)
+	}
+
+	rawIDToken, ok := oauth2Token.Extra("id_token").(string)
+	if !ok {
+		return "", fmt.Errorf("id_token not found!")
+	}
+
+	// fmt.Printf("TOKEN: %s\n", rawIDToken)
+	return rawIDToken, nil
+}

cmd/xl-storage-format-v2-legacy.go

@@ -83,7 +83,22 @@ func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error)
 	switch v {
 	// We accept un-set as latest version.
 	case 0, xlMetaVersion:
-		return j.UnmarshalMsg(bts)
+		o, err = j.UnmarshalMsg(bts)
+
+		// Clean up PartEtags on v1
+		if j.ObjectV2 != nil {
+			allEmpty := true
+			for _, tag := range j.ObjectV2.PartETags {
+				if len(tag) != 0 {
+					allEmpty = false
+					break
+				}
+			}
+			if allEmpty {
+				j.ObjectV2.PartETags = nil
+			}
+		}
+		return o, err
 	}
 	return bts, fmt.Errorf("unknown xlMetaVersion: %d", v)
 }

cmd/xl-storage-format-v2.go

@@ -264,9 +264,13 @@ func (x xlMetaV2VersionHeader) String() string {
 
 // matchesNotStrict returns whether x and o have both have non-zero version,
 // their versions match and their type match.
+// If they have zero version, modtime must match.
 func (x xlMetaV2VersionHeader) matchesNotStrict(o xlMetaV2VersionHeader) bool {
-	return x.VersionID != [16]byte{} &&
-		x.VersionID == o.VersionID &&
+	if x.VersionID == [16]byte{} {
+		return x.VersionID == o.VersionID &&
+			x.Type == o.Type && o.ModTime == x.ModTime
+	}
+	return x.VersionID == o.VersionID &&
 		x.Type == o.Type
 }
 
@@ -508,7 +512,14 @@ func (j *xlMetaV2Object) Signature() [4]byte {
 	c.ErasureIndex = 0
 
 	// Nil 0 size allownil, so we don't differentiate between nil and 0 len.
-	if len(c.PartETags) == 0 {
+	allEmpty := true
+	for _, tag := range c.PartETags {
+		if len(tag) != 0 {
+			allEmpty = false
+			break
+		}
+	}
+	if allEmpty {
 		c.PartETags = nil
 	}
 	if len(c.PartActualSizes) == 0 {
@@ -927,7 +938,7 @@ func (x *xlMetaV2) loadLegacy(buf []byte) error {
 		return errors.New("unknown major metadata version")
 	}
 	if allMeta == nil {
-		return errCorruptedFormat
+		return errFileCorrupt
 	}
 	// bts will shrink as we decode.
 	bts := allMeta
@@ -1676,6 +1687,9 @@ func (x xlMetaV2) ListVersions(volume, path string) ([]FileInfo, error) {
 // Quorum should be > 1 and <= len(versions).
 // If strict is set to false, entries that match type
 func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) {
+	if quorum <= 0 {
+		quorum = 1
+	}
 	if len(versions) < quorum || len(versions) == 0 {
 		return nil
 	}
@@ -1686,14 +1700,16 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 		// No need for non-strict checks if quorum is 1.
 		strict = true
 	}
+	// Shallow copy input
+	versions = append(make([][]xlMetaV2ShallowVersion, 0, len(versions)), versions...)
+
 	// Our result
 	merged = make([]xlMetaV2ShallowVersion, 0, len(versions[0]))
 	tops := make([]xlMetaV2ShallowVersion, len(versions))
 	for {
 		// Step 1 create slice with all top versions.
 		tops = tops[:0]
-		var topSig [4]byte
-		var topID [16]byte
+		var topSig xlMetaV2VersionHeader
 		consistent := true // Are all signatures consistent (shortcut)
 		for _, vers := range versions {
 			if len(vers) == 0 {
@@ -1703,10 +1719,9 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 			ver := vers[0]
 			if len(tops) == 0 {
 				consistent = true
-				topSig = ver.header.Signature
-				topID = ver.header.VersionID
+				topSig = ver.header
 			} else {
-				consistent = consistent && topSig == ver.header.Signature && topID == ver.header.VersionID
+				consistent = consistent && ver.header == topSig
 			}
 			tops = append(tops, vers[0])
 		}
@@ -1732,7 +1747,7 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 					continue
 				}
 				if i == 0 || ver.header.sortsBefore(latest.header) {
-					if i == 0 {
+					if i == 0 || latestCount == 0 {
 						latestCount = 1
 					} else if !strict && ver.header.matchesNotStrict(latest.header) {
 						latestCount++
@@ -1744,12 +1759,45 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 				}
 
 				// Mismatch, but older.
-				if !strict && ver.header.matchesNotStrict(latest.header) {
-					// If non-nil version ID and it matches, assume match, but keep newest.
-					if ver.header.sortsBefore(latest.header) {
-						latest = ver
-					}
+				if latestCount > 0 && !strict && ver.header.matchesNotStrict(latest.header) {
 					latestCount++
+					continue
+				}
+				if latestCount > 0 && ver.header.VersionID == latest.header.VersionID {
+					// Version IDs match, but otherwise unable to resolve.
+					// We are either strict, or don't have enough information to match.
+					// Switch to a pure counting algo.
+					x := make(map[xlMetaV2VersionHeader]int, len(tops))
+					for _, a := range tops {
+						if a.header.VersionID != ver.header.VersionID {
+							continue
+						}
+						if !strict {
+							a.header.Signature = [4]byte{}
+						}
+						x[a.header]++
+					}
+					latestCount = 0
+					for k, v := range x {
+						if v < latestCount {
+							continue
+						}
+						if v == latestCount && latest.header.sortsBefore(k) {
+							// Tiebreak, use sort.
+							continue
+						}
+						for _, a := range tops {
+							hdr := a.header
+							if !strict {
+								hdr.Signature = [4]byte{}
+							}
+							if hdr == k {
+								latest = a
+							}
+						}
+						latestCount = v
+					}
+					break
 				}
 			}
 			if latestCount >= quorum {

cmd/xl-storage-format-v2_test.go

@@ -19,11 +19,15 @@ package cmd
 
 import (
 	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
 	"sort"
 	"testing"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/klauspost/compress/zip"
 	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	xhttp "github.com/minio/minio/internal/http"
@@ -476,3 +480,272 @@ func Test_xlMetaV2Shallow_Load(t *testing.T) {
 		test(t, &xl)
 	})
 }
+
+func Test_mergeXLV2Versions(t *testing.T) {
+	dataZ, err := ioutil.ReadFile("testdata/xl-meta-consist.zip")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var vers [][]xlMetaV2ShallowVersion
+	zr, err := zip.NewReader(bytes.NewReader(dataZ), int64(len(dataZ)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, file := range zr.File {
+		if file.UncompressedSize64 == 0 {
+			continue
+		}
+		in, err := file.Open()
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer in.Close()
+		buf, err := io.ReadAll(in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var xl xlMetaV2
+		err = xl.LoadOrConvert(buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+		vers = append(vers, xl.versions)
+	}
+	for _, v2 := range vers {
+		for _, ver := range v2 {
+			b, _ := json.Marshal(ver.header)
+			t.Log(string(b))
+			var x xlMetaV2Version
+			_, _ = x.unmarshalV(0, ver.meta)
+			b, _ = json.Marshal(x)
+			t.Log(string(b), x.getSignature())
+		}
+	}
+
+	for i := range vers {
+		t.Run(fmt.Sprintf("non-strict-q%d", i), func(t *testing.T) {
+			merged := mergeXLV2Versions(i, false, vers...)
+			if len(merged) == 0 {
+				t.Error("Did not get any results")
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("strict-q%d", i), func(t *testing.T) {
+			merged := mergeXLV2Versions(i, true, vers...)
+			if len(merged) == 0 {
+				t.Error("Did not get any results")
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("signature-q%d", i), func(t *testing.T) {
+			// Mutate signature, non strict
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.Signature = [4]byte{byte(i + 10), 0, 0, 0}
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, false, vMod...)
+			if len(merged) == 0 {
+				t.Error("Did not get any results")
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("modtime-q%d", i), func(t *testing.T) {
+			// Mutate modtime, but rest is consistent.
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.ModTime += int64(i)
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, false, vMod...)
+			if len(merged) == 0 && i < 2 {
+				t.Error("Did not get any results")
+				return
+			}
+			if len(merged) > 0 && i >= 2 {
+				t.Error("Got unexpected results")
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("flags-q%d", i), func(t *testing.T) {
+			// Mutate signature, non strict
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.Flags += xlFlags(i)
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, false, vMod...)
+			if len(merged) == 0 {
+				t.Error("Did not get any results")
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("versionid-q%d", i), func(t *testing.T) {
+			// Mutate signature, non strict
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.VersionID[0] += byte(i)
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, false, vMod...)
+			if len(merged) == 0 && i < 2 {
+				t.Error("Did not get any results")
+				return
+			}
+			if len(merged) > 0 && i >= 2 {
+				t.Error("Got unexpected results")
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("strict-signature-q%d", i), func(t *testing.T) {
+			// Mutate signature, non strict
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.Signature = [4]byte{byte(i + 10), 0, 0, 0}
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, true, vMod...)
+			if len(merged) == 0 && i < 2 {
+				t.Error("Did not get any results")
+				return
+			}
+			if len(merged) > 0 && i >= 2 {
+				t.Error("Got unexpected results")
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("strict-modtime-q%d", i), func(t *testing.T) {
+			// Mutate signature, non strict
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.ModTime += int64(i + 10)
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, true, vMod...)
+			if len(merged) == 0 && i < 2 {
+				t.Error("Did not get any results")
+				return
+			}
+			if len(merged) > 0 && i >= 2 {
+				t.Error("Got unexpected results", len(merged), merged[0].header)
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("strict-flags-q%d", i), func(t *testing.T) {
+			// Mutate signature, non strict
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.Flags += xlFlags(i + 10)
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, true, vMod...)
+			if len(merged) == 0 && i < 2 {
+				t.Error("Did not get any results")
+				return
+			}
+			if len(merged) > 0 && i >= 2 {
+				t.Error("Got unexpected results", len(merged))
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+		t.Run(fmt.Sprintf("strict-type-q%d", i), func(t *testing.T) {
+			// Mutate signature, non strict
+			vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers))
+			for i, ver := range vers {
+				newVers := make([]xlMetaV2ShallowVersion, 0, len(ver))
+				for _, v := range ver {
+					v.header.Type += VersionType(i + 10)
+					newVers = append(newVers, v)
+				}
+				vMod = append(vMod, newVers)
+			}
+			merged := mergeXLV2Versions(i, true, vMod...)
+			if len(merged) == 0 && i < 2 {
+				t.Error("Did not get any results")
+				return
+			}
+			if len(merged) > 0 && i >= 2 {
+				t.Error("Got unexpected results", len(merged))
+				return
+			}
+			for _, ver := range merged {
+				if ver.header.Type == invalidVersionType {
+					t.Errorf("Invalid result returned: %v", ver.header)
+				}
+			}
+		})
+	}
+}

cmd/xl-storage.go

@@ -41,12 +41,10 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/color"
-	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/disk"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
-	"github.com/minio/pkg/env"
 	"github.com/yargevad/filepathx"
 )
 
@@ -60,6 +58,9 @@ const (
 	// For hardrives it is possible to set this to a lower value to avoid any
 	// spike in latency. But currently we are simply keeping it optimal for SSDs.
 
+	// bigFileThreshold is the point where we add readahead to put operations.
+	bigFileThreshold = 128 * humanize.MiByte
+
 	// XL metadata file carries per object metadata.
 	xlStorageFormatFile = "xl.meta"
 )
@@ -107,6 +108,8 @@ type xlStorage struct {
 	diskInfoCache timedValue
 	sync.RWMutex
 
+	formatData []byte
+
 	// mutex to prevent concurrent read operations overloading walks.
 	walkMu     sync.Mutex
 	walkReadMu sync.Mutex
@@ -202,99 +205,96 @@ func newLocalXLStorage(path string) (*xlStorage, error) {
 	})
 }
 
-// Sanitize - sanitizes the `format.json`, cleanup tmp.
-// all other future cleanups should be added here.
-func (s *xlStorage) Sanitize() error {
-	if err := formatErasureMigrate(s.diskPath); err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-
-	return formatErasureCleanupTmp(s.diskPath)
-}
-
 // Initialize a new storage disk.
-func newXLStorage(ep Endpoint) (*xlStorage, error) {
+func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 	path := ep.Path
-	var err error
 	if path, err = getValidPath(path); err != nil {
 		return nil, err
 	}
 
 	var rootDisk bool
-	if env.Get("MINIO_CI_CD", "") != "" {
+	if globalIsCICD {
 		rootDisk = true
 	} else {
 		rootDisk, err = disk.IsRootDisk(path, SlashSeparator)
 		if err != nil {
 			return nil, err
 		}
-		if !rootDisk {
-			// If for some reason we couldn't detect the
-			// root disk use - MINIO_ROOTDISK_THRESHOLD_SIZE
-			// to figure out if the disk is root disk or not.
-			if rootDiskSize := env.Get(config.EnvRootDiskThresholdSize, ""); rootDiskSize != "" {
-				info, err := disk.GetInfo(path)
-				if err != nil {
-					return nil, err
-				}
-				size, err := humanize.ParseBytes(rootDiskSize)
-				if err != nil {
-					return nil, err
-				}
-				// size of the disk is less than the threshold or
-				// equal to the size of the disk at path, treat
-				// such disks as rootDisks and reject them.
-				rootDisk = info.Total <= size
+		if !rootDisk && globalRootDiskThreshold > 0 {
+			// If for some reason we couldn't detect the root disk
+			// use - MINIO_ROOTDISK_THRESHOLD_SIZE to figure out if
+			// this disk is a root disk.
+			info, err := disk.GetInfo(path)
+			if err != nil {
+				return nil, err
 			}
+
+			// treat those disks with size less than or equal to the
+			// threshold as rootDisks.
+			rootDisk = info.Total <= globalRootDiskThreshold
 		}
 	}
 
-	p := &xlStorage{
+	s = &xlStorage{
 		diskPath:   path,
 		endpoint:   ep,
-		globalSync: env.Get(config.EnvFSOSync, config.EnableOff) == config.EnableOn,
+		globalSync: globalFSOSync,
 		rootDisk:   rootDisk,
 		poolIndex:  -1,
 		setIndex:   -1,
 		diskIndex:  -1,
 	}
 
-	// Create all necessary bucket folders if possible.
-	if err = p.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket, minioMetaSpeedTestBucket); err != nil {
+	go formatErasureCleanupTmp(s.diskPath) // cleanup any old data.
+
+	formatData, formatFi, err := formatErasureMigrate(s.diskPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		if os.IsPermission(err) {
+			return nil, errDiskAccessDenied
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		}
 		return nil, err
 	}
+	s.formatData = formatData
+	s.formatFileInfo = formatFi
 
-	// Check if backend is writable and supports O_DIRECT
-	var rnd [8]byte
-	_, _ = rand.Read(rnd[:])
-	tmpFile := ".writable-check-" + hex.EncodeToString(rnd[:]) + ".tmp"
-	filePath := pathJoin(p.diskPath, minioMetaTmpBucket, tmpFile)
-	w, err := OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
-	if err != nil {
-		switch {
-		case isSysErrInvalidArg(err):
-			return p, errUnsupportedDisk
-		case osIsPermission(err):
-			return p, errDiskAccessDenied
-		case isSysErrIO(err):
-			return p, errFaultyDisk
-		case isSysErrNotDir(err):
-			return p, errDiskNotDir
+	if len(s.formatData) == 0 { // Unformatted disk check if O_DIRECT is supported.
+		// Check if backend is writable and supports O_DIRECT
+		var rnd [32]byte
+		_, _ = rand.Read(rnd[:])
+		filePath := pathJoin(s.diskPath, ".writable-check-"+hex.EncodeToString(rnd[:])+".tmp")
+		w, err := s.openFileDirect(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL)
+		if err != nil {
+			return s, err
 		}
-		return p, err
-	}
-	if _, err = w.Write(alignedBuf); err != nil {
+		_, err = w.Write(alignedBuf)
 		w.Close()
-		if isSysErrInvalidArg(err) {
-			return p, errUnsupportedDisk
+		if err != nil {
+			if isSysErrInvalidArg(err) {
+				return s, errUnsupportedDisk
+			}
+			return s, err
 		}
-		return p, err
+		Remove(filePath)
+
+		// Create all necessary bucket folders if possible.
+		if err = makeFormatErasureMetaVolumes(s); err != nil {
+			return nil, err
+		}
+	} else {
+		format := &formatErasureV3{}
+		json := jsoniter.ConfigCompatibleWithStandardLibrary
+		if err = json.Unmarshal(s.formatData, &format); err != nil {
+			return s, errCorruptedFormat
+		}
+		s.diskID = format.Erasure.This
+		s.formatLastCheck = time.Now()
+		s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1
 	}
-	w.Close()
-	Remove(filePath)
 
 	// Success.
-	return p, nil
+	return s, nil
 }
 
 // getDiskInfo returns given disk information.
@@ -302,7 +302,6 @@ func getDiskInfo(diskPath string) (di disk.Info, err error) {
 	if err = checkPathLength(diskPath); err == nil {
 		di, err = disk.GetInfo(diskPath)
 	}
-
 	switch {
 	case osIsNotExist(err):
 		err = errDiskNotFound
@@ -365,7 +364,7 @@ func (s *xlStorage) SetDiskLoc(poolIdx, setIdx, diskIdx int) {
 func (s *xlStorage) Healing() *healingTracker {
 	healingFile := pathJoin(s.diskPath, minioMetaBucket,
 		bucketMetaPrefix, healingTrackerFilename)
-	b, err := xioutil.ReadFile(healingFile)
+	b, err := ioutil.ReadFile(healingFile)
 	if err != nil {
 		return nil
 	}
@@ -618,8 +617,8 @@ func (s *xlStorage) GetDiskID() (string, error) {
 	fileInfo := s.formatFileInfo
 	lastCheck := s.formatLastCheck
 
-	// check if we have a valid disk ID that is less than 1 second old.
-	if fileInfo != nil && diskID != "" && time.Since(lastCheck) <= time.Second {
+	// check if we have a valid disk ID that is less than 1 seconds old.
+	if fileInfo != nil && diskID != "" && time.Since(lastCheck) <= 1*time.Second {
 		s.RUnlock()
 		return diskID, nil
 	}
@@ -639,7 +638,7 @@ func (s *xlStorage) GetDiskID() (string, error) {
 	}
 
 	formatFile := pathJoin(s.diskPath, minioMetaBucket, formatConfigFile)
-	b, err := xioutil.ReadFile(formatFile)
+	b, err := ioutil.ReadFile(formatFile)
 	if err != nil {
 		// If the disk is still not initialized.
 		if osIsNotExist(err) {
@@ -670,6 +669,7 @@ func (s *xlStorage) GetDiskID() (string, error) {
 
 	s.Lock()
 	defer s.Unlock()
+	s.formatData = b
 	s.diskID = format.Erasure.This
 	s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1
 	s.formatFileInfo = fi
@@ -885,13 +885,19 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 	var (
 		dataDir     string
 		lastVersion bool
+		updated     bool
 	)
 
 	for _, fi := range fis {
 		dataDir, lastVersion, err = xlMeta.DeleteVersion(fi)
 		if err != nil {
+			if !fi.Deleted && (err == errFileNotFound || err == errFileVersionNotFound) {
+				// Ignore these since
+				continue
+			}
 			return err
 		}
+		updated = true
 		if dataDir != "" {
 			versionID := fi.VersionID
 			if versionID == "" {
@@ -915,6 +921,10 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 	}
 
 	if !lastVersion {
+		if !updated {
+			return nil
+		}
+
 		buf, err = xlMeta.AppendTo(metaDataPoolGet())
 		defer metaDataPoolPut(buf)
 		if err != nil {
@@ -1401,6 +1411,17 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 // This API is meant to be used on files which have small memory footprint, do
 // not use this on large files as it would cause server to crash.
 func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
+	// Specific optimization to avoid re-read from the drives for `format.json`
+	// in-case the caller is a network operation.
+	if volume == minioMetaBucket && path == formatConfigFile {
+		s.RLock()
+		formatData := make([]byte, len(s.formatData))
+		copy(formatData, s.formatData)
+		s.RUnlock()
+		if len(formatData) > 0 {
+			return formatData, nil
+		}
+	}
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
 		return nil, err
@@ -1521,6 +1542,30 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of
 	return int64(len(buffer)), nil
 }
 
+func (s *xlStorage) openFileDirect(path string, mode int) (f *os.File, err error) {
+	// Create top level directories if they don't exist.
+	// with mode 0o777 mkdir honors system umask.
+	mkdirAll(pathutil.Dir(path), 0o777) // don't need to fail here
+
+	w, err := OpenFileDirectIO(path, mode, 0o666)
+	if err != nil {
+		switch {
+		case isSysErrInvalidArg(err):
+			return nil, errUnsupportedDisk
+		case osIsPermission(err):
+			return nil, errDiskAccessDenied
+		case isSysErrIO(err):
+			return nil, errFaultyDisk
+		case isSysErrNotDir(err):
+			return nil, errDiskNotDir
+		case os.IsNotExist(err):
+			return nil, errDiskNotFound
+		}
+	}
+
+	return w, nil
+}
+
 func (s *xlStorage) openFileSync(filePath string, mode int) (f *os.File, err error) {
 	// Create top level directories if they don't exist.
 	// with mode 0777 mkdir honors system umask.
@@ -1636,7 +1681,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	}
 
 	alignment := offset%xioutil.DirectioAlignSize == 0
-	if !alignment {
+	if !alignment || globalAPIConfig.isDisableODirect() {
 		if err = disk.DisableDirectIO(file); err != nil {
 			file.Close()
 			return nil, err
@@ -1981,6 +2026,14 @@ func (s *xlStorage) Delete(ctx context.Context, volume string, path string, recu
 	return s.deleteFile(volumeDir, filePath, recursive)
 }
 
+func skipAccessChecks(volume string) (ok bool) {
+	switch volume {
+	case minioMetaTmpBucket, minioMetaBucket, minioMetaMultipartBucket, minioMetaTmpDeletedBucket:
+		ok = true
+	}
+	return ok
+}
+
 // RenameData - rename source path to destination path atomically, metadata and data directory.
 func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
 	defer func() {
@@ -2007,23 +2060,27 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		return err
 	}
 
-	// Stat a volume entry.
-	if err = Access(srcVolumeDir); err != nil {
-		if osIsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return errFaultyDisk
+	if !skipAccessChecks(srcVolume) {
+		// Stat a volume entry.
+		if err = Access(srcVolumeDir); err != nil {
+			if osIsNotExist(err) {
+				return errVolumeNotFound
+			} else if isSysErrIO(err) {
+				return errFaultyDisk
+			}
+			return err
 		}
-		return err
 	}
 
-	if err = Access(dstVolumeDir); err != nil {
-		if osIsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return errFaultyDisk
+	if !skipAccessChecks(dstVolume) {
+		if err = Access(dstVolumeDir); err != nil {
+			if osIsNotExist(err) {
+				return errVolumeNotFound
+			} else if isSysErrIO(err) {
+				return errFaultyDisk
+			}
+			return err
 		}
-		return err
 	}
 
 	srcFilePath := pathutil.Join(srcVolumeDir, pathJoin(srcPath, xlStorageFormatFile))
@@ -2234,9 +2291,6 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 					s.deleteFile(dstVolumeDir, legacyDataPath, true)
 				}
 				s.deleteFile(dstVolumeDir, dstDataPath, false)
-				if err != errFileNotFound {
-					logger.LogIf(ctx, err)
-				}
 				return osErrToFileErr(err)
 			}
 		}
@@ -2248,10 +2302,6 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 				s.deleteFile(dstVolumeDir, legacyDataPath, true)
 			}
 			s.deleteFile(dstVolumeDir, dstFilePath, false)
-
-			if err != errFileNotFound {
-				logger.LogIf(ctx, err)
-			}
 			return osErrToFileErr(err)
 		}
 
@@ -2269,8 +2319,6 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 				s.deleteFile(dstVolumeDir, legacyDataPath, true)
 			}
 			s.deleteFile(dstVolumeDir, dstFilePath, false)
-
-			logger.LogIf(ctx, err)
 			return err
 		}
 	}
@@ -2293,25 +2341,27 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 	if err != nil {
 		return err
 	}
-	// Stat a volume entry.
-	if err = Access(srcVolumeDir); err != nil {
-		if osIsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return errFaultyDisk
+	if !skipAccessChecks(srcVolume) {
+		// Stat a volume entry.
+		if err = Access(srcVolumeDir); err != nil {
+			if osIsNotExist(err) {
+				return errVolumeNotFound
+			} else if isSysErrIO(err) {
+				return errFaultyDisk
+			}
+			return err
 		}
-		return err
 	}
-
-	if err = Access(dstVolumeDir); err != nil {
-		if osIsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return errFaultyDisk
+	if !skipAccessChecks(dstVolume) {
+		if err = Access(dstVolumeDir); err != nil {
+			if osIsNotExist(err) {
+				return errVolumeNotFound
+			} else if isSysErrIO(err) {
+				return errFaultyDisk
+			}
+			return err
 		}
-		return err
 	}
-
 	srcIsDir := HasSuffix(srcPath, SlashSeparator)
 	dstIsDir := HasSuffix(dstPath, SlashSeparator)
 	// Either src and dst have to be directories or files, else return error.

cmd/xl-storage_test.go

@@ -30,6 +30,8 @@ import (
 	"strings"
 	"syscall"
 	"testing"
+
+	"github.com/google/uuid"
 )
 
 func TestCheckPathLength(t *testing.T) {
@@ -126,13 +128,14 @@ func newXLStorageTestSetup() (*xlStorageDiskIDCheck, string, error) {
 	if err != nil {
 		return nil, "", err
 	}
+
 	// Create a sample format.json file
-	err = storage.WriteAll(context.Background(), minioMetaBucket, formatConfigFile, []byte(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`))
-	if err != nil {
+	if err = storage.WriteAll(context.Background(), minioMetaBucket, formatConfigFile, []byte(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`)); err != nil {
 		return nil, "", err
 	}
+
 	disk := newXLStorageDiskIDCheck(storage)
-	disk.diskID = "da017d62-70e3-45f1-8a1a-587707e69ad1"
+	disk.SetDiskID("da017d62-70e3-45f1-8a1a-587707e69ad1")
 	return disk, diskPath, nil
 }
 
@@ -1665,6 +1668,98 @@ func TestXLStorageRenameFile(t *testing.T) {
 	}
 }
 
+// TestXLStorageDeleteVersion will test if version deletes and bulk deletes work as expected.
+func TestXLStorageDeleteVersion(t *testing.T) {
+	// create xlStorage test setup
+	xl, path, err := newXLStorageTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create xlStorage test setup, %s", err)
+	}
+	defer os.RemoveAll(path)
+	ctx := context.Background()
+
+	volume := "myvol-vol"
+	object := "my-object"
+	if err := xl.MakeVol(ctx, volume); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+	var versions [50]string
+	for i := range versions {
+		versions[i] = uuid.New().String()
+		fi := FileInfo{
+			Name: object, Volume: volume, VersionID: versions[i], ModTime: UTCNow(), DataDir: uuid.NewString(), Size: 10000,
+			Erasure: ErasureInfo{
+				Algorithm:    erasureAlgorithm,
+				DataBlocks:   4,
+				ParityBlocks: 4,
+				BlockSize:    blockSizeV2,
+				Index:        1,
+				Distribution: []int{0, 1, 2, 3, 4, 5, 6, 7},
+				Checksums:    nil,
+			},
+		}
+		if err := xl.WriteMetadata(ctx, volume, object, fi); err != nil {
+			t.Fatalf("Unable to create object, %s", err)
+		}
+	}
+	var deleted [len(versions)]bool
+	checkVerExist := func(t testing.TB) {
+		t.Helper()
+		for i := range versions {
+			shouldExist := !deleted[i]
+			fi, err := xl.ReadVersion(ctx, volume, object, versions[i], false)
+			if shouldExist {
+				if err != nil {
+					t.Fatalf("Version %s should exist, but got err %v", versions[i], err)
+				}
+				return
+			}
+			if err != errFileVersionNotFound {
+				t.Fatalf("Version %s should not exist, but returned: %#v", versions[i], fi)
+			}
+		}
+	}
+
+	// Delete version 0...
+	checkVerExist(t)
+	err = xl.DeleteVersion(ctx, volume, object, FileInfo{Name: object, Volume: volume, VersionID: versions[0]}, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	deleted[0] = true
+	checkVerExist(t)
+
+	// Delete 10 in bulk, including a non-existing.
+	fis := []FileInfoVersions{{Name: object, Volume: volume}}
+	for i := range versions[:10] {
+		fis[0].Versions = append(fis[0].Versions, FileInfo{Name: object, Volume: volume, VersionID: versions[i]})
+		deleted[i] = true
+	}
+	errs := xl.DeleteVersions(ctx, volume, fis)
+	if errs[0] != nil {
+		t.Fatalf("expected nil error, got %v", errs[0])
+	}
+	checkVerExist(t)
+
+	// Delete them all... (some again)
+	fis[0].Versions = nil
+	for i := range versions[:] {
+		fis[0].Versions = append(fis[0].Versions, FileInfo{Name: object, Volume: volume, VersionID: versions[i]})
+		deleted[i] = true
+	}
+	errs = xl.DeleteVersions(ctx, volume, fis)
+	if errs[0] != nil {
+		t.Fatalf("expected nil error, got %v", errs[0])
+	}
+	checkVerExist(t)
+
+	// Meta should be deleted now...
+	fi, err := xl.ReadVersion(ctx, volume, object, "", false)
+	if err != errFileNotFound {
+		t.Fatalf("Object %s should not exist, but returned: %#v", object, fi)
+	}
+}
+
 // TestXLStorage xlStorage.StatInfoFile()
 func TestXLStorageStatInfoFile(t *testing.T) {
 	// create xlStorage test setup

docs/bucket/replication/README.md

@@ -178,6 +178,8 @@ When an object is deleted from the source bucket, the corresponding replica vers
 
 Note that due to this extension behavior, AWS SDK's may not support the extension functionality pertaining to replicating versioned deletes.
 
+Note that just like with [AWS](https://docs.aws.amazon.com/AmazonS3/latest/userguide/delete-marker-replication.html), Delete marker replication is disallowed in MinIO when the replication rule has tags.
+
 To add a replication rule allowing both delete marker replication, versioned delete replication or both specify the --replicate flag with comma separated values as in the example below.
 
 Additional permission of "s3:ReplicateDelete" action would need to be specified on the access key configured for the target cluster if Delete Marker replication or versioned delete replication is enabled.

docs/bucket/versioning/DESIGN.md

@@ -2,31 +2,47 @@
 
 ## Description of `xl.meta`
 
-`xl.meta` is a new self describing backend format used by MinIO to support AWS S3 compatible versioning. This file is the source of truth for each `version` at rest. `xl.meta` is a msgpack file serialized from a well defined data structure. To understand `xl.meta` here are the few things to start with
+`xl.meta` is a new self describing backend format used by MinIO to support AWS S3 compatible versioning.
+This file is the source of truth for each `version` at rest. `xl.meta` is a msgpack file serialized from a 
+well defined data structure. To understand `xl.meta` here are the few things to start with
 
-`xl.meta` carries first 8 bytes an XL header which describes the current format and the format version, allowing the unmarshaller's to automatically use the right data structures to parse the subsequent content in the stream.
+`xl.meta` carries first 8 bytes an XL header which describes the current format and the format version, 
+allowing the unmarshaller's to automatically use the right data structures to parse the subsequent content in the stream.
 
-These are the current entries
-```go
-var (
-        // XL header specifies the format
-        // one extra byte left for future use
-        xlHeader = [4]byte{'X', 'L', '2', ' '}
+### v1.0
 
-        // XLv2 version 1 specifies the current
-        // version of the XLv2 format, 3 extra bytes
-        // left for future use.
-        xlVersionV1 = [4]byte{'1', ' ', ' ', ' '}
-)
-```
+| Entry     | Encoding    | Content
+| ----------|-------------|----------------------------------------
+| xlHeader  | [4]byte     | `'X', 'L', '2', ' '`
+| xlVersion | [4]byte     | `'1', ' ', ' ', ' '`
+| xlMetaV2  | msgp object | All versions as single messagepack object
+| [EOF] | |
 
-Once the header is validated, we proceed to the actual data structure of the `xl.meta` format. `xl.meta` carries three types of object entries which designate the type of version object stored.
+### v1.1+
+
+Version 1.1 added inline data, which will be placed after the metadata.
+
+Therefore, the metadata is wrapped as a binary array for easy skipping.
+
+| Entry          | Encoding       | Content
+| ---------------|----------------|----------------------------------------
+| xlHeader       | [4]byte        | `'X', 'L', '2', ' '`
+| xlVersionMajor | uint16         | Major xl-meta version.
+| xlVersionMinor | uint16         | Minor xl-meta version.
+| xlMetaV2       | msgp bin array | Bin array with serialized metadata
+| crc            | msgp uint      | Lower 32 bits of 64 bit xxhash of previous array contents (v1.2+ only)
+| inline data    | binary         | Inline data if any, see Inline Data section for encoding.  
+| [EOF] | |
+
+## v1.0-v1.2 Versions
+
+`xl.meta` carries three types of object entries which designate the type of version object stored.
 
 - ObjectType (default)
 - LegacyObjectType (preserves existing deployments and older xl.json format)
 - DeleteMarker (a versionId to capture the DELETE sequences implemented primarily for AWS spec compatibility)
 
-A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/minio/minio/blob/master/docs/bucket/versioning/xl-meta.go) program.
+A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/minio/minio/tree/master/docs/debugging#decoding-metadata) program.
 ```json
 {
   "Versions": [
@@ -40,25 +56,12 @@ A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` usin
         "EcN": 2,
         "EcBSize": 10485760,
         "EcIndex": 3,
-        "EcDist": [
-          3,
-          4,
-          1,
-          2
-        ],
+        "EcDist": [3, 4, 1, 2],
         "CSumAlgo": 1,
-        "PartNums": [
-          1
-        ],
-        "PartETags": [
-          ""
-        ],
-        "PartSizes": [
-          314
-        ],
-        "PartASizes": [
-          282
-        ],
+        "PartNums": [1],
+        "PartETags": [""],
+        "PartSizes": [314],
+        "PartASizes": [282],
         "Size": 314,
         "MTime": 1591820730,
         "MetaSys": {
@@ -77,3 +80,58 @@ A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` usin
   ]
 }
 ```
+
+### v1.3+ versions
+
+Version 1.3 introduces changes to help with [faster metadata reads and updates](https://blog.min.io/minio-versioning-metadata-deep-dive/)
+
+| Entry           | Encoding                    | Content
+| ----------------|-----------------------------|----------------------------------------
+| xlHeaderVersion | msgp uint                   | header version identifier
+| xlMetaVersion   | msgp uint                   | metadata version identifier
+| versions        | msgp int                    | Number of versions following
+| header_1        | msgp bin array              | Header of version 1
+| metadata_1      | msgp bin array              | Metadata of version 1
+| ...header_n     | msgp bin array              | Header of last version 
+| ...metadata_n   | msgp bin array              | Metadata of last version
+
+Each header contains a mspg array (tuple) encoded object:
+
+xlHeaderVersion version == 1:
+
+```
+//msgp:tuple xlMetaV2VersionHeader
+type xlMetaV2VersionHeader struct {
+	VersionID [16]byte  // Version UUID, raw.
+	ModTime   int64     // Unix nanoseconds.
+	Signature [4]byte   // Signature of metadata.
+	Type      uint8     // Type if the version
+	Flags     uint8
+}
+```
+
+The following flags are defined:
+
+```
+const (
+	FreeVersion = 1 << 0
+	UsesDataDir = 1 << 1
+	InlineData  = 1 << 2
+)
+```
+
+The "Metadata" section contains a single version, encoded in similar fashion as each version in the `Versions` array
+of the previous version.
+
+# Inline Data
+
+Inline data is optional. If no inline data is present, it is encoded as 0 bytes.
+
+| Entry               | Encoding                    | Content
+| --------------------|-----------------------------|----------------------------------------
+| xlMetaInlineDataVer | byte                        | version identifier
+| id -> data          | msgp map[string][]byte      | Map of string id -> byte content
+
+Currently only xlMetaInlineDataVer == 1 exists.
+
+The ID is the string encoded Version ID of which the data corresponds.

docs/distributed/DECOMMISSION.md

@@ -0,0 +1,112 @@
+## Decommissioning
+
+Decommissiong is a mechanism in MinIO to drain older pools (usually with old hardware) and migrate the content from such pools to a newer pools (usually better hardware). Decommissioning spreads the data across all pools - for example if you decommission `pool1`, all the data from `pool1` shall be spread across `pool2` and `pool3` respectively.
+
+### Features
+
+- A pool in decommission still allows READ access to all its contents, newer WRITEs will be automatically scheduled to only new pools.
+- All versioned buckets maintain the same order for "versions" for each objects after being decommissioned to the newer pools.
+- A pool in decommission resumes from where it was left off (for example - in-case of cluster restarts or restarts attempted after a failed decommission attempt).
+
+### How to decommission a pool?
+```
+λ mc admin decommission start alias/ http://minio{1...2}/data{1...4}
+```
+
+### Status decommissioning a pool
+
+#### Decommissioning without args lists all pools
+```
+λ mc admin decommission status alias/
+┌─────┬─────────────────────────────────┬──────────────────────────────────┬────────┐
+│ ID  │ Pools                           │ Capacity                         │ Status │
+│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Active │
+│ 2nd │ http://minio{3...4}/data{1...4} │ 329 GiB (used) / 421 GiB (total) │ Active │
+└─────┴─────────────────────────────────┴──────────────────────────────────┴────────┘
+```
+
+#### Decommissioning status
+```
+λ mc admin decommission status alias/ http://minio{1...2}/data{1...4}
+Decommissioning rate at 36 MiB/sec [4 TiB/50 TiB]
+Started: 1 minute ago
+```
+
+Once it is **Complete**
+
+```
+λ mc admin decommission status alias/ http://minio{1...2}/data{1...4}
+Decommission of pool http://minio{1...2}/data{1...4} is complete, you may now remove it from server command line
+```
+
+#### A pool not under decommissioning will throw an error
+```
+λ mc admin decommission status alias/ http://minio{1...2}/data{1...4}
+ERROR: This pool is not scheduled for decommissioning currently.
+```
+
+### Canceling a decommission?
+Stop an on-going decommission in progress, mainly used in situations when the load may be too high and you may want to schedule the decommission at a later point in time.
+
+`mc admin decommission cancel` without an argument, lists out any on-going decommission in progress.
+
+```
+λ mc admin decommission cancel alias/
+┌─────┬─────────────────────────────────┬──────────────────────────────────┬──────────┐
+│ ID  │ Pools                           │ Capacity                         │ Status   │
+│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Draining │
+└─────┴─────────────────────────────────┴──────────────────────────────────┴──────────┘
+```
+
+> NOTE: Canceled decommission will not make the pool active again, since we might have  potentially partial namespace on the other pools, to avoid this scenario be absolutely sure to make decommissioning a planned well thought activity. This is not to be run on a daily basis.
+
+```
+λ mc admin decommission cancel alias/ http://minio{1...2}/data{1...4}
+┌─────┬─────────────────────────────────┬──────────────────────────────────┬────────────────────┐
+│ ID  │ Pools                           │ Capacity                         │ Status             │
+│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Draining(Canceled) │
+└─────┴─────────────────────────────────┴──────────────────────────────────┴────────────────────┘
+```
+
+If for some reason decommission fails in between, the `status` will indicate decommission as failed instead.
+```
+λ mc admin decommission status alias/
+┌─────┬─────────────────────────────────┬──────────────────────────────────┬──────────────────┐
+│ ID  │ Pools                           │ Capacity                         │ Status           │
+│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Draining(Failed) │
+│ 2nd │ http://minio{3...4}/data{1...4} │ 329 GiB (used) / 421 GiB (total) │ Active           │
+└─────┴─────────────────────────────────┴──────────────────────────────────┴──────────────────┘
+```
+
+### Restart a canceled or failed decommission?
+
+```
+λ mc admin decommission start alias/ http://minio{1...2}/data{1...4}
+```
+
+### When decommission is 'Complete'?
+Once decommission is complete, it will be indicated with *Complete* status.  *Complete* means that now you can now safely remove the first pool argument from the MinIO command line.
+```
+λ mc admin decommission status alias/
+┌─────┬─────────────────────────────────┬──────────────────────────────────┬──────────┐
+│ ID  │ Pools                           │ Capacity                         │ Status   │
+│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Complete │
+│ 2nd │ http://minio{3...4}/data{1...4} │ 329 GiB (used) / 421 GiB (total) │ Active   │
+└─────┴─────────────────────────────────┴──────────────────────────────────┴──────────┘
+```
+
+- On baremetal setups if you have `MINIO_VOLUMES="http://minio{1...2}/data{1...4} http://minio{3...4}/data{1...4}"` you can remove the first argument `http://minio{1...2}/data{1...4}` to update your `MINIO_VOLUMES` setting and using `systemctl restart minio` on all the servers in the setup in parallel.
+
+- On Kubernetes setups MinIO the statefulset specification needs to be modified by changing the command line input for the MinIO container. Once the relevant changes are done proceed to execute `kubectl apply -f statefulset.yaml`.
+
+- On Operator based MinIO deployments you need to modify the `tenant.yaml` specification and modify the `pools:` section from two entries to a single entry, once relevant changes are done proceed to execute `kubectl apply -f tenant.yaml`.
+
+> Without a 'Complete' status any 'Active' or 'Draining' pool(s) are not allowed to be removed once configured.
+
+### NOTE
+- Empty delete marker's i.e objects with no other successor versions are not transitioned to the new pool, to avoid any empty metadata being recreated on the newer pool. We do not think this is needed, please open a GitHub issue if you think otherwise.
+
+### TODO
+- Richer progress UI is not present at the moment, this will be addressed in subsequent releases. Currently however a RATE of data transfer and usage increase is displayed via `mc`.
+- Transitioned Hot Tier's as pooled setups are not currently supported, attempting to decommission buckets with ILM Transition will be rejected by the server. This will be supported in future releases.
+- Embedded Console UI does not support Decommissioning through the UI yet. This will be supported in future releases.

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-01-04T07-41-07Z
+  image: quay.io/minio/minio:RELEASE.2022-01-08T03-11-54Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/site-replication/gen-oidc-sts-cred.go

@@ -0,0 +1,78 @@
+//go:build ignore
+// +build ignore
+
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+// This programs mocks user interaction against Dex IDP and generates STS
+// credentials. It is for MinIO testing purposes only.
+//
+// Run like:
+//
+// $ MINIO_ENDPOINT=http://localhost:9000 go run gen-oidc-sts-cred.go
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+
+	cr "github.com/minio/minio-go/v7/pkg/credentials"
+	cmd "github.com/minio/minio/cmd"
+)
+
+func main() {
+	ctx := context.Background()
+
+	endpoint := os.Getenv("MINIO_ENDPOINT")
+	if endpoint == "" {
+		log.Fatalf("Please specify a MinIO server endpoint environment variable like:\n\n\texport MINIO_ENDPOINT=http://localhost:9000")
+	}
+
+	appParams := cmd.OpenIDClientAppParams{
+		ClientID:     "minio-client-app",
+		ClientSecret: "minio-client-app-secret",
+		ProviderURL:  "http://127.0.0.1:5556/dex",
+		RedirectURL:  "http://127.0.0.1:10000/oauth_callback",
+	}
+
+	oidcToken, err := cmd.MockOpenIDTestUserInteraction(ctx, appParams, "dillon@example.io", "dillon")
+	if err != nil {
+		log.Fatalf("Failed to generate OIDC token: %v", err)
+	}
+
+	webID := cr.STSWebIdentity{
+		Client:      &http.Client{},
+		STSEndpoint: endpoint,
+		GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) {
+			return &cr.WebIdentityToken{
+				Token: oidcToken,
+			}, nil
+		},
+	}
+
+	value, err := webID.Retrieve()
+	if err != nil {
+		log.Fatalf("Expected to generate credentials: %v", err)
+	}
+
+	// Print credentials separated by colons:
+	fmt.Printf("%s:%s:%s\n", value.AccessKeyID, value.SecretAccessKey, value.SessionToken)
+}

docs/site-replication/run-multi-site-ldap.sh

@@ -10,7 +10,7 @@ cleanup() {
     echo "Cleaning up instances of MinIO"
     pkill minio
     pkill -9 minio
-    rm -rf /tmp/minio{1,2,3}
+    rm -rf /tmp/minio-ldap-idp{1,2,3}
 }
 
 cleanup
@@ -40,9 +40,9 @@ if [ ! -f ./mc ]; then
         && chmod +x mc
 fi
 
-minio server --address ":9001" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 &
-minio server --address ":9002" /tmp/minio2/{1...4} >/tmp/minio2_1.log 2>&1 &
-minio server --address ":9003" /tmp/minio3/{1...4} >/tmp/minio3_1.log 2>&1 &
+minio server --config-dir /tmp/minio-ldap --address ":9001" /tmp/minio-ldap-idp1/{1...4} >/tmp/minio1_1.log 2>&1 &
+minio server --config-dir /tmp/minio-ldap --address ":9002" /tmp/minio-ldap-idp2/{1...4} >/tmp/minio2_1.log 2>&1 &
+minio server --config-dir /tmp/minio-ldap --address ":9003" /tmp/minio-ldap-idp3/{1...4} >/tmp/minio3_1.log 2>&1 &
 
 sleep 10
 
@@ -166,6 +166,36 @@ if [ $? -ne 0 ]; then
     exit_1;
 fi
 
+vID=$(./mc stat minio2/newbucket/README.md --json | jq .versionID)
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v"
+if [ $? -ne 0 ]; then
+    echo "expecting tag set to be successful. exiting.."
+    exit_1;
+fi
+sleep 5
+
+./mc tag remove --version-id "${vID}" minio2/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting tag removal to be successful. exiting.."
+    exit_1;
+fi
+sleep 5
+
+replStatus_minio2=$(./mc stat minio2/newbucket/README.md --json | jq -r .replicationStatus)
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+if [ ${replStatus_minio2} != "COMPLETED" ]; then
+    echo "expected tag removal to have replicated, exiting..."
+    exit_1;
+fi
+
 ./mc rm minio3/newbucket/README.md
 sleep 5
 
@@ -205,3 +235,5 @@ if [ "${enabled_minio1}" != "Enabled" ]; then
     echo "expected bucket to be mirrored with object-lock enabled, exiting..."
     exit_1;
 fi
+
+cleanup

docs/site-replication/run-multi-site-minio-idp.sh

@@ -0,0 +1,258 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+    cleanup
+    exit 1
+}
+
+cleanup() {
+    echo "Cleaning up instances of MinIO"
+    pkill minio
+    pkill -9 minio
+    rm -rf /tmp/minio-internal-idp{1,2,3}
+}
+
+cleanup
+
+unset MINIO_KMS_KES_CERT_FILE
+unset MINIO_KMS_KES_KEY_FILE
+unset MINIO_KMS_KES_ENDPOINT
+unset MINIO_KMS_KES_KEY_NAME
+
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+export MINIO_KMS_AUTO_ENCRYPTION=off
+export MINIO_PROMETHEUS_AUTH_TYPE=public
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+
+if [ ! -f ./mc ]; then
+    wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \
+        && chmod +x mc
+fi
+
+minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-internal-idp1/{1...4} >/tmp/minio1_1.log 2>&1 &
+minio server --config-dir /tmp/minio-internal --address ":9002" /tmp/minio-internal-idp2/{1...4} >/tmp/minio2_1.log 2>&1 &
+minio server --config-dir /tmp/minio-internal --address ":9003" /tmp/minio-internal-idp3/{1...4} >/tmp/minio3_1.log 2>&1 &
+
+sleep 10
+
+export MC_HOST_minio1=http://minio:minio123@localhost:9001
+export MC_HOST_minio2=http://minio:minio123@localhost:9002
+export MC_HOST_minio3=http://minio:minio123@localhost:9003
+
+./mc admin replicate add minio1 minio2
+
+./mc admin user add minio1 foobar foo12345
+
+## add foobar-g group with foobar
+./mc admin group add minio2 foobar-g foobar
+
+./mc admin policy set minio1 consoleAdmin user=foobar
+sleep 5
+
+./mc admin user info minio2 foobar
+
+./mc admin group info minio1 foobar-g
+
+./mc admin policy add minio1 rw ./docs/site-replication/rw.json
+
+sleep 5
+./mc admin policy info minio2 rw >/dev/null 2>&1
+
+./mc admin replicate status minio1
+
+## Add a new empty site
+./mc admin replicate add minio1 minio2 minio3
+
+sleep 10
+
+./mc admin policy info minio3 rw >/dev/null 2>&1
+
+./mc admin policy remove minio3 rw
+
+./mc admin replicate status minio3
+
+sleep 10
+
+./mc admin policy info minio1 rw
+if [ $? -eq 0 ]; then
+    echo "expecting the command to fail, exiting.."
+    exit_1;
+fi
+
+./mc admin policy info minio2 rw
+if [ $? -eq 0 ]; then
+    echo "expecting the command to fail, exiting.."
+    exit_1;
+fi
+
+./mc admin policy info minio3 rw
+if [ $? -eq 0 ]; then
+    echo "expecting the command to fail, exiting.."
+    exit_1;
+fi
+
+./mc admin user info minio1 foobar
+if [ $? -ne 0 ]; then
+    echo "policy mapping missing on 'minio1', exiting.."
+    exit_1;
+fi
+
+./mc admin user info minio2 foobar
+if [ $? -ne 0 ]; then
+    echo "policy mapping missing on 'minio2', exiting.."
+    exit_1;
+fi
+
+./mc admin user info minio3 foobar
+if [ $? -ne 0 ]; then
+    echo "policy mapping missing on 'minio3', exiting.."
+    exit_1;
+fi
+
+./mc admin group info minio3 foobar-g
+if [ $? -ne 0 ]; then
+    echo "group mapping missing on 'minio3', exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct add minio2 foobar --access-key testsvc --secret-key testsvc123
+if [ $? -ne 0 ]; then
+    echo "adding svc account failed, exiting.."
+    exit_1;
+fi
+
+sleep 10
+
+./mc admin user svcacct info minio1 testsvc
+if [ $? -ne 0 ]; then
+    echo "svc account not mirrored, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct info minio2 testsvc
+if [ $? -ne 0 ]; then
+    echo "svc account not mirrored, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct rm minio1 testsvc
+if [ $? -ne 0 ]; then
+    echo "removing svc account failed, exiting.."
+    exit_1;
+fi
+
+sleep 10
+./mc admin user svcacct info minio2 testsvc
+if [ $? -eq 0 ]; then
+    echo "svc account found after delete, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct info minio3 testsvc
+if [ $? -eq 0 ]; then
+    echo "svc account found after delete, exiting.."
+    exit_1;
+fi
+
+./mc mb minio1/newbucket
+
+sleep 5
+./mc stat minio2/newbucket
+if [ $? -ne 0 ]; then
+    echo "expecting bucket to be present. exiting.."
+    exit_1;
+fi
+
+./mc stat minio3/newbucket
+if [ $? -ne 0 ]; then
+    echo "expecting bucket to be present. exiting.."
+    exit_1;
+fi
+
+./mc cp README.md minio2/newbucket/
+
+sleep 5
+./mc stat minio1/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+./mc stat minio3/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+vID=$(./mc stat minio2/newbucket/README.md --json | jq .versionID)
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v"
+if [ $? -ne 0 ]; then
+    echo "expecting tag set to be successful. exiting.."
+    exit_1;
+fi
+sleep 5
+
+./mc tag remove --version-id "${vID}" minio2/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting tag removal to be successful. exiting.."
+    exit_1;
+fi
+sleep 5
+
+replStatus_minio2=$(./mc stat minio2/newbucket/README.md --json | jq -r .replicationStatus )
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+if [ ${replStatus_minio2} != "COMPLETED" ]; then
+    echo "expected tag removal to have replicated, exiting..."
+    exit_1;
+fi
+
+./mc rm minio3/newbucket/README.md
+sleep 5
+
+./mc stat minio2/newbucket/README.md
+if [ $? -eq 0 ]; then
+    echo "expected file to be deleted, exiting.."
+    exit_1;
+fi
+
+./mc stat minio1/newbucket/README.md
+if [ $? -eq 0 ]; then
+    echo "expected file to be deleted, exiting.."
+    exit_1;
+fi
+
+./mc mb --with-lock minio3/newbucket-olock
+sleep 5
+
+enabled_minio2=$(./mc stat --json minio2/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio2}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi
+
+enabled_minio1=$(./mc stat --json minio1/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio1}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi

docs/site-replication/run-multi-site-oidc.sh

@@ -0,0 +1,223 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2120
+exit_1() {
+    cleanup
+    exit 1
+}
+
+cleanup() {
+    echo "Cleaning up instances of MinIO"
+    pkill minio
+    pkill -9 minio
+    rm -rf /tmp/minio{1,2,3}
+}
+
+cleanup
+
+unset MINIO_KMS_KES_CERT_FILE
+unset MINIO_KMS_KES_KEY_FILE
+unset MINIO_KMS_KES_ENDPOINT
+unset MINIO_KMS_KES_KEY_NAME
+
+export MINIO_BROWSER=off
+export MINIO_ROOT_USER="minio"
+export MINIO_ROOT_PASSWORD="minio123"
+export MINIO_KMS_AUTO_ENCRYPTION=off
+export MINIO_PROMETHEUS_AUTH_TYPE=public
+export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=
+export MINIO_IDENTITY_OPENID_CONFIG_URL="http://localhost:5556/dex/.well-known/openid-configuration"
+export MINIO_IDENTITY_OPENID_CLIENT_ID="minio-client-app"
+export MINIO_IDENTITY_OPENID_CLIENT_SECRET="minio-client-app-secret"
+export MINIO_IDENTITY_OPENID_CLAIM_NAME="groups"
+export MINIO_IDENTITY_OPENID_SCOPES="openid,groups"
+
+export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:10000/oauth_callback"
+minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 &
+
+export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:11000/oauth_callback"
+minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4} >/tmp/minio2_1.log 2>&1 &
+
+export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:12000/oauth_callback"
+minio server --address ":9003" --console-address ":12000" /tmp/minio3/{1...4} >/tmp/minio3_1.log 2>&1 &
+
+if [ ! -f ./mc ]; then
+    wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \
+        && chmod +x mc
+fi
+
+sleep 10
+
+export MC_HOST_minio1=http://minio:minio123@localhost:9001
+export MC_HOST_minio2=http://minio:minio123@localhost:9002
+export MC_HOST_minio3=http://minio:minio123@localhost:9003
+
+./mc admin replicate add minio1 minio2 minio3
+
+./mc admin policy add minio1 projecta ./docs/site-replication/rw.json
+sleep 5
+
+./mc admin policy info minio2 projecta >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+    echo "expecting the command to succeed, exiting.."
+    exit_1;
+fi
+./mc admin policy info minio3 projecta >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+    echo "expecting the command to succeed, exiting.."
+    exit_1;
+fi
+
+./mc admin policy remove minio3 projecta
+
+sleep 10
+./mc admin policy info minio1 projecta
+if [ $? -eq 0 ]; then
+    echo "expecting the command to fail, exiting.."
+    exit_1;
+fi
+
+./mc admin policy info minio2 projecta
+if [ $? -eq 0 ]; then
+    echo "expecting the command to fail, exiting.."
+    exit_1;
+fi
+
+./mc admin policy add minio1 projecta ./docs/site-replication/rw.json
+sleep 5
+
+# Generate STS credential with STS call to minio1
+STS_CRED=$(MINIO_ENDPOINT=http://localhost:9001 go run ./docs/site-replication/gen-oidc-sts-cred.go)
+
+MC_HOST_foo=http://${STS_CRED}@localhost:9001 ./mc ls foo
+if [ $? -ne 0 ]; then
+    echo "Expected sts credential to work, exiting.."
+    exit_1;
+fi
+
+sleep 2
+
+# Check that the STS credential works on minio2 and minio3.
+MC_HOST_foo=http://${STS_CRED}@localhost:9002 ./mc ls foo
+if [ $? -ne 0 ]; then
+    echo "Expected sts credential to work, exiting.."
+    exit_1;
+fi
+
+MC_HOST_foo=http://${STS_CRED}@localhost:9003 ./mc ls foo
+if [ $? -ne 0 ]; then
+    echo "Expected sts credential to work, exiting.."
+    exit_1;
+fi
+
+STS_ACCESS_KEY=$(echo ${STS_CRED} | cut -d ':' -f 1)
+
+# Create service account for STS user
+./mc admin user svcacct add minio2 $STS_ACCESS_KEY --access-key testsvc --secret-key testsvc123
+if [ $? -ne 0 ]; then
+    echo "adding svc account failed, exiting.."
+    exit_1;
+fi
+
+sleep 10
+
+./mc admin user svcacct info minio1 testsvc
+if [ $? -ne 0 ]; then
+    echo "svc account not mirrored, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct info minio2 testsvc
+if [ $? -ne 0 ]; then
+    echo "svc account not mirrored, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct rm minio1 testsvc
+if [ $? -ne 0 ]; then
+    echo "removing svc account failed, exiting.."
+    exit_1;
+fi
+
+sleep 10
+./mc admin user svcacct info minio2 testsvc
+if [ $? -eq 0 ]; then
+    echo "svc account found after delete, exiting.."
+    exit_1;
+fi
+
+./mc admin user svcacct info minio3 testsvc
+if [ $? -eq 0 ]; then
+    echo "svc account found after delete, exiting.."
+    exit_1;
+fi
+
+./mc mb minio1/newbucket
+
+sleep 5
+./mc stat minio2/newbucket
+if [ $? -ne 0 ]; then
+    echo "expecting bucket to be present. exiting.."
+    exit_1;
+fi
+
+./mc stat minio3/newbucket
+if [ $? -ne 0 ]; then
+    echo "expecting bucket to be present. exiting.."
+    exit_1;
+fi
+
+./mc cp README.md minio2/newbucket/
+
+sleep 5
+./mc stat minio1/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+./mc stat minio3/newbucket/README.md
+if [ $? -ne 0 ]; then
+    echo "expecting object to be present. exiting.."
+    exit_1;
+fi
+
+./mc rm minio3/newbucket/README.md
+sleep 5
+
+./mc stat minio2/newbucket/README.md
+if [ $? -eq 0 ]; then
+    echo "expected file to be deleted, exiting.."
+    exit_1;
+fi
+
+./mc stat minio1/newbucket/README.md
+if [ $? -eq 0 ]; then
+    echo "expected file to be deleted, exiting.."
+    exit_1;
+fi
+
+./mc mb --with-lock minio3/newbucket-olock
+sleep 5
+
+enabled_minio2=$(./mc stat --json minio2/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio2}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi
+
+enabled_minio1=$(./mc stat --json minio1/newbucket-olock| jq -r .metadata.ObjectLock.enabled)
+if [ $? -ne 0 ]; then
+    echo "expected bucket to be mirrored with object-lock but not present, exiting..."
+    exit_1;
+fi
+
+if [ "${enabled_minio1}" != "Enabled" ]; then
+    echo "expected bucket to be mirrored with object-lock enabled, exiting..."
+    exit_1;
+fi

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.12.0
 	github.com/fatih/color v1.13.0
 	github.com/go-ldap/ldap/v3 v3.2.4
-	github.com/go-openapi/loads v0.20.2
+	github.com/go-openapi/loads v0.21.0
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang-jwt/jwt/v4 v4.1.0
 	github.com/gomodule/redigo v1.8.8
@@ -40,19 +40,19 @@ require (
 	github.com/klauspost/compress v1.13.6
 	github.com/klauspost/cpuid/v2 v2.0.9
 	github.com/klauspost/pgzip v1.2.5
-	github.com/klauspost/readahead v1.3.1
+	github.com/klauspost/readahead v1.4.0
 	github.com/klauspost/reedsolomon v1.9.15
 	github.com/lib/pq v1.9.0
 	github.com/miekg/dns v1.1.43
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.13.2
+	github.com/minio/console v0.13.3
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.2.4
-	github.com/minio/minio-go/v7 v7.0.20
+	github.com/minio/madmin-go v1.2.7
+	github.com/minio/minio-go/v7 v7.0.21
 	github.com/minio/parquet-go v1.1.0
-	github.com/minio/pkg v1.1.11
+	github.com/minio/pkg v1.1.15
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.2.1
@@ -72,7 +72,7 @@ require (
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/procfs v0.7.3
 	github.com/rs/cors v1.7.0
-	github.com/rs/dnscache v0.0.0-20210201191234-295bba877686
+	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/secure-io/sio-go v0.3.1
 	github.com/shirou/gopsutil/v3 v3.21.9
 	github.com/streadway/amqp v1.0.0
@@ -114,19 +114,18 @@ require (
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/fatih/structs v1.1.0 // indirect
-	github.com/georgysavva/scany v0.2.7 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
 	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-openapi/analysis v0.20.0 // indirect
-	github.com/go-openapi/errors v0.19.9 // indirect
+	github.com/go-openapi/analysis v0.20.1 // indirect
+	github.com/go-openapi/errors v0.20.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
-	github.com/go-openapi/runtime v0.19.24 // indirect
-	github.com/go-openapi/spec v0.20.3 // indirect
-	github.com/go-openapi/strfmt v0.20.0 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
-	github.com/go-openapi/validate v0.20.2 // indirect
+	github.com/go-openapi/jsonreference v0.19.6 // indirect
+	github.com/go-openapi/runtime v0.21.0 // indirect
+	github.com/go-openapi/spec v0.20.4 // indirect
+	github.com/go-openapi/strfmt v0.21.1 // indirect
+	github.com/go-openapi/swag v0.19.15 // indirect
+	github.com/go-openapi/validate v0.20.3 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/goccy/go-json v0.7.9 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -169,8 +168,7 @@ require (
 	github.com/minio/filepath v1.0.0 // indirect
 	github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7 // indirect
-	github.com/minio/operator/logsearchapi v0.0.0-20211011212245-31460bbbc4b7 // indirect
+	github.com/minio/operator v0.0.0-20220110040724-a5d59a342b7f // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -178,9 +176,9 @@ require (
 	github.com/nats-io/nats-streaming-server v0.21.2 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pkg/profile v1.6.0 // indirect
 	github.com/pkg/xattr v0.4.3 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3 // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
 	github.com/prometheus/common v0.31.1 // indirect
@@ -189,8 +187,6 @@ require (
 	github.com/rjeczalik/notify v0.9.2 // indirect
 	github.com/rs/xid v1.3.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/tidwall/gjson v1.11.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
@@ -200,7 +196,7 @@ require (
 	github.com/unrolled/secure v1.0.9 // indirect
 	github.com/xdg/stringprep v1.0.0 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
-	go.mongodb.org/mongo-driver v1.4.6 // indirect
+	go.mongodb.org/mongo-driver v1.7.5 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect

go.sum

@@ -252,9 +252,6 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
-github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0uLm/bb2Atc+S8=
-github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636FkGM=
@@ -274,9 +271,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c=
-github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -303,7 +299,6 @@ github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0/go.mod h1:J70FGZSbzsjecRTiTzER+
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0 h1:Fe5DW39aaoS/fqZiYlylEqQWIKznnbatWSHpWdFA3oQ=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
-github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -354,7 +349,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -377,8 +371,6 @@ github.com/frankban/quicktest v1.12.1 h1:P6vQcHwZYgVGIpUzKB5DXzkEeYJppJOStPLuh9a
 github.com/frankban/quicktest v1.12.1/go.mod h1:qLE0fzW0VuyUAJgPU19zByoIr0HtCHN/r/VLSOOIySU=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/georgysavva/scany v0.2.7 h1:SBEuurTvWOUp7FnGBOjeSF9XWaWmVzc91h9baPo6y2s=
-github.com/georgysavva/scany v0.2.7/go.mod h1:bcxPhzeQFQqAUmjlZVwTGlu6AnWFSOiHpalfBe0xQ6U=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gizak/termui/v3 v3.1.0/go.mod h1:bXQEBkJpzxUAKf0+xq9MSWAvWZlE7c+aidmyFlkYTrY=
@@ -423,8 +415,9 @@ github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9sn
 github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
 github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ=
 github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk=
-github.com/go-openapi/analysis v0.20.0 h1:UN09o0kNhleunxW7LR+KnltD0YrJ8FF03pSqvAN3Vro=
 github.com/go-openapi/analysis v0.20.0/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og=
+github.com/go-openapi/analysis v0.20.1 h1:zdVbw8yoD4SWZeq+cWdGgquaB0W4VrsJvDJHJND/Ktc=
+github.com/go-openapi/analysis v0.20.1/go.mod h1:BMchjvaHDykmRMsK40iPtvyOfFdMMxlOmQr9FBZk+Og=
 github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
@@ -432,8 +425,9 @@ github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA
 github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.19.9 h1:9SnKdGhiPZHF3ttwFMiCBEb8jQ4IDdrK+5+a0oTygA4=
 github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
+github.com/go-openapi/errors v0.20.1 h1:j23mMDtRxMwIobkpId7sWh7Ddcx4ivaoqUbfXx5P+a8=
+github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
@@ -446,8 +440,9 @@ github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3Hfo
 github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
-github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
 github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
+github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs=
+github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns=
 github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
@@ -458,15 +453,17 @@ github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2e
 github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
 github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
 github.com/go-openapi/loads v0.20.0/go.mod h1:2LhKquiE513rN5xC6Aan6lYOSddlL8Mp20AW9kpviM4=
-github.com/go-openapi/loads v0.20.2 h1:z5p5Xf5wujMxS1y8aP+vxwW5qYT2zdJBbXKmQUG3lcc=
 github.com/go-openapi/loads v0.20.2/go.mod h1:hTVUotJ+UonAMMZsvakEgmWKgtulweO9vYP2bQYKA/o=
+github.com/go-openapi/loads v0.21.0 h1:jYtUO4wwP7psAweisP/MDoOpdzsYEESdoPcsWjHDR68=
+github.com/go-openapi/loads v0.21.0/go.mod h1:rHYve9nZrQ4CJhyeIIFJINGCg1tQpx2yJrrNo8sf1ws=
 github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
 github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
 github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
 github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
 github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
-github.com/go-openapi/runtime v0.19.24 h1:TqagMVlRAOTwllE/7hNKx6rQ10O6T8ZzeJdMjSTKaD4=
 github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
+github.com/go-openapi/runtime v0.21.0 h1:giZ8eT26R+/rx6RX2MkYjZPY8vPYVKDhP/mOazrQHzM=
+github.com/go-openapi/runtime v0.21.0/go.mod h1:aQg+kaIQEn+A2CRSY1TxbM8+sT9g2V3aLc1FbIAnbbs=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
@@ -478,8 +475,9 @@ github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHK
 github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
 github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
 github.com/go-openapi/spec v0.20.1/go.mod h1:93x7oh+d+FQsmsieroS4cmR3u0p/ywH649a3qwC9OsQ=
-github.com/go-openapi/spec v0.20.3 h1:uH9RQ6vdyPSs2pSy9fL8QPspDF2AMIMPtmK5coSSjtQ=
 github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg=
+github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M=
+github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I=
 github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
@@ -488,8 +486,11 @@ github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6
 github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
 github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
 github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
-github.com/go-openapi/strfmt v0.20.0 h1:l2omNtmNbMc39IGptl9BuXBEKcZfS8zjrTsPKTiJiDM=
 github.com/go-openapi/strfmt v0.20.0/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
+github.com/go-openapi/strfmt v0.20.2/go.mod h1:43urheQI9dNtE5lTZQfuFJvjYJKPrxicATpEfZwHUNk=
+github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg=
+github.com/go-openapi/strfmt v0.21.1 h1:G6s2t5V5kGCHLVbSdZ/6lI8Wm4OzoPFkc3/cjAsKQrM=
+github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
@@ -499,8 +500,9 @@ github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT
 github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
 github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M=
 github.com/go-openapi/swag v0.19.13/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
-github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
+github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo=
@@ -509,8 +511,8 @@ github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbN
 github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
 github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI=
 github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
-github.com/go-openapi/validate v0.20.2 h1:AhqDegYV3J3iQkMPJSXkvzymHKMTw0BST3RK3hTT4ts=
-github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0=
+github.com/go-openapi/validate v0.20.3 h1:GZPPhhKSZrE8HjB4eEkoYAZmoWA4+tCemSgINH1/vKw=
+github.com/go-openapi/validate v0.20.3/go.mod h1:goDdqVGiigM3jChcrYJxD2joalke3ZXeftD16byIjA4=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -565,7 +567,6 @@ github.com/goccy/go-json v0.7.9 h1:mSp3uo1tr6MXQTYopSNhHTUnJhd2zQ4Yk+HdJZP+ZRY=
 github.com/goccy/go-json v0.7.9/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
-github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -579,7 +580,6 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
 github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -666,6 +666,7 @@ github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSN
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
 github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -816,63 +817,6 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
-github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
-github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
-github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
-github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
-github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
-github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA=
-github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE=
-github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s=
-github.com/jackc/pgconn v1.4.0/go.mod h1:Y2O3ZDF0q4mMacyWV3AstPJpeHXWGEetiFttmq5lahk=
-github.com/jackc/pgconn v1.5.0/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
-github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpTFe8WUnPZWN5KY/mB8FGMIYRdd8P8Jr0fAI=
-github.com/jackc/pgconn v1.7.0 h1:pwjzcYyfmz/HQOQlENvG1OcDqauTGaqlVahq934F0/U=
-github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7JMeA=
-github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
-github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
-github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
-github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
-github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
-github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
-github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
-github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
-github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
-github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
-github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM=
-github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
-github.com/jackc/pgproto3/v2 v2.0.5 h1:NUbEWPmCQZbMmYlTjVoNPhc0CfnYyz2bfUAh6A5ZVJM=
-github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
-github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
-github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
-github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
-github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
-github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
-github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
-github.com/jackc/pgtype v1.2.0/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0=
-github.com/jackc/pgtype v1.3.0/go.mod h1:b0JqxHvPmljG+HQ5IsvQ0yqeSi4nGcDTVjFoiLDb0Ik=
-github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkALtxZMCH411K+tKzNpwzCKU+AnPzBKZ+I+Po=
-github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ=
-github.com/jackc/pgtype v1.3.1-0.20200612023650-09efc3839047/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
-github.com/jackc/pgtype v1.5.0 h1:jzBqRk2HFG2CV4AIwgCI2PwTgm6UUoCAK2ofHHRirtc=
-github.com/jackc/pgtype v1.5.0/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
-github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o=
-github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
-github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
-github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
-github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
-github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXgo+kA=
-github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oAlxAg=
-github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o=
-github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg=
-github.com/jackc/pgx/v4 v4.9.0 h1:6STjDqppM2ROy5p1wNDcsC7zJTjSHeuCsguZmXyzx7c=
-github.com/jackc/pgx/v4 v4.9.0/go.mod h1:MNGWmViCgqbZck9ujOOBN63gK9XVGILXWCvKLGKmnms=
-github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
-github.com/jackc/puddle v1.1.2 h1:mpQEXihFnWGDy6X98EOTh81JYuxn7txby8ilJ3iIPGM=
-github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
@@ -893,9 +837,6 @@ github.com/jedib0t/go-pretty/v6 v6.0.5/go.mod h1:MTr6FgcfNdnN5wPVBzJ6mhJeDyiF0yB
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
-github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
 github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -903,7 +844,6 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
@@ -955,8 +895,8 @@ github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBF
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
-github.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0=
-github.com/klauspost/readahead v1.3.1/go.mod h1:AH9juHzNH7xqdqFHrMRSHeH2Ps+vFf+kblDqzPFiLJg=
+github.com/klauspost/readahead v1.4.0 h1:w4hQ3BpdLjBnRQkZyNi+nwdHU7eGP9buTexWK9lU7gY=
+github.com/klauspost/readahead v1.4.0/go.mod h1:7bolpMKhT5LKskLwYXGSDOyA2TYtMFgdgV0Y8gy7QhA=
 github.com/klauspost/reedsolomon v1.9.15 h1:g2erWKD2M6rgnPf89fCji6jNlhMKMdXcuNHMW1SYCIo=
 github.com/klauspost/reedsolomon v1.9.15/go.mod h1:eqPAcE7xar5CIzcdfwydOEdcmchAKAP/qs14y4GCBOk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -994,12 +934,8 @@ github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFe
 github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lestrrat-go/pdebug/v3 v3.0.1/go.mod h1:za+m+Ve24yCxTEhR59N7UlnJomWwCiIqbJRmKeiADU4=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lib/pq v1.9.0 h1:L8nSXQQzAYByakOFMTwpjRoHsMJklur4Gi59b6VivR8=
 github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
@@ -1039,9 +975,7 @@ github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -1056,7 +990,6 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -1075,8 +1008,8 @@ github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos=
 github.com/minio/colorjson v1.0.1/go.mod h1:oPM3zQQY8Gz9NGtgvuBEjQ+gPZLKAGc7T+kjMlwtOgs=
-github.com/minio/console v0.13.2 h1:8dgHnT/hLN5ubCx49hR19KBLUGS+i0E37Nx1gPiTVlE=
-github.com/minio/console v0.13.2/go.mod h1:jXzCkW7CbIZjlNQ9zMbUd9sQk/f9B7zJ6ieSl9fWioM=
+github.com/minio/console v0.13.3 h1:Vm6AkRtiJryofCH9pVL/4MTU2MC8310FgoZfPtCT0CQ=
+github.com/minio/console v0.13.3/go.mod h1:5GEPzNwGgXyFb1twfxzOBwiTtA1W9W0PkIs02g1s1Ks=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf h1:wylCc/PdvdTIqYqVNEU9LJAZBanvfGY1TwTnjM3zQaA=
@@ -1092,8 +1025,8 @@ github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQc
 github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
 github.com/minio/madmin-go v1.1.15/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
 github.com/minio/madmin-go v1.1.23/go.mod h1:wv8zCroSCnpjjQdmgsdJEkFH2oD4w9J40OZqbhxjiJ4=
-github.com/minio/madmin-go v1.2.4 h1:o+0X6ENO/AtkRxCbD4FXT6YCRIH7HgDGb+WzfJFDzMQ=
-github.com/minio/madmin-go v1.2.4/go.mod h1:/rOfQv4ohkXJ+7EaSnhg9IJEX7cobX08zkSLfh8G3Ks=
+github.com/minio/madmin-go v1.2.7 h1:zxXxflVKf3J1TUCnWoT2rNbDLOJQMbsNbrbNsqXD3Ew=
+github.com/minio/madmin-go v1.2.7/go.mod h1:/rOfQv4ohkXJ+7EaSnhg9IJEX7cobX08zkSLfh8G3Ks=
 github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2 h1:xocb1RGyrDJ8PxkNn0NSbaBlfdU6J/Ag9QK62pb7nR8=
 github.com/minio/mc v0.0.0-20211207230606-23a05f5a17f2/go.mod h1:siI9jWTzj1KsNXgz6NOL/S7OTaAUM0OMi+zEkF08gnA=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -1102,20 +1035,20 @@ github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEp
 github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
 github.com/minio/minio-go/v7 v7.0.16-0.20211108161804-a7a36ee131df/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
-github.com/minio/minio-go/v7 v7.0.20 h1:0+Xt1SkCKDgcx5cmo3UxXcJ37u5Gy+/2i/+eQYqmYJw=
 github.com/minio/minio-go/v7 v7.0.20/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
-github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7 h1:dkfuMNslMjGoJ4ArAMSoQhidYNdm3SgzLBP+f96O3/E=
-github.com/minio/operator v0.0.0-20211011212245-31460bbbc4b7/go.mod h1:lDpuz8nwsfhKlfiBaA3Z8AW019fWEAjO2gltfLbdorE=
-github.com/minio/operator/logsearchapi v0.0.0-20211011212245-31460bbbc4b7 h1:vFtQqCt67ETp0JAkOKRWTKkgwFv14Vc1jJSxmQ8wJE0=
-github.com/minio/operator/logsearchapi v0.0.0-20211011212245-31460bbbc4b7/go.mod h1:R+38Pf3wfm+JMiyLPb/r8OMrBm0vK2hZgUT4y4aYoSY=
+github.com/minio/minio-go/v7 v7.0.21 h1:xrc4BQr1Fa4s5RwY0xfMjPZFJ1bcYBCCHYlngBdWV+k=
+github.com/minio/minio-go/v7 v7.0.21/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
+github.com/minio/operator v0.0.0-20220110040724-a5d59a342b7f h1:68HNranltwKT74ty57k+fbTnYcEZ50Nt7fnmcuX51jU=
+github.com/minio/operator v0.0.0-20220110040724-a5d59a342b7f/go.mod h1:k00quEXDtYJ97BenWmJ3RCpcPhH/jqSNeobrZ6/vTiQ=
 github.com/minio/parquet-go v1.1.0 h1:j2Fn1/h7Ts/0qzdMZU9oCUKr0IJwRTD9Hg9QJyVaN6A=
 github.com/minio/parquet-go v1.1.0/go.mod h1:nnAkbt2CG/DCQ3trcV3uyvwns4VjyoINF5vMqF5efOE=
 github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
-github.com/minio/pkg v1.0.11/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
 github.com/minio/pkg v1.1.3/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
-github.com/minio/pkg v1.1.11 h1:FQMYrOZzZenQUVwvxM0YjoAZB15Wb+m3TZSu33Lynx0=
 github.com/minio/pkg v1.1.11/go.mod h1:2WJAxesjzmPK9MnLZKm5n1hVYfBg04f2GQs6N5ImNU8=
+github.com/minio/pkg v1.1.14/go.mod h1:2WJAxesjzmPK9MnLZKm5n1hVYfBg04f2GQs6N5ImNU8=
+github.com/minio/pkg v1.1.15 h1:bH66MmilBVZQG4q2/8EjxI5EU7khtqdBKkaAL2BHUTI=
+github.com/minio/pkg v1.1.15/go.mod h1:2WJAxesjzmPK9MnLZKm5n1hVYfBg04f2GQs6N5ImNU8=
 github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
 github.com/minio/selfupdate v0.4.0 h1:A7t07pN4Ch1tBTIRStW0KhUVyykz+2muCqFsITQeEW8=
 github.com/minio/selfupdate v0.4.0/go.mod h1:mcDkzMgq8PRcpCRJo/NlPY7U45O5dfYl2Y0Rg7IustY=
@@ -1215,6 +1148,7 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
+github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -1246,6 +1180,7 @@ github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go
 github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
@@ -1355,13 +1290,11 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
-github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 h1:IJ6Df0uxPDtNoByV0KkzVKNseWvZFCNM/S9UoyOMCSI=
-github.com/rs/dnscache v0.0.0-20210201191234-295bba877686/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
+github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs=
+github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
-github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -1385,15 +1318,11 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada h1:WokF3GuxBeL+n4Lk4Fa8v9mbdjlrl7bHuneF4N1bk2I=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/gopsutil/v3 v3.21.4/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
-github.com/shirou/gopsutil/v3 v3.21.5/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
 github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88=
 github.com/shirou/gopsutil/v3 v3.21.8/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
 github.com/shirou/gopsutil/v3 v3.21.9 h1:Vn4MUz2uXhqLSiCbGFRc0DILbMVLAY92DSkT8bsYrHg=
 github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
-github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/shopspring/decimal v0.0.0-20200419222939-1884f454f8ea/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
@@ -1452,7 +1381,6 @@ github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1Sd
 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -1526,6 +1454,9 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
 github.com/xanzy/go-gitlab v0.31.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
@@ -1536,6 +1467,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
 github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -1543,7 +1475,6 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/zeebo/xxh3 v1.0.0 h1:6eLPZCVXpsGnhv8RiWBEJs5kenm2W1CMwon19/l8ODc=
 github.com/zeebo/xxh3 v1.0.0/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA=
-github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
@@ -1568,8 +1499,11 @@ go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS
 go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
 go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
 go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
-go.mongodb.org/mongo-driver v1.4.6 h1:rh7GdYmDrb8AQSkF8yteAus8qYOgOASWDOv1BWqBXkU=
 go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
+go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw=
+go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
+go.mongodb.org/mongo-driver v1.7.5 h1:ny3p0reEpgsR2cfA5cjgwFZg3Cv/ofFh/8jbhGtz9VI=
+go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -1600,7 +1534,6 @@ go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
 go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
 go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
 go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
@@ -1616,8 +1549,6 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -1628,11 +1559,9 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -1644,7 +1573,6 @@ golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -1824,7 +1752,6 @@ golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1950,7 +1877,6 @@ golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -1964,7 +1890,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -2030,8 +1955,6 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2219,7 +2142,6 @@ gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y=
 gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo=
-gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 3.4.5
-appVersion: RELEASE.2022-01-04T07-41-07Z
+version: 3.4.6
+appVersion: RELEASE.2022-01-08T03-11-54Z
 keywords:
   - minio
   - storage

helm/minio/templates/NOTES.txt

@@ -14,7 +14,7 @@ You can now access MinIO server on http://localhost:9000. Follow the below steps
 
   1. Download the MinIO mc client - https://docs.minio.io/docs/minio-client-quickstart-guide
 
-  2. export MC_HOST_{{ template "minio.fullname" . }}-local=http://$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:{{ .Values.service.port }}
+  2. export MC_HOST_{{ template "minio.fullname" . }}-local=http://$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "minio.secretName" . }} -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "minio.secretName" . }} -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:{{ .Values.service.port }}
 
   3. mc ls {{ template "minio.fullname" . }}-local
 

index.yaml

@@ -1,9 +1,31 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2022-01-08T03-11-54Z
+    created: "2022-01-07T22:23:31.017276397-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.4.6.tgz
+    version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2022-01-05T11:32:39.397078964-08:00"
+    created: "2022-01-07T22:23:31.016291288-08:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -25,7 +47,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2022-01-05T11:32:39.396093317-08:00"
+    created: "2022-01-07T22:23:31.015267411-08:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -47,7 +69,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-01-05T11:32:39.395038813-08:00"
+    created: "2022-01-07T22:23:31.013914449-08:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -69,7 +91,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-01-05T11:32:39.392092639-08:00"
+    created: "2022-01-07T22:23:31.012618149-08:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -91,7 +113,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-01-05T11:32:39.386229629-08:00"
+    created: "2022-01-07T22:23:31.011313126-08:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -113,7 +135,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2022-01-05T11:32:39.385314326-08:00"
+    created: "2022-01-07T22:23:31.010415945-08:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -135,7 +157,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-01-05T11:32:39.384402307-08:00"
+    created: "2022-01-07T22:23:31.009538439-08:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -157,7 +179,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-01-05T11:32:39.383443694-08:00"
+    created: "2022-01-07T22:23:31.008600244-08:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -179,7 +201,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-01-05T11:32:39.382488898-08:00"
+    created: "2022-01-07T22:23:31.007685729-08:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -201,7 +223,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-01-05T11:32:39.381498938-08:00"
+    created: "2022-01-07T22:23:31.006668152-08:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -223,7 +245,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-01-05T11:32:39.379903729-08:00"
+    created: "2022-01-07T22:23:31.005220606-08:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -245,7 +267,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2022-01-05T11:32:39.379008717-08:00"
+    created: "2022-01-07T22:23:31.004333024-08:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -267,7 +289,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2022-01-05T11:32:39.378110405-08:00"
+    created: "2022-01-07T22:23:31.003383269-08:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -289,7 +311,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2022-01-05T11:32:39.377199371-08:00"
+    created: "2022-01-07T22:23:31.00245312-08:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -311,7 +333,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2022-01-05T11:32:39.376277102-08:00"
+    created: "2022-01-07T22:23:31.001470508-08:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -333,7 +355,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-01-05T11:32:39.375328694-08:00"
+    created: "2022-01-07T22:23:31.000439575-08:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -355,7 +377,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-01-05T11:32:39.374195493-08:00"
+    created: "2022-01-07T22:23:30.999005495-08:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -377,7 +399,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-01-05T11:32:39.372924905-08:00"
+    created: "2022-01-07T22:23:30.998030319-08:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -399,7 +421,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-01-05T11:32:39.372163928-08:00"
+    created: "2022-01-07T22:23:30.997073512-08:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -421,7 +443,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-01-05T11:32:39.371263273-08:00"
+    created: "2022-01-07T22:23:30.996141707-08:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -443,7 +465,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-01-05T11:32:39.370334869-08:00"
+    created: "2022-01-07T22:23:30.995177995-08:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -465,7 +487,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-01-05T11:32:39.367649887-08:00"
+    created: "2022-01-07T22:23:30.994243802-08:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -487,7 +509,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2022-01-05T11:32:39.36660403-08:00"
+    created: "2022-01-07T22:23:30.993250641-08:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -509,7 +531,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-01-05T11:32:39.365037889-08:00"
+    created: "2022-01-07T22:23:30.991794257-08:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -531,7 +553,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-01-05T11:32:39.364114151-08:00"
+    created: "2022-01-07T22:23:30.99054488-08:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -553,7 +575,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-01-05T11:32:39.36316085-08:00"
+    created: "2022-01-07T22:23:30.989246019-08:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -573,7 +595,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-01-05T11:32:39.362224907-08:00"
+    created: "2022-01-07T22:23:30.988375668-08:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -593,7 +615,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-01-05T11:32:39.361269015-08:00"
+    created: "2022-01-07T22:23:30.986818536-08:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -613,7 +635,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-01-05T11:32:39.36037317-08:00"
+    created: "2022-01-07T22:23:30.983755023-08:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -633,7 +655,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-01-05T11:32:39.359327826-08:00"
+    created: "2022-01-07T22:23:30.982736376-08:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -653,7 +675,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-01-05T11:32:39.353680391-08:00"
+    created: "2022-01-07T22:23:30.981247281-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -673,7 +695,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-01-05T11:32:39.352750567-08:00"
+    created: "2022-01-07T22:23:30.979933759-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -693,7 +715,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2022-01-05T11:32:39.351852438-08:00"
+    created: "2022-01-07T22:23:30.978416915-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -711,4 +733,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2022-01-05T11:32:39.350778967-08:00"
+generated: "2022-01-07T22:23:30.97673974-08:00"

internal/bucket/lifecycle/lifecycle.go

@@ -74,6 +74,16 @@ type Lifecycle struct {
 	Rules   []Rule   `xml:"Rule"`
 }
 
+// HasTransition returns 'true' if lifecycle document has Transition enabled.
+func (lc Lifecycle) HasTransition() bool {
+	for _, rule := range lc.Rules {
+		if rule.Transition.IsEnabled() {
+			return true
+		}
+	}
+	return false
+}
+
 // UnmarshalXML - decodes XML data.
 func (lc *Lifecycle) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
 	switch start.Name.Local {

internal/bucket/lifecycle/transition.go

@@ -106,6 +106,11 @@ type Transition struct {
 	set bool
 }
 
+// IsEnabled returns if transition is enabled.
+func (t Transition) IsEnabled() bool {
+	return t.set
+}
+
 // MarshalXML encodes transition field into an XML form.
 func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
 	if !t.set {

internal/bucket/replication/rule.go

@@ -153,6 +153,7 @@ var (
 	errDeleteReplicationMissing               = Errorf("Delete replication must be specified")
 	errInvalidDeleteReplicationStatus         = Errorf("Delete replication is either enable|disable")
 	errInvalidExistingObjectReplicationStatus = Errorf("Existing object replication status is invalid")
+	errTagsDeleteMarkerReplicationDisallowed  = Errorf("Delete marker replication is not supported if any Tag filter is specified")
 )
 
 // validateID - checks if ID is valid or not.
@@ -239,6 +240,9 @@ func (r Rule) Validate(bucket string, sameTarget bool) error {
 	if r.Destination.Bucket == bucket && sameTarget {
 		return errDestinationSourceIdentical
 	}
+	if !r.Filter.Tag.IsEmpty() && (r.DeleteMarkerReplication.Status == Enabled) {
+		return errTagsDeleteMarkerReplicationDisallowed
+	}
 	return r.ExistingObjectReplication.Validate()
 }
 

internal/config/api/api.go

@@ -43,6 +43,7 @@ const (
 	apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval"
 	apiStaleUploadsExpiry          = "stale_uploads_expiry"
 	apiDeleteCleanupInterval       = "delete_cleanup_interval"
+	apiDisableODirect              = "disable_odirect"
 
 	EnvAPIRequestsMax              = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline         = "MINIO_API_REQUESTS_DEADLINE"
@@ -59,6 +60,7 @@ const (
 	EnvAPIStaleUploadsExpiry          = "MINIO_API_STALE_UPLOADS_EXPIRY"
 	EnvAPIDeleteCleanupInterval       = "MINIO_API_DELETE_CLEANUP_INTERVAL"
 	EnvDeleteCleanupInterval          = "MINIO_DELETE_CLEANUP_INTERVAL"
+	EnvAPIDisableODirect              = "MINIO_API_DISABLE_ODIRECT"
 )
 
 // Deprecated key and ENVs
@@ -118,6 +120,10 @@ var (
 			Key:   apiDeleteCleanupInterval,
 			Value: "5m",
 		},
+		config.KV{
+			Key:   apiDisableODirect,
+			Value: "off",
+		},
 	}
 )
 
@@ -135,6 +141,7 @@ type Config struct {
 	StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
 	StaleUploadsExpiry          time.Duration `json:"stale_uploads_expiry"`
 	DeleteCleanupInterval       time.Duration `json:"delete_cleanup_interval"`
+	DisableODirect              bool          `json:"disable_odirect"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -254,6 +261,8 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	disableODirect := env.Get(EnvAPIDisableODirect, kvs.Get(apiDisableODirect)) == config.EnableOn
+
 	return Config{
 		RequestsMax:                 requestsMax,
 		RequestsDeadline:            requestsDeadline,
@@ -267,5 +276,6 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		StaleUploadsCleanupInterval: staleUploadsCleanupInterval,
 		StaleUploadsExpiry:          staleUploadsExpiry,
 		DeleteCleanupInterval:       deleteCleanupInterval,
+		DisableODirect:              disableODirect,
 	}, nil
 }

internal/config/api/help.go

@@ -94,5 +94,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiDisableODirect,
+			Description: "set to disable O_DIRECT for reads under special conditions. NOTE: it is not recommended to disable O_DIRECT without prior testing.",
+			Optional:    true,
+			Type:        "boolean",
+		},
 	}
 )

internal/dsync/drwmutex.go

@@ -234,8 +234,8 @@ func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, sourc
 			case <-refreshTimer.C:
 				refreshTimer.Reset(drwMutexRefreshInterval)
 
-				refreshed, err := refresh(ctx, dm.clnt, id, source, quorum)
-				if err == nil && !refreshed {
+				noQuorum, err := refreshLock(ctx, dm.clnt, id, source, quorum)
+				if err == nil && noQuorum {
 					// Clean the lock locally and in remote nodes
 					forceUnlock(ctx, dm.clnt, id)
 					// Execute the caller lock loss callback
@@ -273,10 +273,12 @@ func forceUnlock(ctx context.Context, ds *Dsync, id string) {
 
 type refreshResult struct {
 	offline   bool
-	succeeded bool
+	refreshed bool
 }
 
-func refresh(ctx context.Context, ds *Dsync, id, source string, quorum int) (bool, error) {
+// Refresh the given lock in all nodes, return true to indicate if a lock
+// does not exist in enough quorum nodes.
+func refreshLock(ctx context.Context, ds *Dsync, id, source string, quorum int) (bool, error) {
 	restClnts, _ := ds.GetLockers()
 
 	// Create buffered channel of size equal to total number of nodes.
@@ -302,16 +304,12 @@ func refresh(ctx context.Context, ds *Dsync, id, source string, quorum int) (boo
 			defer cancel()
 
 			refreshed, err := c.Refresh(ctx, args)
-			if refreshed && err == nil {
-				ch <- refreshResult{succeeded: true}
+			if err != nil {
+				ch <- refreshResult{offline: true}
+				log("dsync: Unable to call Refresh failed with %s for %#v at %s\n", err, args, c)
 			} else {
-				if err != nil {
-					ch <- refreshResult{offline: true}
-					log("dsync: Unable to call Refresh failed with %s for %#v at %s\n", err, args, c)
-				} else {
-					ch <- refreshResult{succeeded: false}
-					log("dsync: Refresh returned false for %#v at %s\n", args, c)
-				}
+				ch <- refreshResult{refreshed: refreshed}
+				log("dsync: Refresh returned false for %#v at %s\n", args, c)
 			}
 		}(index, c)
 	}
@@ -322,39 +320,32 @@ func refresh(ctx context.Context, ds *Dsync, id, source string, quorum int) (boo
 	// b) received too many refreshed for quorum to be still possible
 	// c) timed out
 	//
-	i, refreshFailed, refreshSucceeded := 0, 0, 0
+	lockNotFound, lockRefreshed := 0, 0
 	done := false
 
-	for ; i < len(restClnts); i++ {
+	for i := 0; i < len(restClnts); i++ {
 		select {
-		case refresh := <-ch:
-			if refresh.offline {
+		case refreshResult := <-ch:
+			if refreshResult.offline {
 				continue
 			}
-			if refresh.succeeded {
-				refreshSucceeded++
+			if refreshResult.refreshed {
+				lockRefreshed++
 			} else {
-				refreshFailed++
+				lockNotFound++
 			}
-			if refreshFailed > quorum {
-				// We know that we are not going to succeed with refresh
+			if lockRefreshed >= quorum || lockNotFound > len(restClnts)-quorum {
 				done = true
 			}
 		case <-ctx.Done():
 			// Refreshing is canceled
 			return false, ctx.Err()
 		}
-
 		if done {
 			break
 		}
 	}
 
-	refreshQuorum := refreshSucceeded >= quorum
-	if !refreshQuorum {
-		refreshQuorum = refreshFailed < quorum
-	}
-
 	// We may have some unused results in ch, release them async.
 	go func() {
 		wg.Wait()
@@ -363,7 +354,8 @@ func refresh(ctx context.Context, ds *Dsync, id, source string, quorum int) (boo
 		}
 	}()
 
-	return refreshQuorum, nil
+	noQuorum := lockNotFound > len(restClnts)-quorum
+	return noQuorum, nil
 }
 
 // lock tries to acquire the distributed lock, returning true or false.

internal/dsync/dsync-client_test.go

@@ -0,0 +1,149 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package dsync
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"net/http"
+	"net/url"
+	"time"
+
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/rest"
+)
+
+// ReconnectRESTClient is a wrapper type for rest.Client which provides reconnect on first failure.
+type ReconnectRESTClient struct {
+	u    *url.URL
+	rest *rest.Client
+}
+
+// newClient constructs a ReconnectRESTClient object with addr and endpoint initialized.
+// It _doesn't_ connect to the remote endpoint. See Call method to see when the
+// connect happens.
+func newClient(endpoint string) NetLocker {
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		panic(err)
+	}
+
+	tr := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConnsPerHost:   1024,
+		WriteBufferSize:       32 << 10, // 32KiB moving up from 4KiB default
+		ReadBufferSize:        32 << 10, // 32KiB moving up from 4KiB default
+		IdleConnTimeout:       15 * time.Second,
+		ResponseHeaderTimeout: 15 * time.Minute, // Set conservative timeouts for MinIO internode.
+		TLSHandshakeTimeout:   15 * time.Second,
+		ExpectContinueTimeout: 15 * time.Second,
+		// Go net/http automatically unzip if content-type is
+		// gzip disable this feature, as we are always interested
+		// in raw stream.
+		DisableCompression: true,
+	}
+
+	return &ReconnectRESTClient{
+		u:    u,
+		rest: rest.NewClient(u, tr, nil),
+	}
+}
+
+// Close closes the underlying socket file descriptor.
+func (restClient *ReconnectRESTClient) IsOnline() bool {
+	// If rest client has not connected yet there is nothing to close.
+	return restClient.rest != nil
+}
+
+func (restClient *ReconnectRESTClient) IsLocal() bool {
+	return false
+}
+
+// Close closes the underlying socket file descriptor.
+func (restClient *ReconnectRESTClient) Close() error {
+	return nil
+}
+
+var (
+	errLockConflict = errors.New("lock conflict")
+	errLockNotFound = errors.New("lock not found")
+)
+
+func toLockError(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	switch err.Error() {
+	case errLockConflict.Error():
+		return errLockConflict
+	case errLockNotFound.Error():
+		return errLockNotFound
+	}
+	return err
+}
+
+// Call makes a REST call to the remote endpoint using the msgp codec
+func (restClient *ReconnectRESTClient) Call(method string, args LockArgs) (status bool, err error) {
+	buf, err := args.MarshalMsg(nil)
+	if err != nil {
+		return false, err
+	}
+	body := bytes.NewReader(buf)
+	respBody, err := restClient.rest.Call(context.Background(), method,
+		url.Values{}, body, body.Size())
+	defer xhttp.DrainBody(respBody)
+
+	switch toLockError(err) {
+	case nil:
+		return true, nil
+	case errLockConflict, errLockNotFound:
+		return false, nil
+	default:
+		return false, err
+	}
+}
+
+func (restClient *ReconnectRESTClient) RLock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/rlock", args)
+}
+
+func (restClient *ReconnectRESTClient) Lock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/lock", args)
+}
+
+func (restClient *ReconnectRESTClient) RUnlock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/runlock", args)
+}
+
+func (restClient *ReconnectRESTClient) Unlock(ctx context.Context, args LockArgs) (status bool, err error) {
+	return restClient.Call("/v1/unlock", args)
+}
+
+func (restClient *ReconnectRESTClient) Refresh(ctx context.Context, args LockArgs) (refreshed bool, err error) {
+	return restClient.Call("/v1/refresh", args)
+}
+
+func (restClient *ReconnectRESTClient) ForceUnlock(ctx context.Context, args LockArgs) (reply bool, err error) {
+	return restClient.Call("/v1/force-unlock", args)
+}
+
+func (restClient *ReconnectRESTClient) String() string {
+	return restClient.u.String()
+}