Validate config before saving changes after config reset (#14203)

some upgraded objects might not get listed due
to different quorum ratios across objects.

make sure to list all objects that satisfy the
maximum possible quorum.

.github/ISSUE_TEMPLATE.md

@@ -7,6 +7,9 @@ assignees: ''
 
 ---
 
+## NOTE
+If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.
+
 <!--- Provide a general summary of the issue in the Title above -->
 
 ## Expected Behavior

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,9 @@ assignees: ''
 
 ---
 
+## NOTE
+If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.
+
 <!--- Provide a general summary of the issue in the Title above -->
 
 ## Expected Behavior

.github/lock.yml

@@ -1,39 +0,0 @@
-# Configuration for Lock Threads - https://github.com/dessant/lock-threads-app
-
-# Number of days of inactivity before a closed issue or pull request is locked
-daysUntilLock: 365
-
-# Skip issues and pull requests created before a given timestamp. Timestamp must
-# follow ISO 8601 (`YYYY-MM-DD`). Set to `false` to disable
-skipCreatedBefore: false
-
-# Issues and pull requests with these labels will be ignored. Set to `[]` to disable
-exemptLabels: []
-
-# Label to add before locking, such as `outdated`. Set to `false` to disable
-lockLabel: true
-
-# Comment to post before locking. Set to `false` to disable
-lockComment: >-
-
-  This thread has been automatically locked since there has not been
-  any recent activity after it was closed. Please open a new issue for
-  related bugs.
-
-# Assign `resolved` as the reason for locking. Set to `false` to disable
-setLockReason: true
-
-# Limit to only `issues` or `pulls`
-only: issues
-
-# Optionally, specify configuration settings just for `issues` or `pulls`
-# issues:
-#   exemptLabels:
-#     - help-wanted
-#   lockLabel: outdated
-
-# pulls:
-#   daysUntilLock: 30
-
-# Repository to extend settings from
-# _extends: repo

.github/stale.yml

@@ -14,7 +14,7 @@ onlyLabels: []
 exemptLabels:
   - "security"
   - "pending discussion"
-  - "do not close"
+  - "do-not-close"
 
 # Set to true to ignore issues in a project (defaults to false)
 exemptProjects: false

.github/workflows/iam-integrations.yaml

@@ -88,4 +88,8 @@ jobs:
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |
-          make test-site-replication
+          make test-site-replication-ldap
+      - name: Test OIDC for automatic site replication
+        if: matrix.openid == 'http://127.0.0.1:5556/dex'
+        run: |
+          make test-site-replication-oidc

.github/workflows/lock.yml

@@ -0,0 +1,25 @@
+name: 'Lock Threads'
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+
+concurrency:
+  group: lock
+
+jobs:
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/lock-threads@v3
+        with:
+          github-token: ${{ github.token }}
+          issue-inactive-days: '730'
+          exclude-any-issue-labels: 'do-not-close'
+          issue-lock-reason: 'resolved'
+          process-only: 'issues'
+          log-output: true

.github/workflows/replication.yaml

@@ -38,3 +38,9 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-replication
+      - name: Test MinIO IDP for automatic site replication
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-site-replication-minio
+

.golangci.yml

@@ -24,6 +24,15 @@ linters:
     - unconvert
     - varcheck
     - gocritic
+    - gofumpt
+
+linters-settings:
+  gofumpt:
+    lang-version: "1.17"
+
+    # Choose whether or not to use the extra rules that are disabled
+    # by default
+    extra-rules: false
 
 issues:
   exclude-use-default: false

Makefile

@@ -20,7 +20,6 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.43.0
-	@echo "Installing gofumpt" && go install mvdan.cc/gofumpt@latest
 	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
@@ -35,9 +34,7 @@ check-gen: ## check for updated autogenerated files
 
 lint: ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
-	@${GOPATH}/bin/golangci-lint cache clean
 	@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
-	@${GOPATH}/bin/gofumpt -l .
 
 check: test
 test: verifiers build ## builds minio, runs linters, tests
@@ -62,9 +59,17 @@ test-replication: install ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
 
-test-site-replication: install ## verify automatic site replication
-	@echo "Running tests for automatic site replication of IAM"
-	@(env bash $(PWD)/docs/site-replication/run-multi-site.sh)
+test-site-replication-ldap: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with LDAP)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
+
+test-site-replication-oidc: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with OIDC)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
+
+test-site-replication-minio: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"

README.md

@@ -227,21 +227,25 @@ For example: `export MINIO_SERVER_URL="https://minio.example.net"`
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
 
 # Upgrading MinIO
-MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster simultaneously and restart them, as shown in the following command from the MinIO client (mc):
+Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all transactions on MinIO are atomic. So upgrading all the servers simultaneously is the recommended way to upgrade MinIO.
 
+> NOTE: requires internet access to update directly from https://dl.min.io, optionally you can host any mirrors at https://my-artifactory.example.com/minio/
+
+- For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/minio/baremetal/reference/minio-mc-admin/mc-admin-update.html)
 ```
 mc admin update <minio alias, e.g., myminio>
 ```
 
-> NOTE: some releases might not allow rolling upgrades, this is always called out in the release notes and it is generally advised to read release notes before upgrading. In such a situation `mc admin update` is the recommended upgrading mechanism to upgrade all servers at once.
+- For deployments without external internet access (e.g. airgapped environments), download the binary from https://dl.min.io and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and do `mc admin service restart alias/`.
 
-## Important things to remember during MinIO upgrades
+- For RPM/DEB installations, upgrade packages **parallelly** on all servers. Once upgraded, perform `systemctl restart minio` across all nodes in **parallel**. RPM/DEB based installations are usually automated using [`ansible`](https://github.com/minio/ansible-minio).
 
-- Container environments are advised to update the container images instead of updating binaries inside the container.
-- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
-- `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
-- `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
-- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
+## Upgrade Checklist
+- Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
+- Read the release notes for the targeted MinIO release *before* performing any installation, there is no forced requirement to upgrade to latest releases every week. If it has a bug fix you are looking for then yes, else avoid actively upgrading a running production system.
+- Make sure MinIO process has write access to `/opt/bin` if you plan to use `mc admin update`. This is needed for MinIO to download the latest binary from https://dl.min.io and save it locally for upgrades.
+- `mc admin update` is not supported in kubernetes/container environments, container environments provide their own mechanisms for container updates.
+- **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
 
 # Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)

buildscripts/race.sh

@@ -2,6 +2,7 @@
 
 set -e
 
-for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=1 go test -v -tags kqueue -race --timeout 100m "$d"
+## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
+for d in $(go list ./... | grep -v dsync); do
+    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done

cmd/admin-bucket-handlers.go

@@ -65,7 +65,8 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	if _, err = parseBucketQuota(bucket, data); err != nil {
+	quotaConfig, err := parseBucketQuota(bucket, data)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -75,6 +76,21 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
+	bucketMeta := madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeQuotaConfig,
+		Bucket: bucket,
+		Quota:  data,
+	}
+	if quotaConfig.Quota == 0 {
+		bucketMeta.Quota = nil
+	}
+
+	// Call site replication hook.
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/admin-handler-utils.go

@@ -88,8 +88,26 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 		}
 	case SRError:
 		apiErr = errorCodes.ToAPIErrWithErr(e.Code, e.Cause)
+	case decomError:
+		apiErr = APIError{
+			Code:           "XMinioDecommissionNotAllowed",
+			Description:    e.Err,
+			HTTPStatusCode: http.StatusBadRequest,
+		}
 	default:
 		switch {
+		case errors.Is(err, errDecommissionAlreadyRunning):
+			apiErr = APIError{
+				Code:           "XMinioDecommissionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, errDecommissionComplete):
+			apiErr = APIError{
+				Code:           "XMinioDecommissionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errConfigNotFound):
 			apiErr = APIError{
 				Code:           "XMinioConfigError",
@@ -102,6 +120,18 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusForbidden,
 			}
+		case errors.Is(err, errIAMServiceAccount):
+			apiErr = APIError{
+				Code:           "XMinioIAMServiceAccount",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, errIAMServiceAccountUsed):
+			apiErr = APIError{
+				Code:           "XMinioIAMServiceAccountUsed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errIAMNotInitialized):
 			apiErr = APIError{
 				Code:           "XMinioIAMNotInitialized",

cmd/admin-handlers-config-kv.go

@@ -74,6 +74,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+	if err = validateConfig(cfg); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
 
 	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)

cmd/admin-handlers-pools.go

@@ -0,0 +1,179 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "StartDecommission")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	v := vars["pool"]
+
+	idx := globalEndpoints.GetPoolIdx(v)
+	if idx == -1 {
+		// We didn't find any matching pools, invalid input
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	if err := pools.Decommission(r.Context(), idx); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "CancelDecommission")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	v := vars["pool"]
+
+	idx := globalEndpoints.GetPoolIdx(v)
+	if idx == -1 {
+		// We didn't find any matching pools, invalid input
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	if err := pools.DecommissionCancel(ctx, idx); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "StatusPool")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	v := vars["pool"]
+
+	idx := globalEndpoints.GetPoolIdx(v)
+	if idx == -1 {
+		// We didn't find any matching pools, invalid input
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	status, err := pools.Status(r.Context(), idx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(&status))
+}
+
+func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListPools")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Legacy args style such as non-ellipses style is not supported with this API.
+	if globalEndpoints.Legacy() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	pools, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	poolsStatus := make([]PoolStatus, len(globalEndpoints))
+	for idx := range globalEndpoints {
+		status, err := pools.Status(r.Context(), idx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		poolsStatus[idx] = status
+	}
+
+	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
+}

cmd/admin-handlers-site-replication.go

@@ -45,8 +45,7 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	}
 
 	var sites []madmin.PeerSite
-	err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey)
-	if err != nil {
+	if err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -87,7 +86,7 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg); err != nil {
+	if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -178,6 +177,10 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping)
 	case madmin.SRIAMItemSTSAcc:
 		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential)
+	case madmin.SRIAMItemIAMUser:
+		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser)
+	case madmin.SRIAMItemGroupInfo:
+		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -222,6 +225,20 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
 			}
 		}
+	case madmin.SRBucketMetaTypeQuotaConfig:
+		if item.Quota == nil {
+			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil)
+		} else {
+			quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig); err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+		}
 	case madmin.SRBucketMetaTypeTags:
 		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
 	case madmin.SRBucketMetaTypeObjectLockConfig:
@@ -310,3 +327,104 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 
 	return json.Unmarshal(data, v)
 }
+
+// SiteReplicationStatus - GET /minio/admin/v3/site-replication/status
+func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationStatus")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	if objectAPI == nil {
+		return
+	}
+
+	info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(info); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo
+func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationMetaInfo")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	if objectAPI == nil {
+		return
+	}
+
+	info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(info); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
+// SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit
+func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationEdit")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+	var site madmin.PeerInfo
+	err := parseJSONBody(ctx, r.Body, &site, cred.SecretKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	body, err := json.Marshal(status)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
+//
+// used internally to tell current cluster to update endpoint for peer
+func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerEdit")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var pi madmin.PeerInfo
+	if err := parseJSONBody(ctx, r.Body, &pi, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}

cmd/admin-handlers-users.go

@@ -62,9 +62,20 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemIAMUser,
+		IAMUser: &madmin.SRIAMUser{
+			AccessKey:   accessKey,
+			IsDeleteReq: true,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
-// ListUsers - GET /minio/admin/v3/list-users?bucket={bucket}
+// ListBucketUsers - GET /minio/admin/v3/list-users?bucket={bucket}
 func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBucketUsers")
 
@@ -220,11 +231,20 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 	} else {
 		err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
 	}
-
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemGroupInfo,
+		GroupInfo: &madmin.SRGroupInfo{
+			UpdateReq: updReq,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // GetGroup - /minio/admin/v3/group?group=mygroup1
@@ -310,6 +330,20 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemGroupInfo,
+		GroupInfo: &madmin.SRGroupInfo{
+			UpdateReq: madmin.GroupAddRemove{
+				Group:    group,
+				Status:   madmin.GroupStatus(status),
+				IsRemove: false,
+			},
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
@@ -327,7 +361,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
 
-	// This API is not allowed to lookup accessKey user status
+	// This API is not allowed to lookup master access key user status
 	if accessKey == globalActiveCred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
@@ -337,6 +371,20 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemIAMUser,
+		IAMUser: &madmin.SRIAMUser{
+			AccessKey:   accessKey,
+			IsDeleteReq: false,
+			UserReq: &madmin.AddOrUpdateUserReq{
+				Status: madmin.AccountStatus(status),
+			},
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
@@ -427,6 +475,18 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemIAMUser,
+		IAMUser: &madmin.SRIAMUser{
+			AccessKey:   accessKey,
+			IsDeleteReq: false,
+			UserReq:     &ureq,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
@@ -585,14 +645,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Call hook for cluster-replication.
-	//
-	// FIXME: This wont work in an OpenID situation as the parent credential
-	// may not be present on peer clusters to provide inherited policies.
-	// Also, we should not be replicating root user's service account - as
-	// they are not authenticated by a common external IDP, so we skip when
-	// opts.ldapUser == "".
-	if _, isLDAPAccount := opts.claims[ldapUserN]; isLDAPAccount {
+	// Call hook for cluster-replication if the service account is not for a
+	// root user.
+	if newCred.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -716,14 +771,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Call site replication hook. Only LDAP accounts are supported for
-	// replication operations.
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+	// Call site replication hook - non-root user accounts are replicated.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -740,6 +789,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 			return
 		}
 	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -794,18 +844,15 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 
 	var svcAccountPolicy iampolicy.Policy
 
-	impliedPolicy := policy == nil
-
-	// If policy is empty, check for policy of the parent user
-	if !impliedPolicy {
-		svcAccountPolicy = svcAccountPolicy.Merge(*policy)
+	if policy != nil {
+		svcAccountPolicy = *policy
 	} else {
 		policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, false)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		svcAccountPolicy = svcAccountPolicy.Merge(globalIAMSys.GetCombinedPolicy(policiesNames...))
+		svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
 	}
 
 	policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
@@ -817,7 +864,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
 		AccountStatus: svcAccount.Status,
-		ImpliedPolicy: impliedPolicy,
+		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
 	}
 
@@ -963,22 +1010,14 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	// Save svc acc claims before deletion (for site replication hook).
-	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, serviceAccount)
-	if err != nil && err != errNoSuchServiceAccount {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Call site replication hook. Only LDAP accounts are supported for
-	// replication operations.
-	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+	// Call site replication hook - non-root user accounts are replicated.
+	if svcAccount.ParentUser != globalActiveCred.AccessKey {
 		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
@@ -1058,11 +1097,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	var err error
 	if !globalIsGateway {
 		// Load the latest calculated data usage
-		dataUsageInfo, err = loadDataUsageFromBackend(ctx, objectAPI)
-		if err != nil {
-			// log the error, continue with the accounting response
-			logger.LogIf(ctx, err)
-		}
+		dataUsageInfo, _ = loadDataUsageFromBackend(ctx, objectAPI)
 	}
 
 	// If etcd, dns federation configured list buckets from etcd.
@@ -1131,11 +1166,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
 			if enablePrefixUsage {
-				if pu, err := loadPrefixUsageFromBackend(ctx, objectAPI, bucket.Name); err == nil {
-					prefixUsage = pu
-				} else {
-					logger.LogIf(ctx, err)
-				}
+				prefixUsage, _ = loadPrefixUsageFromBackend(ctx, objectAPI, bucket.Name)
 			}
 
 			lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)

cmd/admin-handlers.go

@@ -978,8 +978,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	deleteBucket := func() {
-		loc := pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix)
-		objectAPI.DeleteBucket(context.Background(), loc, DeleteBucketOptions{
+		objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaBucket, "speedtest"), DeleteBucketOptions{
 			Force:      true,
 			NoRecreate: true,
 		})

cmd/admin-router.go

@@ -74,8 +74,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
-
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(gz(httpTraceAll(adminAPI.BackgroundHealStatusHandler)))
+
+			// Pool operations
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/pools/list").HandlerFunc(gz(httpTraceAll(adminAPI.ListPools)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/pools/status").HandlerFunc(gz(httpTraceAll(adminAPI.StatusPool))).Queries("pool", "{pool:.*}")
+
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
 		}
 
 		// Profiling operations
@@ -193,11 +199,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
+
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
 		}
 
 		if globalIsDistErasure {

cmd/background-newdisks-heal-ops.go

@@ -289,20 +289,20 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
-	for _, ep := range globalEndpoints {
-		for _, endpoint := range ep.Endpoints {
-			if !endpoint.IsLocal {
-				continue
-			}
-			// Try to connect to the current endpoint
-			// and reformat if the current disk is not formatted
-			disk, _, err := connectEndpoint(endpoint)
-			if errors.Is(err, errUnformattedDisk) {
-				disksToHeal = append(disksToHeal, endpoint)
-			} else if err == nil && disk != nil && disk.Healing() != nil {
-				disksToHeal = append(disksToHeal, disk.Endpoint())
-			}
+	for _, disk := range globalLocalDrives {
+		_, err := disk.GetDiskID()
+		if errors.Is(err, errUnformattedDisk) {
+			disksToHeal = append(disksToHeal, disk.Endpoint())
+			continue
 		}
+		if disk.Healing() != nil {
+			disksToHeal = append(disksToHeal, disk.Endpoint())
+		}
+	}
+	if len(disksToHeal) == globalEndpoints.NEndpoints() {
+		// When all disks == all command line endpoints
+		// this is a fresh setup, no need to trigger healing.
+		return Endpoints{}
 	}
 	return disksToHeal
 }

cmd/bitrot.go

@@ -119,8 +119,10 @@ func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath strin
 // Close all the readers.
 func closeBitrotReaders(rs []io.ReaderAt) {
 	for _, r := range rs {
-		if br, ok := r.(io.Closer); ok {
-			br.Close()
+		if r != nil {
+			if br, ok := r.(io.Closer); ok {
+				br.Close()
+			}
 		}
 	}
 }
@@ -128,8 +130,10 @@ func closeBitrotReaders(rs []io.ReaderAt) {
 // Close all the writers.
 func closeBitrotWriters(ws []io.Writer) {
 	for _, w := range ws {
-		if bw, ok := w.(io.Closer); ok {
-			bw.Close()
+		if w != nil {
+			if bw, ok := w.(io.Closer); ok {
+				bw.Close()
+			}
 		}
 	}
 }

cmd/bootstrap-peer-server.go

@@ -108,6 +108,7 @@ var skipEnvs = map[string]struct{}{
 	"MINIO_OPTS":         {},
 	"MINIO_CERT_PASSWD":  {},
 	"MINIO_SERVER_DEBUG": {},
+	"MINIO_DSYNC_TRACE":  {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {

cmd/bucket-handlers.go

@@ -576,6 +576,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Disable timeouts and cancellation
+	ctx = bgContext(ctx)
+
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
 		Versioned:        versioned,
@@ -948,6 +951,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// explicit permissions for the user.
 	if !globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
 		Action:          iampolicy.PutObjectAction,
 		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
 		BucketName:      bucket,

cmd/bucket-lifecycle.go

@@ -165,6 +165,9 @@ type transitionState struct {
 	killCh     chan struct{}
 
 	activeTasks int32
+
+	lastDayMu    sync.RWMutex
+	lastDayStats map[string]*lastDayTierStats
 }
 
 func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
@@ -186,6 +189,7 @@ func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionStat
 		ctx:          ctx,
 		objAPI:       objAPI,
 		killCh:       make(chan struct{}),
+		lastDayStats: make(map[string]*lastDayTierStats),
 	}
 }
 
@@ -213,14 +217,46 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(ctx, objectAPI, oi); err != nil {
+			var tier string
+			var err error
+			if tier, err = transitionObject(ctx, objectAPI, oi); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
 			}
 			atomic.AddInt32(&t.activeTasks, -1)
+
+			ts := tierStats{
+				TotalSize:   uint64(oi.Size),
+				NumVersions: 1,
+			}
+			if oi.IsLatest {
+				ts.NumObjects = 1
+			}
+			t.addLastDayStats(tier, ts)
 		}
 	}
 }
 
+func (t *transitionState) addLastDayStats(tier string, ts tierStats) {
+	t.lastDayMu.Lock()
+	defer t.lastDayMu.Unlock()
+
+	if _, ok := t.lastDayStats[tier]; !ok {
+		t.lastDayStats[tier] = &lastDayTierStats{}
+	}
+	t.lastDayStats[tier].addStats(ts)
+}
+
+func (t *transitionState) getDailyAllTierStats() dailyAllTierStats {
+	t.lastDayMu.RLock()
+	defer t.lastDayMu.RUnlock()
+
+	res := make(dailyAllTierStats, len(t.lastDayStats))
+	for tier, st := range t.lastDayStats {
+		res[tier] = st.clone()
+	}
+	return res
+}
+
 // UpdateWorkers at the end of this function leaves n goroutines waiting for
 // transition tasks
 func (t *transitionState) UpdateWorkers(n int) {
@@ -365,15 +401,16 @@ func genTransitionObjName(bucket string) (string, error) {
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) error {
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) (string, error) {
 	lc, err := globalLifecycleSys.Get(oi.Bucket)
 	if err != nil {
-		return err
+		return "", err
 	}
+	tier := lc.TransitionTier(oi.ToLifecycleOpts())
 	opts := ObjectOptions{
 		Transition: TransitionOptions{
 			Status: lifecycle.TransitionPending,
-			Tier:   lc.TransitionTier(oi.ToLifecycleOpts()),
+			Tier:   tier,
 			ETag:   oi.ETag,
 		},
 		VersionID:        oi.VersionID,
@@ -381,7 +418,7 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 		VersionSuspended: globalBucketVersioningSys.Suspended(oi.Bucket),
 		MTime:            oi.ModTime,
 	}
-	return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
+	return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
 
 // getTransitionedObjectReader returns a reader from the transitioned tier.

cmd/bucket-object-lock.go

@@ -58,6 +58,10 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
 // enforceRetentionForDeletion checks if it is appropriate to remove an
 // object according to locking configuration when this is lifecycle/ bucket quota asking.
 func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locked bool) {
+	if objInfo.DeleteMarker {
+		return false
+	}
+
 	lhold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
 	if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn {
 		return true

cmd/bucket-policy-handlers_test.go

@@ -37,60 +37,72 @@ import (
 func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
-			policy.NewResourceSet(policy.NewResource(bucketName, "")),
-			condition.NewFunctions(),
-		)},
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
+				policy.NewResourceSet(policy.NewResource(bucketName, "")),
+				condition.NewFunctions(),
+			),
+		},
 	}
 }
 
 func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(
-				policy.GetBucketLocationAction,
-				policy.ListBucketMultipartUploadsAction,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.GetBucketLocationAction,
+					policy.ListBucketMultipartUploadsAction,
+				),
+				policy.NewResourceSet(policy.NewResource(bucketName, "")),
+				condition.NewFunctions(),
 			),
-			policy.NewResourceSet(policy.NewResource(bucketName, "")),
-			condition.NewFunctions(),
-		)},
+		},
 	}
 }
 
 func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(policy.GetObjectAction),
-			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
-			condition.NewFunctions(),
-		)},
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetObjectAction),
+				policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+				condition.NewFunctions(),
+			),
+		},
 	}
 }
 
 func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
 	return &policy.Policy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{policy.NewStatement(
-			policy.Allow,
-			policy.NewPrincipal("*"),
-			policy.NewActionSet(
-				policy.AbortMultipartUploadAction,
-				policy.DeleteObjectAction,
-				policy.ListMultipartUploadPartsAction,
-				policy.PutObjectAction,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				"",
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.AbortMultipartUploadAction,
+					policy.DeleteObjectAction,
+					policy.ListMultipartUploadPartsAction,
+					policy.PutObjectAction,
+				),
+				policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+				condition.NewFunctions(),
 			),
-			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
-			condition.NewFunctions(),
-		)},
+		},
 	}
 }
 

cmd/bucket-quota.go

@@ -154,7 +154,7 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 	// ModTime to find the oldest objects in bucket to delete. In
 	// the context of bucket quota enforcement - number of hits are
 	// irrelevant.
-	scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
+	scorer, err := newFileScorer(toFree, UTCNow().Unix(), 1)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return
@@ -162,11 +162,6 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 
 	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
 	for obj := range objInfoCh {
-		if obj.DeleteMarker {
-			// Delete markers are automatically added for FIFO purge.
-			scorer.addFileWithObjInfo(obj, 1)
-			continue
-		}
 		// skip objects currently under retention
 		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			continue
@@ -178,6 +173,7 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 	if scorer.seenBytes <= cfg.Quota {
 		return
 	}
+
 	// Calculate how much we want to delete now.
 	toFreeNow := scorer.seenBytes - cfg.Quota
 	// We were less over quota than we thought. Adjust so we delete less.
@@ -211,9 +207,9 @@ func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket s
 		_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
 			Versioned: versioned,
 		})
-		for i := range deleteErrs {
-			if deleteErrs[i] != nil {
-				logger.LogIf(ctx, deleteErrs[i])
+		for j := range deleteErrs {
+			if deleteErrs[j] != nil {
+				logger.LogIf(ctx, deleteErrs[j])
 				continue
 			}
 

cmd/bucket-replication.go

@@ -28,6 +28,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
@@ -769,7 +770,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 	}
 
 	t, _ := tags.ParseObjectTags(oi1.UserTags)
-	if !reflect.DeepEqual(oi2.UserTags, t.ToMap()) {
+	if !reflect.DeepEqual(oi2.UserTags, t.ToMap()) || (oi2.UserTagCount != len(t.ToMap())) {
 		return replicateMetadata
 	}
 
@@ -860,8 +861,9 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		return
 	}
 	tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
-		Name: object,
-		SSEC: crypto.SSEC.IsEncrypted(objInfo.UserDefined),
+		Name:     object,
+		SSEC:     crypto.SSEC.IsEncrypted(objInfo.UserDefined),
+		UserTags: objInfo.UserTags,
 	})
 	// Lock the object name before starting replication.
 	// Use separate lock that doesn't collide with regular objects.
@@ -1184,9 +1186,17 @@ func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket,
 	defer func() {
 		if err != nil {
 			// block and abort remote upload upon failure.
-			if aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID); aerr != nil {
-				aerr = fmt.Errorf("Unable to cleanup failed multipart replication %s on remote %s/%s: %w", uploadID, bucket, object, aerr)
-				logger.LogIf(ctx, aerr)
+			attempts := 1
+			for attempts <= 3 {
+				aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID)
+				if aerr == nil {
+					return
+				}
+				logger.LogIf(ctx,
+					fmt.Errorf("Trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
+						humanize.Ordinal(attempts), uploadID, bucket, object, aerr))
+				attempts++
+				time.Sleep(time.Second)
 			}
 		}
 	}()

cmd/bucket-stats.go

@@ -38,7 +38,7 @@ func (rl ReplicationLatency) merge(other ReplicationLatency) (newReplLatency Rep
 // Get upload latency of each object size range
 func (rl ReplicationLatency) getUploadLatency() (ret map[string]uint64) {
 	ret = make(map[string]uint64)
-	avg := rl.UploadHistogram.GetAvg()
+	avg := rl.UploadHistogram.GetAvgData()
 	for k, v := range avg {
 		// Convert nanoseconds to milliseconds
 		ret[sizeTagToString(k)] = v.avg() / uint64(time.Millisecond)

cmd/bucket-targets.go

@@ -332,8 +332,10 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 }
 
 // getRemoteTargetInstanceTransport contains a singleton roundtripper.
-var getRemoteTargetInstanceTransport http.RoundTripper
-var getRemoteTargetInstanceTransportOnce sync.Once
+var (
+	getRemoteTargetInstanceTransport     http.RoundTripper
+	getRemoteTargetInstanceTransportOnce sync.Once
+)
 
 // Returns a minio-go Client configured to access remote host described in replication target config.
 func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {

cmd/common-main.go

@@ -40,6 +40,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	fcolor "github.com/fatih/color"
 	"github.com/go-openapi/loads"
 	"github.com/inconshreveable/mousetrap"
@@ -49,6 +50,7 @@ import (
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
 	"github.com/minio/kes"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -132,6 +134,8 @@ func init() {
 	console.SetColor("Debug", fcolor.New())
 
 	gob.Register(StorageErr(""))
+	gob.Register(madmin.TimeInfo{})
+	gob.Register(map[string]interface{}{})
 
 	defaultAWSCredProvider = []credentials.Provider{
 		&credentials.IAM{
@@ -649,6 +653,16 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable")
 	}
 
+	if rootDiskSize := env.Get(config.EnvRootDiskThresholdSize, ""); rootDiskSize != "" {
+		size, err := humanize.ParseBytes(rootDiskSize)
+		if err != nil {
+			logger.Fatal(err, fmt.Sprintf("Invalid %s value in environment variable", config.EnvRootDiskThresholdSize))
+		}
+		globalRootDiskThreshold = size
+	}
+
+	globalIsCICD = env.Get("MINIO_CI_CD", "") != ""
+
 	domains := env.Get(config.EnvDomain, "")
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
@@ -675,7 +689,7 @@ func handleCommonEnvVars() {
 		for _, endpoint := range minioEndpoints {
 			if net.ParseIP(endpoint) == nil {
 				// Checking if the IP is a DNS entry.
-				addrs, err := net.LookupHost(endpoint)
+				addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint)
 				if err != nil {
 					logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
 				}
@@ -908,3 +922,32 @@ func contextCanceled(ctx context.Context) bool {
 		return false
 	}
 }
+
+// bgContext returns a context that can be used for async operations.
+// Cancellation/timeouts are removed, so parent cancellations/timeout will
+// not propagate from parent.
+// Context values are preserved.
+// This can be used for goroutines that live beyond the parent context.
+func bgContext(parent context.Context) context.Context {
+	return bgCtx{parent: parent}
+}
+
+type bgCtx struct {
+	parent context.Context
+}
+
+func (a bgCtx) Done() <-chan struct{} {
+	return nil
+}
+
+func (a bgCtx) Err() error {
+	return nil
+}
+
+func (a bgCtx) Deadline() (deadline time.Time, ok bool) {
+	return time.Time{}, false
+}
+
+func (a bgCtx) Value(key interface{}) interface{} {
+	return a.parent.Value(key)
+}

cmd/config.go

@@ -35,8 +35,8 @@ import (
 
 const (
 	minioConfigPrefix = "config"
-
-	kvPrefix = ".kv"
+	minioConfigBucket = minioMetaBucket + SlashSeparator + minioConfigPrefix
+	kvPrefix          = ".kv"
 
 	// Captures all the previous SetKV operations and allows rollback.
 	minioConfigHistoryPrefix = minioConfigPrefix + "/history"

cmd/data-scanner.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io/fs"
 	"math"
 	"math/rand"
@@ -33,6 +34,7 @@ import (
 	"time"
 
 	"github.com/bits-and-blooms/bloom/v3"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -181,7 +183,8 @@ type folderScanner struct {
 	healFolderInclude     uint32 // Include a clean folder one in n cycles.
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 
-	disks []StorageAPI
+	disks       []StorageAPI
+	disksQuorum int
 
 	// If set updates will be sent regularly to this channel.
 	// Will not be closed when returned.
@@ -247,7 +250,7 @@ var globalScannerStats scannerStats
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
-func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
+func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
@@ -280,13 +283,15 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 
 	// Add disks for set healing.
-	if len(cache.Disks) > 0 {
+	if poolIdx >= 0 && setIdx >= 0 {
 		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
 		if ok {
-			s.disks = objAPI.GetDisksID(cache.Disks...)
-			if len(s.disks) != len(cache.Disks) {
-				console.Debugf(logPrefix+"Missing disks, want %d, found %d. Cannot heal. %s\n", len(cache.Disks), len(s.disks), logSuffix)
-				s.disks = s.disks[:0]
+			if poolIdx < len(objAPI.serverPools) && setIdx < len(objAPI.serverPools[poolIdx].sets) {
+				// Pass the disks belonging to the set.
+				s.disks = objAPI.serverPools[poolIdx].sets[setIdx].getDisks()
+				s.disksQuorum = len(s.disks) / 2
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Matching pool %s, set %s not found", humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1)))
 			}
 		}
 	}
@@ -623,7 +628,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		}
 
 		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
-		if !ok || len(f.disks) == 0 {
+		if !ok || len(f.disks) == 0 || f.disksQuorum == 0 {
 			break
 		}
 
@@ -645,8 +650,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		// This means that the next run will not look for it.
 		// How to resolve results.
 		resolver := metadataResolutionParams{
-			dirQuorum: getReadQuorum(len(f.disks)),
-			objQuorum: getReadQuorum(len(f.disks)),
+			dirQuorum: f.disksQuorum,
+			objQuorum: f.disksQuorum,
 			bucket:    "",
 			strict:    false,
 		}
@@ -676,8 +681,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				path:           prefix,
 				recursive:      true,
 				reportNotFound: true,
-				minDisks:       len(f.disks), // We want full consistency.
-				// Weird, maybe transient error.
+				minDisks:       f.disksQuorum,
 				agreed: func(entry metaCacheEntry) {
 					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)

cmd/data-update-tracker.go

@@ -204,14 +204,14 @@ func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
 // All of these will exit when the context is canceled.
 func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
 	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No drives specified"))
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No local drives specified"))
 		return
 	}
 	d.load(ctx, drives...)
 	go d.startCollector(ctx)
 	// startSaver will unlock.
 	d.mu.Lock()
-	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives)
+	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives...)
 }
 
 // load will attempt to load data tracking information from the supplied drives.
@@ -221,7 +221,7 @@ func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
 // If object is shared the caller should lock it.
 func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No drives specified"))
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No local drives specified"))
 		return
 	}
 	for _, drive := range drives {
@@ -246,7 +246,11 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 // startSaver will start a saver that will write d to all supplied drives at specific intervals.
 // 'd' must be write locked when started and will be unlocked.
 // The saver will save and exit when supplied context is closed.
-func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives []string) {
+func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives ...string) {
+	if len(drives) == 0 {
+		return
+	}
+
 	saveNow := d.save
 	exited := make(chan struct{})
 	d.saveExited = exited

cmd/data-usage-cache.go

@@ -225,7 +225,6 @@ type dataUsageEntryV6 struct {
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
 	Cache map[string]dataUsageEntry
-	Disks []string
 }
 
 //msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
@@ -234,35 +233,30 @@ type dataUsageCache struct {
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
 	Info  dataUsageCacheInfo
-	Disks []string
 	Cache map[string]dataUsageEntryV2
 }
 
 // dataUsageCacheV3 contains a cache of data usage entries version 3.
 type dataUsageCacheV3 struct {
 	Info  dataUsageCacheInfo
-	Disks []string
 	Cache map[string]dataUsageEntryV3
 }
 
 // dataUsageCacheV4 contains a cache of data usage entries version 4.
 type dataUsageCacheV4 struct {
 	Info  dataUsageCacheInfo
-	Disks []string
 	Cache map[string]dataUsageEntryV4
 }
 
 // dataUsageCacheV5 contains a cache of data usage entries version 5.
 type dataUsageCacheV5 struct {
 	Info  dataUsageCacheInfo
-	Disks []string
 	Cache map[string]dataUsageEntryV5
 }
 
 // dataUsageCacheV6 contains a cache of data usage entries version 6.
 type dataUsageCacheV6 struct {
 	Info  dataUsageCacheInfo
-	Disks []string
 	Cache map[string]dataUsageEntryV6
 }
 
@@ -1037,7 +1031,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			return err
 		}
 		d.Info = dold.Info
-		d.Disks = dold.Disks
 		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
 		for k, v := range dold.Cache {
 			d.Cache[k] = dataUsageEntry{
@@ -1061,7 +1054,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			return err
 		}
 		d.Info = dold.Info
-		d.Disks = dold.Disks
 		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
 		for k, v := range dold.Cache {
 			due := dataUsageEntry{
@@ -1101,7 +1093,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			return err
 		}
 		d.Info = dold.Info
-		d.Disks = dold.Disks
 		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
 		for k, v := range dold.Cache {
 			due := dataUsageEntry{
@@ -1153,7 +1144,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			return err
 		}
 		d.Info = dold.Info
-		d.Disks = dold.Disks
 		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
 		for k, v := range dold.Cache {
 			due := dataUsageEntry{
@@ -1205,7 +1195,6 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			return err
 		}
 		d.Info = dold.Info
-		d.Disks = dold.Disks
 		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
 		for k, v := range dold.Cache {
 			var replicationStats *replicationAllStats

cmd/data-usage-cache_gen.go

@@ -338,25 +338,6 @@ func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 				z.Cache[za0001] = za0002
 			}
-		case "Disks":
-			var zb0003 uint32
-			zb0003, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0003) {
-				z.Disks = (z.Disks)[:zb0003]
-			} else {
-				z.Disks = make([]string, zb0003)
-			}
-			for za0003 := range z.Disks {
-				z.Disks[za0003], err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0003)
-					return
-				}
-			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -370,9 +351,9 @@ func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
+	// map header, size 2
 	// write "Info"
-	err = en.Append(0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	err = en.Append(0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
 	if err != nil {
 		return
 	}
@@ -403,32 +384,15 @@ func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	// write "Disks"
-	err = en.Append(0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.Disks)))
-	if err != nil {
-		err = msgp.WrapError(err, "Disks")
-		return
-	}
-	for za0003 := range z.Disks {
-		err = en.WriteString(z.Disks[za0003])
-		if err != nil {
-			err = msgp.WrapError(err, "Disks", za0003)
-			return
-		}
-	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageCache) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
+	// map header, size 2
 	// string "Info"
-	o = append(o, 0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	o = append(o, 0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
 	o, err = z.Info.MarshalMsg(o)
 	if err != nil {
 		err = msgp.WrapError(err, "Info")
@@ -445,12 +409,6 @@ func (z *dataUsageCache) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
-	// string "Disks"
-	o = append(o, 0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.Disks)))
-	for za0003 := range z.Disks {
-		o = msgp.AppendString(o, z.Disks[za0003])
-	}
 	return
 }
 
@@ -508,25 +466,6 @@ func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				}
 				z.Cache[za0001] = za0002
 			}
-		case "Disks":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0003) {
-				z.Disks = (z.Disks)[:zb0003]
-			} else {
-				z.Disks = make([]string, zb0003)
-			}
-			for za0003 := range z.Disks {
-				z.Disks[za0003], bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0003)
-					return
-				}
-			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -548,10 +487,6 @@ func (z *dataUsageCache) Msgsize() (s int) {
 			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
-	s += 6 + msgp.ArrayHeaderSize
-	for za0003 := range z.Disks {
-		s += msgp.StringPrefixSize + len(z.Disks[za0003])
-	}
 	return
 }
 
@@ -811,54 +746,35 @@ func (z *dataUsageCacheV2) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV2, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV2, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				zb0003--
-				var za0002 string
-				var za0003 dataUsageEntryV2
-				za0002, err = dc.ReadString()
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntryV2
+				za0001, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				err = za0003.DecodeMsg(dc)
+				err = za0002.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			err = dc.Skip()
@@ -895,54 +811,35 @@ func (z *dataUsageCacheV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV2, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV2, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				var za0002 string
-				var za0003 dataUsageEntryV2
-				zb0003--
-				za0002, bts, err = msgp.ReadStringBytes(bts)
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntryV2
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				bts, err = za0003.UnmarshalMsg(bts)
+				bts, err = za0002.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -958,15 +855,11 @@ func (z *dataUsageCacheV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheV2) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
-	for za0001 := range z.Disks {
-		s += msgp.StringPrefixSize + len(z.Disks[za0001])
-	}
-	s += 6 + msgp.MapHeaderSize
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
 	if z.Cache != nil {
-		for za0002, za0003 := range z.Cache {
-			_ = za0003
-			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
 	return
@@ -996,54 +889,35 @@ func (z *dataUsageCacheV3) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV3, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV3, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				zb0003--
-				var za0002 string
-				var za0003 dataUsageEntryV3
-				za0002, err = dc.ReadString()
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntryV3
+				za0001, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				err = za0003.DecodeMsg(dc)
+				err = za0002.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			err = dc.Skip()
@@ -1080,54 +954,35 @@ func (z *dataUsageCacheV3) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV3, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV3, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				var za0002 string
-				var za0003 dataUsageEntryV3
-				zb0003--
-				za0002, bts, err = msgp.ReadStringBytes(bts)
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntryV3
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				bts, err = za0003.UnmarshalMsg(bts)
+				bts, err = za0002.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -1143,15 +998,11 @@ func (z *dataUsageCacheV3) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheV3) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
-	for za0001 := range z.Disks {
-		s += msgp.StringPrefixSize + len(z.Disks[za0001])
-	}
-	s += 6 + msgp.MapHeaderSize
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
 	if z.Cache != nil {
-		for za0002, za0003 := range z.Cache {
-			_ = za0003
-			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
 	return
@@ -1181,54 +1032,35 @@ func (z *dataUsageCacheV4) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV4, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV4, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				zb0003--
-				var za0002 string
-				var za0003 dataUsageEntryV4
-				za0002, err = dc.ReadString()
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntryV4
+				za0001, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				err = za0003.DecodeMsg(dc)
+				err = za0002.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			err = dc.Skip()
@@ -1265,54 +1097,35 @@ func (z *dataUsageCacheV4) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV4, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV4, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				var za0002 string
-				var za0003 dataUsageEntryV4
-				zb0003--
-				za0002, bts, err = msgp.ReadStringBytes(bts)
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntryV4
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				bts, err = za0003.UnmarshalMsg(bts)
+				bts, err = za0002.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -1328,15 +1141,11 @@ func (z *dataUsageCacheV4) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheV4) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
-	for za0001 := range z.Disks {
-		s += msgp.StringPrefixSize + len(z.Disks[za0001])
-	}
-	s += 6 + msgp.MapHeaderSize
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
 	if z.Cache != nil {
-		for za0002, za0003 := range z.Cache {
-			_ = za0003
-			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
 	return
@@ -1366,54 +1175,35 @@ func (z *dataUsageCacheV5) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV5, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV5, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				zb0003--
-				var za0002 string
-				var za0003 dataUsageEntryV5
-				za0002, err = dc.ReadString()
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntryV5
+				za0001, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				err = za0003.DecodeMsg(dc)
+				err = za0002.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			err = dc.Skip()
@@ -1450,54 +1240,35 @@ func (z *dataUsageCacheV5) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV5, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV5, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				var za0002 string
-				var za0003 dataUsageEntryV5
-				zb0003--
-				za0002, bts, err = msgp.ReadStringBytes(bts)
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntryV5
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				bts, err = za0003.UnmarshalMsg(bts)
+				bts, err = za0002.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -1513,15 +1284,11 @@ func (z *dataUsageCacheV5) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheV5) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
-	for za0001 := range z.Disks {
-		s += msgp.StringPrefixSize + len(z.Disks[za0001])
-	}
-	s += 6 + msgp.MapHeaderSize
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
 	if z.Cache != nil {
-		for za0002, za0003 := range z.Cache {
-			_ = za0003
-			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
 	return
@@ -1551,54 +1318,35 @@ func (z *dataUsageCacheV6) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], err = dc.ReadString()
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV6, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV6, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				zb0003--
-				var za0002 string
-				var za0003 dataUsageEntryV6
-				za0002, err = dc.ReadString()
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntryV6
+				za0001, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				err = za0003.DecodeMsg(dc)
+				err = za0002.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			err = dc.Skip()
@@ -1635,54 +1383,35 @@ func (z *dataUsageCacheV6) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Info")
 				return
 			}
-		case "Disks":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Disks")
-				return
-			}
-			if cap(z.Disks) >= int(zb0002) {
-				z.Disks = (z.Disks)[:zb0002]
-			} else {
-				z.Disks = make([]string, zb0002)
-			}
-			for za0001 := range z.Disks {
-				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Disks", za0001)
-					return
-				}
-			}
 		case "Cache":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Cache")
 				return
 			}
 			if z.Cache == nil {
-				z.Cache = make(map[string]dataUsageEntryV6, zb0003)
+				z.Cache = make(map[string]dataUsageEntryV6, zb0002)
 			} else if len(z.Cache) > 0 {
 				for key := range z.Cache {
 					delete(z.Cache, key)
 				}
 			}
-			for zb0003 > 0 {
-				var za0002 string
-				var za0003 dataUsageEntryV6
-				zb0003--
-				za0002, bts, err = msgp.ReadStringBytes(bts)
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntryV6
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Cache")
 					return
 				}
-				bts, err = za0003.UnmarshalMsg(bts)
+				bts, err = za0002.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "Cache", za0002)
+					err = msgp.WrapError(err, "Cache", za0001)
 					return
 				}
-				z.Cache[za0002] = za0003
+				z.Cache[za0001] = za0002
 			}
 		default:
 			bts, err = msgp.Skip(bts)
@@ -1698,15 +1427,11 @@ func (z *dataUsageCacheV6) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheV6) Msgsize() (s int) {
-	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
-	for za0001 := range z.Disks {
-		s += msgp.StringPrefixSize + len(z.Disks[za0001])
-	}
-	s += 6 + msgp.MapHeaderSize
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
 	if z.Cache != nil {
-		for za0002, za0003 := range z.Cache {
-			_ = za0003
-			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
 	return

cmd/data-usage.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
@@ -90,7 +91,7 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	buf, err := readConfig(ctx, objAPI, dataUsageObjNamePath)
 	if err != nil {
-		if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
+		if errors.Is(err, errConfigNotFound) {
 			return DataUsageInfo{}, nil
 		}
 		return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)

cmd/data-usage_test.go

@@ -67,7 +67,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return
 	}
 
-	got, err := scanDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
+	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -178,7 +178,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), base, got, getSize)
+		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -289,7 +289,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return
 	}
-	got, err := scanDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
+	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -423,7 +423,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), base, got, getSize)
+		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -575,7 +575,7 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return
 	}
-	want, err := scanDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
+	want, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/disk-cache.go

@@ -1153,7 +1153,7 @@ func (c *cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	if err == nil {
 		// fill cache in the background
 		go func() {
-			_, err := dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, oi, opts)
+			_, err := dcache.CompleteMultipartUpload(bgContext(ctx), bucket, object, uploadID, uploadedParts, oi, opts)
 			if err != nil {
 				// fill cache in the background
 				bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})

cmd/endpoint-ellipses.go

@@ -353,9 +353,11 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			return nil, -1, err
 		}
 		endpointServerPools = append(endpointServerPools, PoolEndpoints{
+			Legacy:       true,
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
+			CmdLine:      strings.Join(args, " "),
 		})
 		setupType = newSetupType
 		return endpointServerPools, setupType, nil
@@ -376,6 +378,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
+			CmdLine:      arg,
 		}); err != nil {
 			return nil, -1, err
 		}

cmd/endpoint.go

@@ -197,14 +197,28 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 // PoolEndpoints represent endpoints in a given pool
 // along with its setCount and setDriveCount.
 type PoolEndpoints struct {
+	// indicates if endpoints are provided in non-ellipses style
+	Legacy       bool
 	SetCount     int
 	DrivesPerSet int
 	Endpoints    Endpoints
+	CmdLine      string
 }
 
 // EndpointServerPools - list of list of endpoints
 type EndpointServerPools []PoolEndpoints
 
+// GetPoolIdx return pool index
+func (l EndpointServerPools) GetPoolIdx(pool string) int {
+	for id, ep := range globalEndpoints {
+		if ep.CmdLine != pool {
+			continue
+		}
+		return id
+	}
+	return -1
+}
+
 // GetLocalPoolIdx returns the pool which endpoint belongs to locally.
 // if ep is remote this code will return -1 poolIndex
 func (l EndpointServerPools) GetLocalPoolIdx(ep Endpoint) int {
@@ -220,6 +234,13 @@ func (l EndpointServerPools) GetLocalPoolIdx(ep Endpoint) int {
 	return -1
 }
 
+// Legacy returns 'true' if the MinIO server commandline was
+// provided with no ellipses pattern, those are considered
+// legacy deployments.
+func (l EndpointServerPools) Legacy() bool {
+	return len(l) == 1 && l[0].Legacy
+}
+
 // Add add pool endpoints
 func (l *EndpointServerPools) Add(zeps PoolEndpoints) error {
 	existSet := set.NewStringSet()
@@ -622,20 +643,33 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		}
 	}
 
-	// Check whether same path is not used in endpoints of a host on different port.
-	{
-		pathIPMap := make(map[string]set.StringSet)
-		for _, endpoint := range endpoints {
-			host := endpoint.Hostname()
-			hostIPSet, _ := getHostIP(host)
-			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
-				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					return endpoints, setupType,
-						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
+	orchestrated := IsKubernetes() || IsDocker()
+	if !orchestrated {
+		// Check whether same path is not used in endpoints of a host on different port.
+		// Only verify this on baremetal setups, DNS is not available in orchestrated
+		// environments so we can't do much here.
+		{
+			pathIPMap := make(map[string]set.StringSet)
+			hostIPCache := make(map[string]set.StringSet)
+			for _, endpoint := range endpoints {
+				host := endpoint.Hostname()
+				hostIPSet, ok := hostIPCache[host]
+				if !ok {
+					hostIPSet, err = getHostIP(host)
+					if err != nil {
+						return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+					}
+					hostIPCache[host] = hostIPSet
+				}
+				if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+					if !IPSet.Intersection(hostIPSet).IsEmpty() {
+						return endpoints, setupType,
+							config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
+					}
+					pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+				} else {
+					pathIPMap[endpoint.Path] = hostIPSet
 				}
-				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
-			} else {
-				pathIPMap[endpoint.Path] = hostIPSet
 			}
 		}
 	}

cmd/erasure-common.go

@@ -22,6 +22,38 @@ import (
 	"sync"
 )
 
+func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
+	disks := er.getDisks()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if disks[i-1] == nil {
+				return
+			}
+			di, err := disks[i-1].DiskInfo(context.Background())
+			if err != nil || di.Healing {
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				// - Do not consume disks which are being healed
+				//
+				// - Future: skip busy disks
+				return
+			}
+
+			mu.Lock()
+			newDisks = append(newDisks, disks[i-1])
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+	return newDisks
+}
+
 func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	// Based on the random shuffling return back randomized disks.

cmd/erasure-decode.go

@@ -322,8 +322,8 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 			errs:        make([]error, len(writers)),
 		}
 
-		err = w.Write(ctx, bufs)
-		if err != nil {
+		if err = w.Write(ctx, bufs); err != nil {
+			logger.LogIf(ctx, err)
 			return err
 		}
 	}

cmd/erasure-encode.go

@@ -52,7 +52,10 @@ func (p *parallelWriter) Write(ctx context.Context, blocks [][]byte) error {
 			if p.errs[i] == nil {
 				if n != len(blocks[i]) {
 					p.errs[i] = io.ErrShortWrite
+					p.writers[i] = nil
 				}
+			} else {
+				p.writers[i] = nil
 			}
 		}(i)
 	}
@@ -93,6 +96,7 @@ func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer
 		blocks, err = e.EncodeData(ctx, buf[:n])
 		if err != nil {
 			logger.LogIf(ctx, err)
+
 			return 0, err
 		}
 

cmd/erasure-healing.go

@@ -246,7 +246,7 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
-	case errors.Is(erErr, errCorruptedFormat):
+	case errors.Is(erErr, errFileCorrupt):
 		return true
 	}
 	if erErr == nil {
@@ -312,13 +312,13 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
-		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
+		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, nil, opts)
 	}
 
 	result.ParityBlocks = result.DiskCount - readQuorum
 	result.DataBlocks = readQuorum
 
-	// List of disks having latest version of the object er.meta
+	// List of disks having latest version of the object xl.meta
 	// (by modtime).
 	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
@@ -356,10 +356,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			// If data is sane on any one disk, we can
 			// extract the correct object size.
 			result.ObjectSize = partsMetadata[i].Size
-			if partsMetadata[i].Erasure.ParityBlocks > 0 && partsMetadata[i].Erasure.DataBlocks > 0 {
-				result.ParityBlocks = partsMetadata[i].Erasure.ParityBlocks
-				result.DataBlocks = partsMetadata[i].Erasure.DataBlocks
-			}
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
 		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
@@ -406,7 +402,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
-	if numAvailableDisks < result.DataBlocks {
+	if numAvailableDisks < readQuorum {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, dataErrs, opts)
 	}
 
@@ -466,9 +462,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	copyPartsMetadata = shufflePartsMetadata(copyPartsMetadata, latestMeta.Erasure.Distribution)
 
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		result.DataBlocks = latestMeta.Erasure.DataBlocks
-		result.ParityBlocks = latestMeta.Erasure.ParityBlocks
-
 		// Heal each part. erasureHealFile() will write the healed
 		// part to .minio/tmp/uuid/ which needs to be renamed later to
 		// the final location.
@@ -547,14 +540,14 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 			// If all disks are having errors, we give up.
 			if disksToHealCount == 0 {
-				return result, fmt.Errorf("all disks had write errors, unable to heal")
+				return result, fmt.Errorf("all disks had write errors, unable to heal %s/%s", bucket, object)
 			}
 
 		}
 
 	}
 
-	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpID, len(storageDisks)/2+1)
+	defer er.renameAll(context.Background(), minioMetaTmpBucket, tmpID)
 
 	// Rename from tmp location to the actual location.
 	for i, disk := range outDatedDisks {
@@ -819,21 +812,20 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 	// remove we simply delete it from namespace.
 	m, ok := isObjectDangling(metaArr, errs, dataErrs)
 	if ok {
-		parityBlocks := m.Erasure.ParityBlocks
-		if m.Erasure.ParityBlocks == 0 {
-			parityBlocks = er.defaultParityCount
-		}
-		dataBlocks := m.Erasure.DataBlocks
-		if m.Erasure.DataBlocks == 0 {
-			dataBlocks = len(storageDisks) - parityBlocks
+		parityBlocks := er.defaultParityCount
+		dataBlocks := len(storageDisks) - parityBlocks
+		if m.IsValid() {
+			parityBlocks = m.Erasure.ParityBlocks
+			dataBlocks = m.Erasure.DataBlocks
 		}
+
 		writeQuorum := dataBlocks
 		if dataBlocks == parityBlocks {
 			writeQuorum++
 		}
+
 		var err error
-		var returnNotFound bool
-		if !opts.DryRun && opts.Remove {
+		if opts.Remove {
 			err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{
 				VersionID: versionID,
 			}, false)
@@ -849,19 +841,10 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 				m.Size = 0
 			}
 
-			// Delete successfully purged dangling content, return ObjectNotFound/VersionNotFound instead.
-			if countErrs(errs, nil) == len(errs) {
-				returnNotFound = true
-			}
-		}
-		if returnNotFound {
-			err = toObjectErr(errFileNotFound, bucket, object)
-			if versionID != "" {
-				err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-			}
-			return er.defaultHealResult(m, storageDisks, storageEndpoints,
-				errs, bucket, object, versionID), err
+			return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+				errs, bucket, object, versionID), nil
 		}
+
 		return er.defaultHealResult(m, storageDisks, storageEndpoints,
 			errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID)
 	}
@@ -878,54 +861,64 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 // files is lesser than number of data blocks.
 func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (validMeta FileInfo, ok bool) {
 	// We can consider an object data not reliable
-	// when er.meta is not found in read quorum disks.
-	// or when er.meta is not readable in read quorum disks.
-	var notFoundErasureMeta, corruptedErasureMeta int
-	for _, readErr := range errs {
-		if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
-			notFoundErasureMeta++
-		} else if errors.Is(readErr, errCorruptedFormat) {
-			corruptedErasureMeta++
-		}
-	}
-	var notFoundParts int
-	for i := range dataErrs {
-		// Only count part errors, if the error is not
-		// same as er.meta error. This is to avoid
-		// double counting when both parts and er.meta
-		// are not available.
-		if errs[i] != dataErrs[i] {
-			if IsErr(dataErrs[i], []error{
-				errFileNotFound,
-				errFileVersionNotFound,
-			}...) {
-				notFoundParts++
+	// when xl.meta is not found in read quorum disks.
+	// or when xl.meta is not readable in read quorum disks.
+	danglingErrsCount := func(cerrs []error) (int, int) {
+		var (
+			notFoundCount  int
+			corruptedCount int
+		)
+		for _, readErr := range cerrs {
+			if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
+				notFoundCount++
+			} else if errors.Is(readErr, errFileCorrupt) {
+				corruptedCount++
 			}
 		}
+		return notFoundCount, corruptedCount
 	}
 
+	ndataErrs := make([]error, len(dataErrs))
+	for i := range dataErrs {
+		if errs[i] != dataErrs[i] {
+			// Only count part errors, if the error is not
+			// same as xl.meta error. This is to avoid
+			// double counting when both parts and xl.meta
+			// are not available.
+			ndataErrs[i] = dataErrs[i]
+		}
+	}
+
+	notFoundMetaErrs, corruptedMetaErrs := danglingErrsCount(errs)
+	notFoundPartsErrs, corruptedPartsErrs := danglingErrsCount(ndataErrs)
+
 	for _, m := range metaArr {
-		if !m.IsValid() {
-			continue
+		if m.IsValid() {
+			validMeta = m
+			break
 		}
-		validMeta = m
-		break
 	}
 
-	if validMeta.Deleted || validMeta.IsRemote() {
-		// notFoundParts is ignored since a
+	if !validMeta.IsValid() {
+		// We have no idea what this file is, leave it as is.
+		return validMeta, false
+	}
+
+	if validMeta.Deleted {
+		// notFoundPartsErrs is ignored since
 		// - delete marker does not have any parts
-		// - transition status of complete has no parts
-		return validMeta, corruptedErasureMeta+notFoundErasureMeta > len(errs)/2
+		return validMeta, corruptedMetaErrs+notFoundMetaErrs > len(errs)/2
 	}
 
-	// We couldn't find any valid meta we are indeed corrupted, return true right away.
-	if validMeta.Erasure.DataBlocks == 0 {
-		return validMeta, true
+	totalErrs := notFoundMetaErrs + corruptedMetaErrs + notFoundPartsErrs + corruptedPartsErrs
+	if validMeta.IsRemote() {
+		// notFoundPartsErrs is ignored since
+		// - transition status of complete has no parts
+		totalErrs = notFoundMetaErrs + corruptedMetaErrs
 	}
 
 	// We have valid meta, now verify if we have enough files with parity blocks.
-	return validMeta, corruptedErasureMeta+notFoundErasureMeta+notFoundParts > validMeta.Erasure.ParityBlocks
+	return validMeta, totalErrs > validMeta.Erasure.ParityBlocks
 }
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.

cmd/erasure-healing_test.go

@@ -34,6 +34,192 @@ import (
 	"github.com/minio/madmin-go"
 )
 
+// Tests isObjectDangling function
+func TestIsObjectDangling(t *testing.T) {
+	fi := newFileInfo("test-object", 2, 2)
+	fi.Erasure.Index = 1
+
+	testCases := []struct {
+		name             string
+		metaArr          []FileInfo
+		errs             []error
+		dataErrs         []error
+		expectedMeta     FileInfo
+		expectedDangling bool
+	}{
+		{
+			name: "FileInfoExists-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				fi,
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				nil,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoExists-case2",
+			metaArr: []FileInfo{
+				{},
+				{},
+				fi,
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoUndecided-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				errDiskNotFound,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name:    "FileInfoUndecided-case2",
+			metaArr: []FileInfo{},
+			errs: []error{
+				errFileNotFound,
+				errDiskNotFound,
+				errDiskNotFound,
+				errFileNotFound,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{},
+			expectedDangling: false,
+		},
+		{
+			name:    "FileInfoUndecided-case3(file deleted)",
+			metaArr: []FileInfo{},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{},
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoDecided-case1",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     fi,
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2-delete-marker",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				{Deleted: true},
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     FileInfo{Deleted: true},
+			expectedDangling: true,
+		},
+		{
+			name: "FileInfoDecided-case2-(duplicate data errors)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				{Deleted: true},
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				nil,
+				nil,
+			},
+			expectedMeta:     FileInfo{Deleted: true},
+			expectedDangling: true,
+		},
+		// Add new cases as seen
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.name, func(t *testing.T) {
+			gotMeta, dangling := isObjectDangling(testCase.metaArr, testCase.errs, testCase.dataErrs)
+			if !gotMeta.Equals(testCase.expectedMeta) {
+				t.Errorf("Expected %#v, got %#v", testCase.expectedMeta, gotMeta)
+			}
+			if dangling != testCase.expectedDangling {
+				t.Errorf("Expected dangling %t, got %t", testCase.expectedDangling, dangling)
+			}
+		})
+	}
+}
+
 // Tests both object and bucket healing.
 func TestHealing(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())

cmd/erasure-multipart.go

@@ -29,6 +29,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -392,6 +393,53 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 	return partInfo, nil
 }
 
+func undoRenamePart(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, errs []error) {
+	// Undo rename object on disks where RenameFile succeeded.
+	g := errgroup.WithNErrs(len(disks))
+	for index, disk := range disks {
+		if disk == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			if errs[index] == nil {
+				_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
+			}
+			return nil
+		}, index)
+	}
+	g.Wait()
+}
+
+// renamePart - renames multipart part to its relevant location under uploadID.
+func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
+	g := errgroup.WithNErrs(len(disks))
+
+	// Rename file on all underlying storage disks.
+	for index := range disks {
+		index := index
+		g.Go(func() error {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			return disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry)
+		}, index)
+	}
+
+	// Wait for all renames to finish.
+	errs := g.Wait()
+
+	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
+	// otherwise return failure. Cleanup successful renames.
+	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	if err == errErasureWriteQuorum {
+		// Undo all the partial rename operations.
+		undoRenamePart(disks, srcBucket, srcEntry, dstBucket, dstEntry, errs)
+	}
+
+	return evalDisks(disks, errs), err
+}
+
 // PutObjectPart - reads incoming stream and internally erasure codes
 // them. This call is similar to single put operation but it is part
 // of the multipart transaction.
@@ -481,7 +529,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.deleteObject(context.Background(), minioMetaTmpBucket, tmpPart, writeQuorum)
+			er.renameAll(context.Background(), minioMetaTmpBucket, tmpPart)
 		}
 	}()
 
@@ -521,7 +569,22 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
-	n, err := erasure.Encode(pctx, data, writers, buffer, writeQuorum)
+	toEncode := io.Reader(data)
+	if data.Size() > bigFileThreshold {
+		// Add input readahead.
+		// We use 2 buffers, so we always have a full buffer of input.
+		bufA := er.bp.Get()
+		bufB := er.bp.Get()
+		defer er.bp.Put(bufA)
+		defer er.bp.Put(bufB)
+		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
+		if err == nil {
+			toEncode = ra
+			defer ra.Close()
+		}
+	}
+
+	n, err := erasure.Encode(pctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
@@ -555,7 +618,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	// Rename temporary part file to its final location.
 	partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix)
-	onlineDisks, err = rename(wctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, false, writeQuorum, nil)
+	onlineDisks, err = renamePart(wctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
@@ -978,21 +1041,8 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 		return toObjectErr(err, bucket, object, uploadID)
 	}
 
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-
-	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "", false)
-
-	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
-	if err != nil {
-		return toObjectErr(err, bucket, object, uploadID)
-	}
-
 	// Cleanup all uploaded parts.
-	if err = er.deleteObject(ctx, minioMetaMultipartBucket, uploadIDPath, writeQuorum); err != nil {
-		return toObjectErr(err, bucket, object, uploadID)
-	}
+	er.renameAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
 
 	// Successfully purged.
 	return nil

cmd/erasure-object.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/klauspost/readahead"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -408,7 +409,8 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}
 
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
-		if reducedErr == errErasureReadQuorum && bucket != minioMetaBucket {
+		if errors.Is(reducedErr, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+			// Skip buckets that live at `.minio.sys` bucket.
 			if _, ok := isObjectDangling(metaArr, errs, nil); ok {
 				reducedErr = errFileNotFound
 				if opts.VersionID != "" {
@@ -522,31 +524,6 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 	return objInfo, wquorum, nil
 }
 
-func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, errs []error) {
-	// Undo rename object on disks where RenameFile succeeded.
-
-	// If srcEntry/dstEntry are objects then add a trailing slash to copy
-	// over all the parts inside the object directory
-	if isDir {
-		srcEntry = retainSlash(srcEntry)
-		dstEntry = retainSlash(dstEntry)
-	}
-	g := errgroup.WithNErrs(len(disks))
-	for index, disk := range disks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			if errs[index] == nil {
-				_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
-			}
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
 func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
 	defer NSUpdated(dstBucket, dstEntry)
@@ -587,46 +564,6 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 	return evalDisks(disks, errs), err
 }
 
-// rename - common function that renamePart and renameObject use to rename
-// the respective underlying storage layer representations.
-func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
-	if isDir {
-		dstEntry = retainSlash(dstEntry)
-		srcEntry = retainSlash(srcEntry)
-	}
-	defer NSUpdated(dstBucket, dstEntry)
-
-	g := errgroup.WithNErrs(len(disks))
-
-	// Rename file on all underlying storage disks.
-	for index := range disks {
-		index := index
-		g.Go(func() error {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			if err := disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
-				if !IsErrIgnored(err, ignoredErr...) {
-					return err
-				}
-			}
-			return nil
-		}, index)
-	}
-
-	// Wait for all renames to finish.
-	errs := g.Wait()
-
-	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
-	// otherwise return failure. Cleanup successful renames.
-	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if err == errErasureWriteQuorum {
-		// Undo all the partial rename operations.
-		undoRename(disks, srcBucket, srcEntry, dstBucket, dstEntry, isDir, errs)
-	}
-	return evalDisks(disks, errs), err
-}
-
 func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	data := r.Reader
 
@@ -700,9 +637,10 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		if disk == nil {
 			continue
 		}
-
-		inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
-		writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
+		if disk.IsOnline() {
+			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
+			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
+		}
 	}
 
 	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
@@ -719,6 +657,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 
 	for i, w := range writers {
 		if w == nil {
+			// Make sure to avoid writing to disks which we couldn't complete in erasure.Encode()
 			onlineDisks[i] = nil
 			continue
 		}
@@ -901,7 +840,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	var online int
 	defer func() {
 		if online != len(onlineDisks) {
-			er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
+			er.renameAll(context.Background(), minioMetaTmpBucket, tempObj)
 		}
 	}()
 
@@ -929,6 +868,10 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			continue
 		}
 
+		if !disk.IsOnline() {
+			continue
+		}
+
 		if len(inlineBuffers) > 0 {
 			sz := shardFileSize
 			if sz < 0 {
@@ -938,10 +881,25 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 			continue
 		}
+
 		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
-	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
+	toEncode := io.Reader(data)
+	if data.Size() > bigFileThreshold {
+		// We use 2 buffers, so we always have a full buffer of input.
+		bufA := er.bp.Get()
+		bufB := er.bp.Get()
+		defer er.bp.Put(bufA)
+		defer er.bp.Put(bufB)
+		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
+		if err == nil {
+			toEncode = ra
+			defer ra.Close()
+		}
+		logger.LogIf(ctx, err)
+	}
+	n, erasureErr := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if erasureErr != nil {
 		return ObjectInfo{}, toObjectErr(erasureErr, minioMetaTmpBucket, tempErasureObj)
@@ -1011,6 +969,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// Rename the successfully written temporary object to final location.
 	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum); err != nil {
+		if errors.Is(err, errFileNotFound) {
+			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
+		}
 		logger.LogIf(ctx, err)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -1059,42 +1020,6 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
 }
 
-// deleteObject - wrapper for delete object, deletes an object from
-// all the disks in parallel, including `xl.meta` associated with the
-// object.
-func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string, writeQuorum int) error {
-	var err error
-
-	disks := er.getDisks()
-	tmpObj := mustGetUUID()
-	if bucket == minioMetaTmpBucket {
-		tmpObj = object
-	} else {
-		// Rename the current object while requiring write quorum, but also consider
-		// that a non found object in a given disk as a success since it already
-		// confirms that the object doesn't have a part in that disk (already removed)
-		disks, err = rename(ctx, disks, bucket, object, minioMetaTmpBucket, tmpObj, true, writeQuorum,
-			[]error{errFileNotFound})
-		if err != nil {
-			return toObjectErr(err, bucket, object)
-		}
-	}
-
-	g := errgroup.WithNErrs(len(disks))
-	for index := range disks {
-		index := index
-		g.Go(func() error {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			return disks[index].Delete(ctx, minioMetaTmpBucket, tmpObj, true)
-		}, index)
-	}
-
-	// return errors if any during deletion
-	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
-}
-
 // DeleteObjects deletes objects/versions in bulk, this function will still automatically split objects list
 // into smaller bulks if some object names are found to be duplicated in the delete list, splitting
 // into smaller bulks will avoid holding twice the write lock of the duplicated object names.
@@ -1112,7 +1037,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		// class for objects which have reduced quorum
 		// storage class only needs to be honored for
 		// Read() requests alone which we already do.
-		writeQuorums[i] = er.defaultWQuorum()
+		writeQuorums[i] = len(storageDisks)/2 + 1
 	}
 
 	versionsMap := make(map[string]FileInfoVersions, len(objects))
@@ -1197,6 +1122,12 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 					continue
 				}
 				for _, v := range dedupVersions[i].Versions {
+					if err == errFileNotFound || err == errFileVersionNotFound {
+						if !dobjects[v.Idx].DeleteMarker {
+							// Not delete marker, if not found, ok.
+							continue
+						}
+					}
 					delObjErrs[index][v.Idx] = err
 				}
 			}
@@ -1222,9 +1153,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 			errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
 		}
 
-		if errs[objIndex] == nil {
-			defer NSUpdated(bucket, objects[objIndex].ObjectName)
-		}
+		defer NSUpdated(bucket, objects[objIndex].ObjectName)
 	}
 
 	// Check failed deletes across multiple objects
@@ -1566,6 +1495,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
 
 	fi.Metadata[xhttp.AmzObjectTagging] = tags
+	fi.ReplicationState = opts.PutReplicationState()
 	for k, v := range opts.UserDefined {
 		fi.Metadata[k] = v
 	}

cmd/erasure-server-pool-decom.go

@@ -0,0 +1,996 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net/http"
+	"sort"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/console"
+)
+
+// PoolDecommissionInfo currently decommissioning information
+type PoolDecommissionInfo struct {
+	StartTime   time.Time `json:"startTime" msg:"st"`
+	StartSize   int64     `json:"startSize" msg:"ss"`
+	TotalSize   int64     `json:"totalSize" msg:"ts"`
+	CurrentSize int64     `json:"currentSize" msg:"cs"`
+
+	Complete bool `json:"complete" msg:"cmp"`
+	Failed   bool `json:"failed" msg:"fl"`
+	Canceled bool `json:"canceled" msg:"cnl"`
+
+	// Internal information.
+	QueuedBuckets         []string `json:"-" msg:"bkts"`
+	DecommissionedBuckets []string `json:"-" msg:"dbkts"`
+
+	// Last bucket/object decommissioned.
+	Bucket string `json:"-" msg:"bkt"`
+	Object string `json:"-" msg:"obj"`
+
+	// Verbose information
+	ItemsDecommissioned     int64 `json:"-" msg:"id"`
+	ItemsDecommissionFailed int64 `json:"-" msg:"idf"`
+	BytesDone               int64 `json:"-" msg:"bd"`
+	BytesFailed             int64 `json:"-" msg:"bf"`
+}
+
+// bucketPop should be called when a bucket is done decommissioning.
+// Adds the bucket to the list of decommissioned buckets and updates resume numbers.
+func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
+	pd.DecommissionedBuckets = append(pd.DecommissionedBuckets, bucket)
+	for i, b := range pd.QueuedBuckets {
+		if b == bucket {
+			// Bucket is done.
+			pd.QueuedBuckets = append(pd.QueuedBuckets[:i], pd.QueuedBuckets[i+1:]...)
+			// Clear tracker info.
+			if pd.Bucket == bucket {
+				pd.Bucket = "" // empty this out for next bucket
+				pd.Object = "" // empty this out for next object
+			}
+			return
+		}
+	}
+}
+
+func (pd *PoolDecommissionInfo) bucketsToDecommission() []string {
+	queuedBuckets := make([]string, len(pd.QueuedBuckets))
+	copy(queuedBuckets, pd.QueuedBuckets)
+	return queuedBuckets
+}
+
+func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
+	for _, b := range pd.DecommissionedBuckets {
+		if b == bucket {
+			return true
+		}
+	}
+	return false
+}
+
+func (pd *PoolDecommissionInfo) bucketPush(bucket string) {
+	for _, b := range pd.QueuedBuckets {
+		if pd.isBucketDecommissioned(b) {
+			return
+		}
+		if b == bucket {
+			return
+		}
+	}
+	pd.QueuedBuckets = append(pd.QueuedBuckets, bucket)
+	pd.Bucket = bucket
+}
+
+// PoolStatus captures current pool status
+type PoolStatus struct {
+	ID           int                   `json:"id" msg:"id"`
+	CmdLine      string                `json:"cmdline" msg:"cl"`
+	LastUpdate   time.Time             `json:"lastUpdate" msg:"lu"`
+	Decommission *PoolDecommissionInfo `json:"decommissionInfo,omitempty" msg:"dec"`
+}
+
+//go:generate msgp -file $GOFILE -unexported
+type poolMeta struct {
+	Version int          `msg:"v"`
+	Pools   []PoolStatus `msg:"pls"`
+}
+
+// A decommission resumable tells us if decommission is worth
+// resuming upon restart of a cluster.
+func (p *poolMeta) returnResumablePools(n int) []PoolStatus {
+	var newPools []PoolStatus
+	for _, pool := range p.Pools {
+		if pool.Decommission == nil {
+			continue
+		}
+		if pool.Decommission.Complete || pool.Decommission.Canceled {
+			// Do not resume decommission upon startup for
+			// - decommission complete
+			// - decommission canceled
+			continue
+		} // In all other situations we need to resume
+		newPools = append(newPools, pool)
+		if n > 0 && len(newPools) == n {
+			return newPools
+		}
+	}
+	return nil
+}
+
+func (p *poolMeta) DecommissionComplete(idx int) bool {
+	if p.Pools[idx].Decommission != nil {
+		p.Pools[idx].LastUpdate = UTCNow()
+		p.Pools[idx].Decommission.Complete = true
+		p.Pools[idx].Decommission.Failed = false
+		p.Pools[idx].Decommission.Canceled = false
+		return true
+	}
+	return false
+}
+
+func (p *poolMeta) DecommissionFailed(idx int) bool {
+	if p.Pools[idx].Decommission != nil {
+		p.Pools[idx].LastUpdate = UTCNow()
+		p.Pools[idx].Decommission.StartTime = time.Time{}
+		p.Pools[idx].Decommission.Complete = false
+		p.Pools[idx].Decommission.Failed = true
+		p.Pools[idx].Decommission.Canceled = false
+		return true
+	}
+	return false
+}
+
+func (p *poolMeta) DecommissionCancel(idx int) bool {
+	if p.Pools[idx].Decommission != nil {
+		p.Pools[idx].LastUpdate = UTCNow()
+		p.Pools[idx].Decommission.StartTime = time.Time{}
+		p.Pools[idx].Decommission.Complete = false
+		p.Pools[idx].Decommission.Failed = false
+		p.Pools[idx].Decommission.Canceled = true
+		return true
+	}
+	return false
+}
+
+func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool {
+	return p.Pools[idx].Decommission.isBucketDecommissioned(bucket)
+}
+
+func (p *poolMeta) BucketDone(idx int, bucket string) {
+	if p.Pools[idx].Decommission == nil {
+		// Decommission not in progress.
+		return
+	}
+	p.Pools[idx].Decommission.bucketPop(bucket)
+}
+
+func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) {
+	if p.Pools[idx].Decommission != nil {
+		bucket = p.Pools[idx].Decommission.Bucket
+		object = p.Pools[idx].Decommission.Object
+	}
+	return
+}
+
+func (p *poolMeta) TrackCurrentBucketObject(idx int, bucket string, object string) {
+	if p.Pools[idx].Decommission == nil {
+		// Decommission not in progress.
+		return
+	}
+	p.Pools[idx].Decommission.Bucket = bucket
+	p.Pools[idx].Decommission.Object = object
+}
+
+func (p *poolMeta) PendingBuckets(idx int) []string {
+	if p.Pools[idx].Decommission == nil {
+		// Decommission not in progress.
+		return nil
+	}
+
+	return p.Pools[idx].Decommission.bucketsToDecommission()
+}
+
+func (p *poolMeta) QueueBuckets(idx int, buckets []BucketInfo) {
+	// add new queued buckets
+	for _, bucket := range buckets {
+		p.Pools[idx].Decommission.bucketPush(bucket.Name)
+	}
+}
+
+var (
+	errDecommissionAlreadyRunning = errors.New("decommission is already in progress")
+	errDecommissionComplete       = errors.New("decommission is complete, please remove the servers from command-line")
+)
+
+func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
+	for i, pool := range p.Pools {
+		if idx == i {
+			continue
+		}
+		if pool.Decommission != nil {
+			// Do not allow multiple decommissions at the same time.
+			// We shall for now only allow one pool decommission at
+			// a time.
+			return fmt.Errorf("%w at index: %d", errDecommissionAlreadyRunning, i)
+		}
+	}
+
+	now := UTCNow()
+	if p.Pools[idx].Decommission == nil {
+		p.Pools[idx].LastUpdate = now
+		p.Pools[idx].Decommission = &PoolDecommissionInfo{
+			StartTime:   now,
+			StartSize:   pi.Free,
+			CurrentSize: pi.Free,
+			TotalSize:   pi.Total,
+		}
+		return nil
+	}
+
+	// Completed pool doesn't need to be decommissioned again.
+	if p.Pools[idx].Decommission.Complete {
+		return errDecommissionComplete
+	}
+
+	// Canceled or Failed decommission can be triggered again.
+	if p.Pools[idx].Decommission.StartTime.IsZero() {
+		if p.Pools[idx].Decommission.Canceled || p.Pools[idx].Decommission.Failed {
+			p.Pools[idx].LastUpdate = now
+			p.Pools[idx].Decommission = &PoolDecommissionInfo{
+				StartTime:   now,
+				StartSize:   pi.Free,
+				CurrentSize: pi.Free,
+				TotalSize:   pi.Total,
+			}
+			return nil
+		}
+	} // In-progress pool doesn't need to be decommissioned again.
+
+	// In all other scenarios an active decommissioning is in progress.
+	return errDecommissionAlreadyRunning
+}
+
+func (p poolMeta) IsSuspended(idx int) bool {
+	return p.Pools[idx].Decommission != nil
+}
+
+func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
+	type poolInfo struct {
+		position     int
+		completed    bool
+		decomStarted bool // started but not finished yet
+	}
+
+	rememberedPools := make(map[string]poolInfo)
+	for idx, pool := range p.Pools {
+		complete := false
+		decomStarted := false
+		if pool.Decommission != nil {
+			if pool.Decommission.Complete {
+				complete = true
+			}
+			decomStarted = true
+		}
+		rememberedPools[pool.CmdLine] = poolInfo{
+			position:     idx,
+			completed:    complete,
+			decomStarted: decomStarted,
+		}
+	}
+
+	specifiedPools := make(map[string]int)
+	for idx, pool := range pools {
+		specifiedPools[pool.endpoints.CmdLine] = idx
+	}
+
+	// Check if specified pools need to remove decommissioned pool.
+	for k := range specifiedPools {
+		pi, ok := rememberedPools[k]
+		if ok && pi.completed {
+			return false, fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k)
+		}
+	}
+
+	// check if remembered pools are in right position or missing from command line.
+	for k, pi := range rememberedPools {
+		if pi.completed {
+			continue
+		}
+		_, ok := specifiedPools[k]
+		if !ok {
+			return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
+		}
+	}
+
+	// check when remembered pools and specified pools are same they are at the expected position
+	if len(rememberedPools) == len(specifiedPools) {
+		for k, pi := range rememberedPools {
+			pos, ok := specifiedPools[k]
+			if !ok {
+				return false, fmt.Errorf("pool(%s) = %s is not specified, please specify on server command line", humanize.Ordinal(pi.position+1), k)
+			}
+			if pos != pi.position {
+				return false, fmt.Errorf("pool order change detected for %s, expected position is (%s) but found (%s)", k, humanize.Ordinal(pi.position+1), humanize.Ordinal(pos+1))
+			}
+		}
+	}
+
+	update := len(rememberedPools) != len(specifiedPools)
+	if update {
+		for k, pi := range rememberedPools {
+			if pi.decomStarted && !pi.completed {
+				return false, fmt.Errorf("pool(%s) = %s is being decommissioned, No changes should be made to the command line arguments. Please complete the decommission in progress", humanize.Ordinal(pi.position+1), k)
+			}
+		}
+	}
+	return update, nil
+}
+
+func (p *poolMeta) load(ctx context.Context, pool *erasureSets, pools []*erasureSets) error {
+	data, err := readConfig(ctx, pool, poolMetaName)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	if len(data) == 0 {
+		// Seems to be empty create a new poolMeta object.
+		return nil
+	}
+	if len(data) <= 4 {
+		return fmt.Errorf("poolMeta: no data")
+	}
+	// Read header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case poolMetaFormat:
+	default:
+		return fmt.Errorf("poolMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case poolMetaVersion:
+	default:
+		return fmt.Errorf("poolMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+
+	// OK, parse data.
+	if _, err = p.UnmarshalMsg(data[4:]); err != nil {
+		return err
+	}
+
+	switch p.Version {
+	case poolMetaVersionV1:
+	default:
+		return fmt.Errorf("unexpected pool meta version: %d", p.Version)
+	}
+
+	return nil
+}
+
+func (p *poolMeta) CountItem(idx int, size int64, failed bool) {
+	pd := p.Pools[idx].Decommission
+	if pd != nil {
+		if failed {
+			pd.ItemsDecommissionFailed++
+			pd.BytesFailed += size
+		} else {
+			pd.ItemsDecommissioned++
+			pd.BytesDone += size
+		}
+		p.Pools[idx].Decommission = pd
+	}
+}
+
+func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSets, duration time.Duration) (bool, error) {
+	if p.Pools[idx].Decommission == nil {
+		return false, errInvalidArgument
+	}
+	now := UTCNow()
+	if now.Sub(p.Pools[idx].LastUpdate) >= duration {
+		if serverDebugLog {
+			console.Debugf("decommission: persisting poolMeta on disk: threshold:%s, poolMeta:%#v\n", now.Sub(p.Pools[idx].LastUpdate), p.Pools[idx])
+		}
+		p.Pools[idx].LastUpdate = now
+		if err := p.save(ctx, pools); err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
+	data := make([]byte, 4, p.Msgsize()+4)
+
+	// Initialize the header.
+	binary.LittleEndian.PutUint16(data[0:2], poolMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], poolMetaVersion)
+
+	buf, err := p.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	// Saves on all pools to make sure decommissioning of first pool is allowed.
+	for _, eset := range pools {
+		if err = saveConfig(ctx, eset, poolMetaName, buf); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+const (
+	poolMetaName      = "pool.bin"
+	poolMetaFormat    = 1
+	poolMetaVersionV1 = 1
+	poolMetaVersion   = poolMetaVersionV1
+)
+
+// Init() initializes pools and saves additional information about them
+// in 'pool.bin', this is eventually used for decommissioning the pool.
+func (z *erasureServerPools) Init(ctx context.Context) error {
+	meta := poolMeta{}
+
+	if err := meta.load(ctx, z.serverPools[0], z.serverPools); err != nil {
+		return err
+	}
+
+	update, err := meta.validate(z.serverPools)
+	if err != nil {
+		return err
+	}
+
+	// if no update is needed return right away.
+	if !update {
+		// We are only supporting single pool decommission at this time
+		// so it makes sense to only resume single pools at any given
+		// time, in future meta.returnResumablePools() might take
+		// '-1' as argument to decommission multiple pools at a time
+		// but this is not a priority at the moment.
+		for _, pool := range meta.returnResumablePools(1) {
+			err := z.Decommission(ctx, pool.ID)
+			switch err {
+			case errDecommissionAlreadyRunning:
+				fallthrough
+			case nil:
+				go z.doDecommissionInRoutine(ctx, pool.ID)
+			}
+		}
+		z.poolMeta = meta
+
+		return nil
+	}
+
+	meta = poolMeta{} // to update write poolMeta fresh.
+	// looks like new pool was added we need to update,
+	// or this is a fresh installation (or an existing
+	// installation with pool removed)
+	meta.Version = poolMetaVersion
+	for idx, pool := range z.serverPools {
+		meta.Pools = append(meta.Pools, PoolStatus{
+			CmdLine:    pool.endpoints.CmdLine,
+			ID:         idx,
+			LastUpdate: UTCNow(),
+		})
+	}
+	if err = meta.save(ctx, z.serverPools); err != nil {
+		return err
+	}
+	z.poolMeta = meta
+	return nil
+}
+
+func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
+	defer gr.Close()
+	objInfo := gr.ObjInfo
+	if objInfo.isMultipart() {
+		uploadID, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
+			VersionID:   objInfo.VersionID,
+			MTime:       objInfo.ModTime,
+			UserDefined: objInfo.UserDefined,
+		})
+		if err != nil {
+			return err
+		}
+		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, uploadID, ObjectOptions{})
+		parts := make([]CompletePart, len(objInfo.Parts))
+		for i, part := range objInfo.Parts {
+			hr, err := hash.NewReader(gr, part.Size, "", "", part.Size)
+			if err != nil {
+				return err
+			}
+			pi, err := z.PutObjectPart(ctx, bucket, objInfo.Name, uploadID,
+				part.Number,
+				NewPutObjReader(hr),
+				ObjectOptions{})
+			if err != nil {
+				return err
+			}
+			parts[i] = CompletePart{
+				ETag:       pi.ETag,
+				PartNumber: pi.PartNumber,
+			}
+		}
+		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, uploadID, parts, ObjectOptions{
+			MTime: objInfo.ModTime,
+		})
+		return err
+	}
+	hr, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size)
+	if err != nil {
+		return err
+	}
+	_, err = z.PutObject(ctx,
+		bucket,
+		objInfo.Name,
+		NewPutObjReader(hr),
+		ObjectOptions{
+			VersionID:   objInfo.VersionID,
+			MTime:       objInfo.ModTime,
+			UserDefined: objInfo.UserDefined,
+		})
+	return err
+}
+
+// versionsSorter sorts FileInfo slices by version.
+//msgp:ignore versionsSorter
+type versionsSorter []FileInfo
+
+func (v versionsSorter) reverse() {
+	sort.Slice(v, func(i, j int) bool {
+		return v[i].ModTime.Before(v[j].ModTime)
+	})
+}
+
+func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bName string) error {
+	var forwardTo string
+	// If we resume to the same bucket, forward to last known item.
+	rbucket, robject := z.poolMeta.ResumeBucketObject(idx)
+	if rbucket != "" && rbucket == bName {
+		forwardTo = robject
+	}
+
+	versioned := globalBucketVersioningSys.Enabled(bName)
+	for _, set := range pool.sets {
+		disks := set.getOnlineDisks()
+		if len(disks) == 0 {
+			logger.LogIf(GlobalContext, fmt.Errorf("no online disks found for set with endpoints %s",
+				set.getEndpoints()))
+			continue
+		}
+
+		decommissionEntry := func(entry metaCacheEntry) {
+			if entry.isDir() {
+				return
+			}
+
+			fivs, err := entry.fileInfoVersions(bName)
+			if err != nil {
+				return
+			}
+
+			// We need a reversed order for Decommissioning,
+			// to create the appropriate stack.
+			versionsSorter(fivs.Versions).reverse()
+
+			for _, version := range fivs.Versions {
+				// TODO: Skip transitioned objects for now.
+				if version.IsRemote() {
+					continue
+				}
+				// We will skip decommissioning delete markers
+				// with single version, its as good as there
+				// is no data associated with the object.
+				if version.Deleted && len(fivs.Versions) == 1 {
+					continue
+				}
+				if version.Deleted {
+					_, err := z.DeleteObject(ctx,
+						bName,
+						version.Name,
+						ObjectOptions{
+							Versioned:         versioned,
+							VersionID:         version.VersionID,
+							MTime:             version.ModTime,
+							DeleteReplication: version.ReplicationState,
+						})
+					if err != nil {
+						logger.LogIf(ctx, err)
+						z.poolMetaMutex.Lock()
+						z.poolMeta.CountItem(idx, 0, true)
+						z.poolMetaMutex.Unlock()
+					} else {
+						set.DeleteObject(ctx,
+							bName,
+							version.Name,
+							ObjectOptions{
+								VersionID: version.VersionID,
+							})
+						z.poolMetaMutex.Lock()
+						z.poolMeta.CountItem(idx, 0, false)
+						z.poolMetaMutex.Unlock()
+					}
+					continue
+				}
+				gr, err := set.GetObjectNInfo(ctx,
+					bName,
+					version.Name,
+					nil,
+					http.Header{},
+					noLock, // all mutations are blocked reads are safe without locks.
+					ObjectOptions{
+						VersionID: version.VersionID,
+					})
+				if err != nil {
+					logger.LogIf(ctx, err)
+					z.poolMetaMutex.Lock()
+					z.poolMeta.CountItem(idx, version.Size, true)
+					z.poolMetaMutex.Unlock()
+					continue
+				}
+				// gr.Close() is ensured by decommissionObject().
+				if err = z.decommissionObject(ctx, bName, gr); err != nil {
+					logger.LogIf(ctx, err)
+					z.poolMetaMutex.Lock()
+					z.poolMeta.CountItem(idx, version.Size, true)
+					z.poolMetaMutex.Unlock()
+					continue
+				}
+				set.DeleteObject(ctx,
+					bName,
+					version.Name,
+					ObjectOptions{
+						VersionID: version.VersionID,
+					})
+				z.poolMetaMutex.Lock()
+				z.poolMeta.CountItem(idx, version.Size, false)
+				z.poolMetaMutex.Unlock()
+			}
+			z.poolMetaMutex.Lock()
+			z.poolMeta.TrackCurrentBucketObject(idx, bName, entry.name)
+			ok, err := z.poolMeta.updateAfter(ctx, idx, z.serverPools, 30*time.Second)
+			logger.LogIf(ctx, err)
+			if ok {
+				globalNotificationSys.ReloadPoolMeta(ctx)
+			}
+			z.poolMetaMutex.Unlock()
+		}
+
+		// How to resolve partial results.
+		resolver := metadataResolutionParams{
+			dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+			objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+			bucket:    bName,
+		}
+
+		if err := listPathRaw(ctx, listPathRawOptions{
+			disks:          disks,
+			bucket:         bName,
+			recursive:      true,
+			forwardTo:      forwardTo,
+			minDisks:       len(disks) / 2, // to capture all quorum ratios
+			reportNotFound: false,
+			agreed:         decommissionEntry,
+			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+				entry, ok := entries.resolve(&resolver)
+				if ok {
+					decommissionEntry(*entry)
+				}
+			},
+			finished: nil,
+		}); err != nil {
+			// Decommissioning failed and won't continue
+			return err
+		}
+	}
+	return nil
+}
+
+func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
+	pool := z.serverPools[idx]
+	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
+		if z.poolMeta.isBucketDecommissioned(idx, bucket) {
+			if serverDebugLog {
+				console.Debugln("decommission: already done, moving on", bucket)
+			}
+
+			z.poolMetaMutex.Lock()
+			z.poolMeta.BucketDone(idx, bucket) // remove from pendingBuckets and persist.
+			z.poolMeta.save(ctx, z.serverPools)
+			z.poolMetaMutex.Unlock()
+			continue
+		}
+		if serverDebugLog {
+			console.Debugln("decommission: currently on bucket", bucket)
+		}
+		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
+			return err
+		}
+		z.poolMetaMutex.Lock()
+		z.poolMeta.BucketDone(idx, bucket)
+		z.poolMeta.save(ctx, z.serverPools)
+		z.poolMetaMutex.Unlock()
+	}
+	return nil
+}
+
+func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx int) {
+	z.poolMetaMutex.Lock()
+	var dctx context.Context
+	dctx, z.decommissionCancelers[idx] = context.WithCancel(GlobalContext)
+	z.poolMetaMutex.Unlock()
+
+	if err := z.decommissionInBackground(dctx, idx); err != nil {
+		logger.LogIf(GlobalContext, err)
+		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
+		return
+	}
+	// Complete the decommission..
+	logger.LogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
+}
+
+func (z *erasureServerPools) IsSuspended(idx int) bool {
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+	return z.poolMeta.IsSuspended(idx)
+}
+
+// Decommission - start decommission session.
+func (z *erasureServerPools) Decommission(ctx context.Context, idx int) error {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	// Make pool unwritable before decommissioning.
+	if err := z.StartDecommission(ctx, idx); err != nil {
+		return err
+	}
+
+	go z.doDecommissionInRoutine(ctx, idx)
+
+	// Successfully started decommissioning.
+	return nil
+}
+
+type decomError struct {
+	Err string
+}
+
+func (d decomError) Error() string {
+	return d.Err
+}
+
+type poolSpaceInfo struct {
+	Free  int64
+	Total int64
+	Used  int64
+}
+
+func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpaceInfo, err error) {
+	if idx < 0 {
+		return pi, errInvalidArgument
+	}
+	if idx+1 > len(z.serverPools) {
+		return pi, errInvalidArgument
+	}
+	info, errs := z.serverPools[idx].StorageInfo(context.Background())
+	for _, err := range errs {
+		if err != nil {
+			return pi, errInvalidArgument
+		}
+	}
+	info.Backend = z.BackendInfo()
+	for _, disk := range info.Disks {
+		if disk.Healing {
+			return pi, decomError{
+				Err: fmt.Sprintf("%s drive is healing, decommission will not be started", disk.Endpoint),
+			}
+		}
+	}
+
+	usableTotal := int64(GetTotalUsableCapacity(info.Disks, info))
+	usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info))
+	return poolSpaceInfo{
+		Total: usableTotal,
+		Free:  usableFree,
+		Used:  usableTotal - usableFree,
+	}, nil
+}
+
+func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, error) {
+	if idx < 0 {
+		return PoolStatus{}, errInvalidArgument
+	}
+
+	z.poolMetaMutex.RLock()
+	defer z.poolMetaMutex.RUnlock()
+
+	pi, err := z.getDecommissionPoolSpaceInfo(idx)
+	if err != nil {
+		return PoolStatus{}, errInvalidArgument
+	}
+
+	poolInfo := z.poolMeta.Pools[idx]
+	if poolInfo.Decommission != nil {
+		poolInfo.Decommission.TotalSize = pi.Total
+		poolInfo.Decommission.CurrentSize = poolInfo.Decommission.StartSize + poolInfo.Decommission.BytesDone
+	} else {
+		poolInfo.Decommission = &PoolDecommissionInfo{
+			TotalSize:   pi.Total,
+			CurrentSize: pi.Free,
+		}
+	}
+	return poolInfo, nil
+}
+
+func (z *erasureServerPools) ReloadPoolMeta(ctx context.Context) (err error) {
+	meta := poolMeta{}
+
+	if err = meta.load(ctx, z.serverPools[0], z.serverPools); err != nil {
+		return err
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	z.poolMeta = meta
+	return nil
+}
+
+func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if z.poolMeta.DecommissionCancel(idx) {
+		z.decommissionCancelers[idx]() // cancel any active thread.
+		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+			return err
+		}
+		globalNotificationSys.ReloadPoolMeta(ctx)
+	}
+	return nil
+}
+
+func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if z.poolMeta.DecommissionFailed(idx) {
+		z.decommissionCancelers[idx]() // cancel any active thread.
+		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+			return err
+		}
+		globalNotificationSys.ReloadPoolMeta(ctx)
+	}
+	return nil
+}
+
+func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if z.poolMeta.DecommissionComplete(idx) {
+		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+			return err
+		}
+		globalNotificationSys.ReloadPoolMeta(ctx)
+	}
+	return nil
+}
+
+func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (err error) {
+	if idx < 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	buckets, err := z.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+
+	// TODO: Support decommissioning transition tiers.
+	for _, bucket := range buckets {
+		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
+			if lc.HasTransition() {
+				return decomError{
+					Err: fmt.Sprintf("Bucket is part of transitioned tier %s: decommission is not allowed in Tier'd setups", bucket.Name),
+				}
+			}
+		}
+	}
+
+	// Buckets data are dispersed in multiple zones/sets, make
+	// sure to decommission the necessary metadata.
+	buckets = append(buckets, BucketInfo{
+		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+	})
+	buckets = append(buckets, BucketInfo{
+		Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+	})
+
+	var pool *erasureSets
+	for pidx := range z.serverPools {
+		if pidx == idx {
+			pool = z.serverPools[idx]
+			break
+		}
+	}
+
+	if pool == nil {
+		return errInvalidArgument
+	}
+
+	pi, err := z.getDecommissionPoolSpaceInfo(idx)
+	if err != nil {
+		return err
+	}
+
+	z.poolMetaMutex.Lock()
+	defer z.poolMetaMutex.Unlock()
+
+	if err = z.poolMeta.Decommission(idx, pi); err != nil {
+		return err
+	}
+	z.poolMeta.QueueBuckets(idx, buckets)
+	if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
+		return err
+	}
+	globalNotificationSys.ReloadPoolMeta(ctx)
+	return nil
+}

cmd/erasure-server-pool-decom_gen_test.go

@@ -0,0 +1,575 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalPoolDecommissionInfo(t *testing.T) {
+	v := PoolDecommissionInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgPoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgPoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalPoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodePoolDecommissionInfo(t *testing.T) {
+	v := PoolDecommissionInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodePoolDecommissionInfo Msgsize() is inaccurate")
+	}
+
+	vn := PoolDecommissionInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodePoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodePoolDecommissionInfo(b *testing.B) {
+	v := PoolDecommissionInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalPoolStatus(t *testing.T) {
+	v := PoolStatus{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgPoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgPoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalPoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodePoolStatus(t *testing.T) {
+	v := PoolStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodePoolStatus Msgsize() is inaccurate")
+	}
+
+	vn := PoolStatus{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodePoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodePoolStatus(b *testing.B) {
+	v := PoolStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshaldecomError(t *testing.T) {
+	v := decomError{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdecomError(b *testing.B) {
+	v := decomError{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdecomError(b *testing.B) {
+	v := decomError{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldecomError(b *testing.B) {
+	v := decomError{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedecomError(t *testing.T) {
+	v := decomError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedecomError Msgsize() is inaccurate")
+	}
+
+	vn := decomError{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedecomError(b *testing.B) {
+	v := decomError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedecomError(b *testing.B) {
+	v := decomError{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalpoolMeta(t *testing.T) {
+	v := poolMeta{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgpoolMeta(b *testing.B) {
+	v := poolMeta{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgpoolMeta(b *testing.B) {
+	v := poolMeta{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalpoolMeta(b *testing.B) {
+	v := poolMeta{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodepoolMeta(t *testing.T) {
+	v := poolMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodepoolMeta Msgsize() is inaccurate")
+	}
+
+	vn := poolMeta{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodepoolMeta(b *testing.B) {
+	v := poolMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodepoolMeta(b *testing.B) {
+	v := poolMeta{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalpoolSpaceInfo(t *testing.T) {
+	v := poolSpaceInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgpoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgpoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalpoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodepoolSpaceInfo(t *testing.T) {
+	v := poolSpaceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodepoolSpaceInfo Msgsize() is inaccurate")
+	}
+
+	vn := poolSpaceInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodepoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodepoolSpaceInfo(b *testing.B) {
+	v := poolSpaceInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/erasure-server-pool-decom_test.go

@@ -0,0 +1,196 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"testing"
+)
+
+func prepareErasurePools() (ObjectLayer, []string, error) {
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, err := newErasureServerPools(context.Background(), pools)
+	if err != nil {
+		return nil, nil, err
+	}
+	return objLayer, fsDirs, nil
+}
+
+func TestPoolMetaValidate(t *testing.T) {
+	objLayer1, fsDirs, err := prepareErasurePools()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeRoots(fsDirs)
+
+	meta := objLayer1.(*erasureServerPools).poolMeta
+	pools := objLayer1.(*erasureServerPools).serverPools
+
+	objLayer2, fsDirs, err := prepareErasurePools()
+	if err != nil {
+		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
+	}
+	defer removeRoots(fsDirs)
+
+	newPools := objLayer2.(*erasureServerPools).serverPools
+	reducedPools := pools[1:]
+	orderChangePools := []*erasureSets{
+		pools[1],
+		pools[0],
+	}
+
+	var nmeta1 poolMeta
+	nmeta1.Version = poolMetaVersion
+	nmeta1.Pools = append(nmeta1.Pools, meta.Pools...)
+	for i, pool := range nmeta1.Pools {
+		if i == 0 {
+			nmeta1.Pools[i] = PoolStatus{
+				CmdLine:    pool.CmdLine,
+				ID:         i,
+				LastUpdate: UTCNow(),
+				Decommission: &PoolDecommissionInfo{
+					Complete: true,
+				},
+			}
+		}
+	}
+
+	var nmeta2 poolMeta
+	nmeta2.Version = poolMetaVersion
+	nmeta2.Pools = append(nmeta2.Pools, meta.Pools...)
+	for i, pool := range nmeta2.Pools {
+		if i == 0 {
+			nmeta2.Pools[i] = PoolStatus{
+				CmdLine:    pool.CmdLine,
+				ID:         i,
+				LastUpdate: UTCNow(),
+				Decommission: &PoolDecommissionInfo{
+					Complete: false,
+				},
+			}
+		}
+	}
+
+	testCases := []struct {
+		meta           poolMeta
+		pools          []*erasureSets
+		expectedUpdate bool
+		expectedErr    bool
+		name           string
+	}{
+		{
+			meta:           meta,
+			pools:          pools,
+			name:           "Correct",
+			expectedErr:    false,
+			expectedUpdate: false,
+		},
+		{
+			meta:           meta,
+			pools:          newPools,
+			name:           "Invalid-Commandline",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           meta,
+			pools:          reducedPools,
+			name:           "Invalid-Reduced",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           meta,
+			pools:          orderChangePools,
+			name:           "Invalid-Orderchange",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta1,
+			pools:          pools,
+			name:           "Invalid-Completed-Pool-Not-Removed",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta2,
+			pools:          pools,
+			name:           "Correct-Decom-Pending",
+			expectedErr:    false,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta2,
+			pools:          reducedPools,
+			name:           "Invalid-Decom-Pending-Pool-Removal",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+		{
+			meta:           nmeta1,
+			pools:          reducedPools,
+			name:           "Correct-Decom-Pool-Removed",
+			expectedErr:    false,
+			expectedUpdate: true,
+		},
+		{
+			meta:           poolMeta{}, // no-pool info available fresh setup.
+			pools:          pools,
+			name:           "Correct-Fresh-Setup",
+			expectedErr:    false,
+			expectedUpdate: true,
+		},
+		{
+			meta:           nmeta2,
+			pools:          orderChangePools,
+			name:           "Invalid-Orderchange-Decom",
+			expectedErr:    true,
+			expectedUpdate: false,
+		},
+	}
+
+	t.Parallel()
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run(testCase.name, func(t *testing.T) {
+			update, err := testCase.meta.validate(testCase.pools)
+			if testCase.expectedErr {
+				t.Log(err)
+			}
+			if err != nil && !testCase.expectedErr {
+				t.Errorf("Expected success, but found %s", err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error("Expected error, but got `nil`")
+			}
+			if update != testCase.expectedUpdate {
+				t.Errorf("Expected %t, got %t", testCase.expectedUpdate, update)
+			}
+		})
+	}
+}

cmd/erasure-server-pool.go

@@ -42,10 +42,15 @@ import (
 type erasureServerPools struct {
 	GatewayUnsupported
 
-	serverPools []*erasureSets
+	poolMetaMutex sync.RWMutex
+	poolMeta      poolMeta
+	serverPools   []*erasureSets
 
 	// Shut down async operations
 	shutdown context.CancelFunc
+
+	// Active decommission canceler
+	decommissionCancelers []context.CancelFunc
 }
 
 func (z *erasureServerPools) SinglePool() bool {
@@ -62,19 +67,14 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 		formats      = make([]*formatErasureV3, len(endpointServerPools))
 		storageDisks = make([][]StorageAPI, len(endpointServerPools))
-		z            = &erasureServerPools{serverPools: make([]*erasureSets, len(endpointServerPools))}
+		z            = &erasureServerPools{
+			serverPools: make([]*erasureSets, len(endpointServerPools)),
+		}
 	)
 
-	var localDrives []string
-
+	var localDrives []StorageAPI
 	local := endpointServerPools.FirstLocal()
 	for i, ep := range endpointServerPools {
-		for _, endpoint := range ep.Endpoints {
-			if endpoint.IsLocal {
-				localDrives = append(localDrives, endpoint.Path)
-			}
-		}
-
 		// If storage class is not set during startup, default values are used
 		// -- Default for Reduced Redundancy Storage class is, parity = 2
 		// -- Default for Standard Storage class is, parity = 2 - disks 4, 5
@@ -94,6 +94,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			return nil, err
 		}
 
+		for _, storageDisk := range storageDisks[i] {
+			if storageDisk != nil && storageDisk.IsLocal() {
+				localDrives = append(localDrives, storageDisk)
+			}
+		}
+
 		if deploymentID == "" {
 			// all zones should have same deployment ID
 			deploymentID = formats[i].ID
@@ -108,13 +114,34 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			return nil, fmt.Errorf("All serverPools should have same deployment ID expected %s, got %s", deploymentID, formats[i].ID)
 		}
 
-		z.serverPools[i], err = newErasureSets(ctx, ep.Endpoints, storageDisks[i], formats[i], commonParityDrives, i)
+		z.serverPools[i], err = newErasureSets(ctx, ep, storageDisks[i], formats[i], commonParityDrives, i)
 		if err != nil {
 			return nil, err
 		}
 	}
+
+	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for {
+		err := z.Init(ctx) // Initializes all pools.
+		if err != nil {
+			if !configRetriableErrors(err) {
+				logger.Fatal(err, "Unable to initialize backend")
+			}
+			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
+			continue
+		}
+		break
+	}
+
+	drives := make([]string, 0, len(localDrives))
+	for _, localDrive := range localDrives {
+		drives = append(drives, localDrive.Endpoint().Path)
+	}
+
+	globalLocalDrives = localDrives
 	ctx, z.shutdown = context.WithCancel(ctx)
-	go intDataUpdateTracker.start(ctx, localDrives...)
+	go intDataUpdateTracker.start(ctx, drives...)
 	return z, nil
 }
 
@@ -155,7 +182,7 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 	found := 0
 	for _, s := range z.serverPools {
 		for _, disks := range s.erasureDisks {
-			for i, disk := range disks {
+			for _, disk := range disks {
 				if disk == OfflineDisk {
 					continue
 				}
@@ -163,10 +190,6 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 				if err != nil {
 					continue
 				}
-				did, err := disk.GetDiskID()
-				if err != nil {
-					did = fmt.Sprintf("disk-%d", i)
-				}
 				for _, si := range stats {
 					found++
 					var r io.ReadCloser
@@ -178,7 +201,9 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 					} else {
 						r = io.NopCloser(bytes.NewBuffer([]byte{}))
 					}
-					err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si)
+					// Keep disk path instead of ID, to ensure that the downloaded zip file can be
+					// easily automated with `minio server hostname{1...n}/disk{1...m}`.
+					err = fn(r, disk.Hostname(), disk.Endpoint().Path, pathJoin(volume, si.Name), si)
 					r.Close()
 					if err != nil {
 						return err
@@ -249,6 +274,10 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 	g := errgroup.WithNErrs(len(z.serverPools))
 	for index := range z.serverPools {
 		index := index
+		// skip suspended pools for any new I/O.
+		if z.IsSuspended(index) {
+			continue
+		}
 		g.Go(func() error {
 			// Get the set where it would be placed.
 			storageInfos[index] = getDiskInfos(ctx, z.serverPools[index].getHashedSet(object).getDisks())
@@ -292,19 +321,24 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 	}
 
 	poolObjInfos := make([]poolObjInfo, len(z.serverPools))
+	poolOpts := make([]ObjectOptions, len(z.serverPools))
+	for i := range z.serverPools {
+		poolOpts[i] = opts
+	}
 
 	var wg sync.WaitGroup
 	for i, pool := range z.serverPools {
 		wg.Add(1)
-		go func(i int, pool *erasureSets) {
+		go func(i int, pool *erasureSets, opts ObjectOptions) {
 			defer wg.Done()
 			// remember the pool index, we may sort the slice original index might be lost.
 			pinfo := poolObjInfo{
 				PoolIndex: i,
 			}
+			opts.VersionID = "" // no need to check for specific versionId
 			pinfo.ObjInfo, pinfo.Err = pool.GetObjectInfo(ctx, bucket, object, opts)
 			poolObjInfos[i] = pinfo
-		}(i, pool)
+		}(i, pool, poolOpts[i])
 	}
 	wg.Wait()
 
@@ -322,6 +356,12 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
 			return -1, pinfo.Err
 		}
+
+		// skip all objects from suspended pools for mutating calls.
+		if z.IsSuspended(pinfo.PoolIndex) && opts.Mutate {
+			continue
+		}
+
 		if isErrObjectNotFound(pinfo.Err) {
 			// No object exists or its a delete marker,
 			// check objInfo to confirm.
@@ -333,23 +373,23 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 			// exist proceed to next pool.
 			continue
 		}
+
 		return pinfo.PoolIndex, nil
 	}
 
 	return -1, toObjectErr(errFileNotFound, bucket, object)
 }
 
-func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
-	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{NoLock: true})
-}
-
-// getPoolIdxExisting returns the (first) found object pool index containing an object.
+// getPoolIdxExistingNoLock returns the (first) found object pool index containing an object.
 // If the object exists, but the latest version is a delete marker, the index with it is still returned.
 // If the object does not exist ObjectNotFound error is returned.
 // If any other error is found, it is returned.
 // The check is skipped if there is only one zone, and 0, nil is always returned in that case.
-func (z *erasureServerPools) getPoolIdxExisting(ctx context.Context, bucket, object string) (idx int, err error) {
-	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{})
+func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
+	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{
+		NoLock: true,
+		Mutate: true,
+	})
 }
 
 func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
@@ -369,9 +409,10 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
 }
 
 // getPoolIdx returns the found previous object and its corresponding pool idx,
-// if none are found falls back to most available space pool.
+// if none are found falls back to most available space pool, this function is
+// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
 func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
-	idx, err = z.getPoolIdxExisting(ctx, bucket, object)
+	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{Mutate: true})
 	if err != nil && !isErrObjectNotFound(err) {
 		return idx, err
 	}
@@ -847,7 +888,11 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 }
 
 func (z *erasureServerPools) deletePrefix(ctx context.Context, bucket string, prefix string) error {
-	for _, zone := range z.serverPools {
+	for idx, zone := range z.serverPools {
+		if z.IsSuspended(idx) {
+			logger.LogIf(ctx, fmt.Errorf("pool %d is suspended, all writes are suspended", idx+1))
+			continue
+		}
 		_, err := zone.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
 		if err != nil {
 			return err
@@ -871,7 +916,8 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return objInfo, err
 	}
@@ -1785,7 +1831,10 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		defer close(errCh)
 
 		var wg sync.WaitGroup
-		for _, erasureSet := range z.serverPools {
+		for idx, erasureSet := range z.serverPools {
+			if z.IsSuspended(idx) {
+				continue
+			}
 			for _, set := range erasureSet.sets {
 				wg.Add(1)
 				go func(set *erasureObjects) {
@@ -1807,6 +1856,9 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 	results := make([]madmin.HealResultItem, len(z.serverPools))
 	var wg sync.WaitGroup
 	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		wg.Add(1)
 		go func(idx int, pool *erasureSets) {
 			defer wg.Done()
@@ -2006,7 +2058,8 @@ func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, obje
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -2022,7 +2075,8 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -2037,7 +2091,8 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -2052,7 +2107,8 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = false
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -2067,7 +2123,8 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 		return z.serverPools[0].TransitionObject(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
 	}
@@ -2082,7 +2139,8 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExisting(ctx, bucket, object)
+	opts.Mutate = true
+	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
 	}

cmd/erasure-sets.go

@@ -71,7 +71,7 @@ type erasureSets struct {
 	erasureLockOwner string
 
 	// List of endpoints provided on the command line.
-	endpoints Endpoints
+	endpoints PoolEndpoints
 
 	// String version of all the endpoints, an optimization
 	// to avoid url.String() conversion taking CPU on
@@ -106,7 +106,7 @@ func isEndpointConnectionStable(diskMap map[Endpoint]StorageAPI, endpoint Endpoi
 	if !disk.IsOnline() {
 		return false
 	}
-	if disk.LastConn().After(lastCheck) {
+	if !lastCheck.IsZero() && disk.LastConn().After(lastCheck) {
 		return false
 	}
 	return true
@@ -160,6 +160,9 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 //   - i'th position is the set index
 //   - j'th position is the disk index in the current set
 func findDiskIndexByDiskID(refFormat *formatErasureV3, diskID string) (int, int, error) {
+	if diskID == "" {
+		return -1, -1, errDiskNotFound
+	}
 	if diskID == offlineDiskUUID {
 		return -1, -1, fmt.Errorf("diskID: %s is offline", diskID)
 	}
@@ -208,7 +211,7 @@ func (s *erasureSets) connectDisks() {
 	var wg sync.WaitGroup
 	setsJustConnected := make([]bool, s.setCount)
 	diskMap := s.getDiskMap()
-	for _, endpoint := range s.endpoints {
+	for _, endpoint := range s.endpoints.Endpoints {
 		if isEndpointConnectionStable(diskMap, endpoint, s.lastConnectDisksOpTime) {
 			continue
 		}
@@ -328,7 +331,7 @@ func (s *erasureSets) GetEndpoints(setIndex int) func() []Endpoint {
 
 		eps := make([]Endpoint, s.setDriveCount)
 		for i := 0; i < s.setDriveCount; i++ {
-			eps[i] = s.endpoints[setIndex*s.setDriveCount+i]
+			eps[i] = s.endpoints.Endpoints[setIndex*s.setDriveCount+i]
 		}
 		return eps
 	}
@@ -350,12 +353,12 @@ func (s *erasureSets) GetDisks(setIndex int) func() []StorageAPI {
 const defaultMonitorConnectEndpointInterval = defaultMonitorNewDiskInterval + time.Second*5
 
 // Initialize new set of erasure coded sets.
-func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageAPI, format *formatErasureV3, defaultParityCount, poolIdx int) (*erasureSets, error) {
+func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks []StorageAPI, format *formatErasureV3, defaultParityCount, poolIdx int) (*erasureSets, error) {
 	setCount := len(format.Erasure.Sets)
 	setDriveCount := len(format.Erasure.Sets[0])
 
-	endpointStrings := make([]string, len(endpoints))
-	for i, endpoint := range endpoints {
+	endpointStrings := make([]string, len(endpoints.Endpoints))
+	for i, endpoint := range endpoints.Endpoints {
 		endpointStrings[i] = endpoint.String()
 	}
 
@@ -399,7 +402,7 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	}
 
 	erasureLockers := map[string]dsync.NetLocker{}
-	for _, endpoint := range endpoints {
+	for _, endpoint := range endpoints.Endpoints {
 		if _, ok := erasureLockers[endpoint.Host]; !ok {
 			erasureLockers[endpoint.Host] = newLockAPI(endpoint)
 		}
@@ -408,50 +411,76 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	for i := 0; i < setCount; i++ {
 		lockerEpSet := set.NewStringSet()
 		for j := 0; j < setDriveCount; j++ {
-			endpoint := endpoints[i*setDriveCount+j]
-			// Only add lockers per endpoint.
+			endpoint := endpoints.Endpoints[i*setDriveCount+j]
+			// Only add lockers only one per endpoint and per erasure set.
 			if locker, ok := erasureLockers[endpoint.Host]; ok && !lockerEpSet.Contains(endpoint.Host) {
 				lockerEpSet.Add(endpoint.Host)
 				s.erasureLockers[i] = append(s.erasureLockers[i], locker)
 			}
-			disk := storageDisks[i*setDriveCount+j]
-			if disk == nil {
-				continue
-			}
-			diskID, derr := disk.GetDiskID()
-			if derr != nil {
-				continue
-			}
-			m, n, err := findDiskIndexByDiskID(format, diskID)
-			if err != nil {
-				continue
-			}
-			if m != i || n != j {
-				logger.LogIf(GlobalContext, fmt.Errorf("Detected unexpected disk ordering refusing to use the disk - poolID: %s, found disk mounted at (set=%s, disk=%s) expected mount at (set=%s, disk=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
-				s.erasureDisks[i][j] = &unrecognizedDisk{storage: disk}
-				continue
-			}
-			disk.SetDiskLoc(s.poolIndex, m, n)
-			s.endpointStrings[m*setDriveCount+n] = disk.String()
-			s.erasureDisks[m][n] = disk
-		}
-
-		// Initialize erasure objects for a given set.
-		s.sets[i] = &erasureObjects{
-			setIndex:              i,
-			poolIndex:             poolIdx,
-			setDriveCount:         setDriveCount,
-			defaultParityCount:    defaultParityCount,
-			getDisks:              s.GetDisks(i),
-			getLockers:            s.GetLockers(i),
-			getEndpoints:          s.GetEndpoints(i),
-			deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
-			nsMutex:               mutex,
-			bp:                    bp,
-			bpOld:                 bpOld,
 		}
 	}
 
+	var wg sync.WaitGroup
+	for i := 0; i < setCount; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+
+			var innerWg sync.WaitGroup
+			for j := 0; j < setDriveCount; j++ {
+				disk := storageDisks[i*setDriveCount+j]
+				if disk == nil {
+					continue
+				}
+				innerWg.Add(1)
+				go func(disk StorageAPI, i, j int) {
+					defer innerWg.Done()
+					diskID, err := disk.GetDiskID()
+					if err != nil {
+						if !errors.Is(err, errUnformattedDisk) {
+							logger.LogIf(ctx, err)
+						}
+						return
+					}
+					if diskID == "" {
+						return
+					}
+					m, n, err := findDiskIndexByDiskID(format, diskID)
+					if err != nil {
+						logger.LogIf(ctx, err)
+						return
+					}
+					if m != i || n != j {
+						logger.LogIf(ctx, fmt.Errorf("Detected unexpected disk ordering refusing to use the disk - poolID: %s, found disk mounted at (set=%s, disk=%s) expected mount at (set=%s, disk=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
+						s.erasureDisks[i][j] = &unrecognizedDisk{storage: disk}
+						return
+					}
+					disk.SetDiskLoc(s.poolIndex, m, n)
+					s.endpointStrings[m*setDriveCount+n] = disk.String()
+					s.erasureDisks[m][n] = disk
+				}(disk, i, j)
+			}
+			innerWg.Wait()
+
+			// Initialize erasure objects for a given set.
+			s.sets[i] = &erasureObjects{
+				setIndex:              i,
+				poolIndex:             poolIdx,
+				setDriveCount:         setDriveCount,
+				defaultParityCount:    defaultParityCount,
+				getDisks:              s.GetDisks(i),
+				getLockers:            s.GetLockers(i),
+				getEndpoints:          s.GetEndpoints(i),
+				deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
+				nsMutex:               mutex,
+				bp:                    bp,
+				bpOld:                 bpOld,
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
 	// start cleanup stale uploads go-routine.
 	go s.cleanupStaleUploads(ctx)
 
@@ -974,17 +1003,25 @@ func (s *erasureSets) DeleteObjects(ctx context.Context, bucket string, objects
 
 	// Invoke bulk delete on objects per set and save
 	// the result of the delete operation
-	for _, objsGroup := range objSetMap {
-		set := s.getHashedSet(objsGroup[0].object.ObjectName)
-		dobjects, errs := set.DeleteObjects(ctx, bucket, toNames(objsGroup), opts)
-		for i, obj := range objsGroup {
-			delErrs[obj.origIndex] = errs[i]
-			delObjects[obj.origIndex] = dobjects[i]
-			if errs[i] == nil {
-				auditObjectErasureSet(ctx, obj.object.ObjectName, set)
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	wg.Add(len(objSetMap))
+	for setIdx, objsGroup := range objSetMap {
+		go func(set *erasureObjects, group []delObj) {
+			defer wg.Done()
+			dobjects, errs := set.DeleteObjects(ctx, bucket, toNames(group), opts)
+			mu.Lock()
+			defer mu.Unlock()
+			for i, obj := range group {
+				delErrs[obj.origIndex] = errs[i]
+				delObjects[obj.origIndex] = dobjects[i]
+				if errs[i] == nil {
+					auditObjectErasureSet(ctx, obj.object.ObjectName, set)
+				}
 			}
-		}
+		}(s.sets[setIdx], objsGroup)
 	}
+	wg.Wait()
 
 	return delObjects, delErrs
 }
@@ -1226,7 +1263,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
-	storageDisks, _ := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints)
+	storageDisks, _ := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints.Endpoints)
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {
@@ -1256,7 +1293,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	}
 
 	// Fetch all the drive info status.
-	beforeDrives := formatsToDrivesInfo(s.endpoints, formats, sErrs)
+	beforeDrives := formatsToDrivesInfo(s.endpoints.Endpoints, formats, sErrs)
 
 	res.After.Drives = make([]madmin.HealDriveInfo, len(beforeDrives))
 	res.Before.Drives = make([]madmin.HealDriveInfo, len(beforeDrives))
@@ -1306,6 +1343,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 			m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This)
 			if err != nil {
+				logger.LogIf(ctx, err)
 				continue
 			}
 

cmd/erasure-sets_test.go

@@ -190,7 +190,8 @@ func TestNewErasureSets(t *testing.T) {
 		t.Fatalf("Unable to format disks for erasure, %s", err)
 	}
 
-	if _, err := newErasureSets(ctx, endpoints, storageDisks, format, ecDrivesNoConfig(16), 0); err != nil {
+	ep := PoolEndpoints{Endpoints: endpoints}
+	if _, err := newErasureSets(ctx, ep, storageDisks, format, ecDrivesNoConfig(16), 0); err != nil {
 		t.Fatalf("Unable to initialize erasure")
 	}
 }

cmd/erasure.go

@@ -214,7 +214,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				}
 			}
 			di.Metrics = &madmin.DiskMetrics{
-				APILatencies: make(map[string]string),
+				APILatencies: make(map[string]interface{}),
 				APICalls:     make(map[string]uint64),
 			}
 			for k, v := range info.Metrics.APILatencies {
@@ -357,23 +357,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 		return nil
 	}
 
-	// Collect disks for healing.
-	allDisks := er.getDisks()
-	allDiskIDs := make([]string, 0, len(allDisks))
-	for _, disk := range allDisks {
-		if disk == OfflineDisk {
-			// its possible that disk is OfflineDisk
-			continue
-		}
-		id, _ := disk.GetDiskID()
-		if id == "" {
-			// its possible that disk is unformatted
-			// or just went offline
-			continue
-		}
-		allDiskIDs = append(allDiskIDs, id)
-	}
-
 	// Load bucket totals
 	oldCache := dataUsageCache{}
 	if err := oldCache.load(ctx, er, dataUsageCacheName); err != nil {
@@ -479,7 +462,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				}
 				cache.Info.BloomFilter = bloom
 				cache.Info.SkipHealing = healing
-				cache.Disks = allDiskIDs
 				cache.Info.NextCycle = wantCycle
 				if cache.Info.Name != bucket.Name {
 					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
@@ -517,6 +499,11 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 					} else {
 						logger.LogIf(ctx, err)
 					}
+					// This ensures that we don't close
+					// bucketResults channel while the
+					// updates-collector goroutine still
+					// holds a reference to this.
+					wg.Wait()
 					continue
 				}
 

cmd/format-erasure.go

@@ -24,6 +24,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"io/ioutil"
 	"reflect"
 	"sync"
@@ -156,13 +157,9 @@ func newFormatErasureV3(numSets int, setLen int) *formatErasureV3 {
 
 // Returns format Erasure version after reading `format.json`, returns
 // successfully the version only if the backend is Erasure.
-func formatGetBackendErasureVersion(formatPath string) (string, error) {
+func formatGetBackendErasureVersion(b []byte) (string, error) {
 	meta := &formatMetaV1{}
-	b, err := xioutil.ReadFile(formatPath)
-	if err != nil {
-		return "", err
-	}
-	if err = json.Unmarshal(b, meta); err != nil {
+	if err := json.Unmarshal(b, meta); err != nil {
 		return "", err
 	}
 	if meta.Version != formatMetaVersionV1 {
@@ -173,7 +170,7 @@ func formatGetBackendErasureVersion(formatPath string) (string, error) {
 	}
 	// Erasure backend found, proceed to detect version.
 	format := &formatErasureVersionDetect{}
-	if err = json.Unmarshal(b, format); err != nil {
+	if err := json.Unmarshal(b, format); err != nil {
 		return "", err
 	}
 	return format.Erasure.Version, nil
@@ -182,50 +179,63 @@ func formatGetBackendErasureVersion(formatPath string) (string, error) {
 // Migrates all previous versions to latest version of `format.json`,
 // this code calls migration in sequence, such as V1 is migrated to V2
 // first before it V2 migrates to V3.n
-func formatErasureMigrate(export string) error {
+func formatErasureMigrate(export string) ([]byte, fs.FileInfo, error) {
 	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
-	version, err := formatGetBackendErasureVersion(formatPath)
+	formatData, formatFi, err := xioutil.ReadFileWithFileInfo(formatPath)
 	if err != nil {
-		return fmt.Errorf("Disk %s: %w", export, err)
+		return nil, nil, err
 	}
+
+	version, err := formatGetBackendErasureVersion(formatData)
+	if err != nil {
+		return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
+	}
+
+	migrate := func(formatPath string, formatData []byte) ([]byte, fs.FileInfo, error) {
+		if err = ioutil.WriteFile(formatPath, formatData, 0o666); err != nil {
+			return nil, nil, err
+		}
+		formatFi, err := Lstat(formatPath)
+		if err != nil {
+			return nil, nil, err
+		}
+		return formatData, formatFi, nil
+	}
+
 	switch version {
 	case formatErasureVersionV1:
-		if err = formatErasureMigrateV1ToV2(export, version); err != nil {
-			return fmt.Errorf("Disk %s: %w", export, err)
+		formatData, err = formatErasureMigrateV1ToV2(formatData, version)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
 		}
 		// Migrate successful v1 => v2, proceed to v2 => v3
 		version = formatErasureVersionV2
 		fallthrough
 	case formatErasureVersionV2:
-		if err = formatErasureMigrateV2ToV3(export, version); err != nil {
-			return fmt.Errorf("Disk %s: %w", export, err)
+		formatData, err = formatErasureMigrateV2ToV3(formatData, export, version)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Disk %s: %w", export, err)
 		}
 		// Migrate successful v2 => v3, v3 is latest
 		// version = formatXLVersionV3
-		fallthrough
+		return migrate(formatPath, formatData)
 	case formatErasureVersionV3:
 		// v3 is the latest version, return.
-		return nil
+		return formatData, formatFi, nil
 	}
-	return fmt.Errorf(`Disk %s: unknown format version %s`, export, version)
+	return nil, nil, fmt.Errorf(`Disk %s: unknown format version %s`, export, version)
 }
 
 // Migrates version V1 of format.json to version V2 of format.json,
 // migration fails upon any error.
-func formatErasureMigrateV1ToV2(export, version string) error {
+func formatErasureMigrateV1ToV2(data []byte, version string) ([]byte, error) {
 	if version != formatErasureVersionV1 {
-		return fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV1, version)
+		return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV1, version)
 	}
 
-	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
-
 	formatV1 := &formatErasureV1{}
-	b, err := xioutil.ReadFile(formatPath)
-	if err != nil {
-		return err
-	}
-	if err = json.Unmarshal(b, formatV1); err != nil {
-		return err
+	if err := json.Unmarshal(data, formatV1); err != nil {
+		return nil, err
 	}
 
 	formatV2 := &formatErasureV2{}
@@ -238,53 +248,38 @@ func formatErasureMigrateV1ToV2(export, version string) error {
 	formatV2.Erasure.Sets[0] = make([]string, len(formatV1.Erasure.JBOD))
 	copy(formatV2.Erasure.Sets[0], formatV1.Erasure.JBOD)
 
-	b, err = json.Marshal(formatV2)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(formatPath, b, 0o666)
+	return json.Marshal(formatV2)
 }
 
 // Migrates V2 for format.json to V3 (Flat hierarchy for multipart)
-func formatErasureMigrateV2ToV3(export, version string) error {
+func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, error) {
 	if version != formatErasureVersionV2 {
-		return fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV2, version)
+		return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV2, version)
 	}
 
-	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
 	formatV2 := &formatErasureV2{}
-	b, err := xioutil.ReadFile(formatPath)
-	if err != nil {
-		return err
-	}
-	err = json.Unmarshal(b, formatV2)
-	if err != nil {
-		return err
+	if err := json.Unmarshal(data, formatV2); err != nil {
+		return nil, err
 	}
 
-	if err = removeAll(pathJoin(export, minioMetaMultipartBucket)); err != nil {
-		return err
-	}
-
-	if err = mkdirAll(pathJoin(export, minioMetaMultipartBucket), 0o755); err != nil {
-		return err
+	tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID())
+	if err := renameAll(pathJoin(export, minioMetaMultipartBucket),
+		tmpOld); err != nil && err != errFileNotFound {
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
+			pathJoin(export, minioMetaMultipartBucket),
+			tmpOld,
+			osErrToFileErr(err)))
 	}
 
 	// format-V2 struct is exactly same as format-V1 except that version is "3"
 	// which indicates the simplified multipart backend.
 	formatV3 := formatErasureV3{}
-
 	formatV3.Version = formatV2.Version
 	formatV3.Format = formatV2.Format
 	formatV3.Erasure = formatV2.Erasure
-
 	formatV3.Erasure.Version = formatErasureVersionV3
 
-	b, err = json.Marshal(formatV3)
-	if err != nil {
-		return err
-	}
-	return ioutil.WriteFile(formatPath, b, 0o666)
+	return json.Marshal(formatV3)
 }
 
 // countErrs - count a specific error.
@@ -382,25 +377,6 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	return nil
 }
 
-var ignoredHiddenDirectories = map[string]struct{}{
-	minioMetaBucket:             {}, // metabucket '.minio.sys'
-	".minio":                    {}, // users may choose to double down the backend as the config folder for certs
-	".snapshot":                 {}, // .snapshot for ignoring NetApp based persistent volumes WAFL snapshot
-	"lost+found":                {}, // 'lost+found' directory default on ext4 filesystems
-	"$RECYCLE.BIN":              {}, // windows specific directory for each drive (hidden)
-	"System Volume Information": {}, // windows specific directory for each drive (hidden)
-}
-
-func isHiddenDirectories(vols ...VolInfo) bool {
-	for _, vol := range vols {
-		if _, ok := ignoredHiddenDirectories[vol.Name]; ok {
-			continue
-		}
-		return false
-	}
-	return true
-}
-
 // loadFormatErasure - loads format.json from disk.
 func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
 	buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
@@ -408,17 +384,6 @@ func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
 		// 'file not found' and 'volume not found' as
 		// same. 'volume not found' usually means its a fresh disk.
 		if err == errFileNotFound || err == errVolumeNotFound {
-			var vols []VolInfo
-			vols, err = disk.ListVols(context.TODO())
-			if err != nil {
-				return nil, err
-			}
-			if !isHiddenDirectories(vols...) {
-				// 'format.json' not found, but we found user data, reject such disks.
-				return nil, fmt.Errorf("some unexpected files '%v' found on %s: %w",
-					vols, disk, errCorruptedFormat)
-			}
-			// No other data found, its a fresh disk.
 			return nil, errUnformattedDisk
 		}
 		return nil, err
@@ -718,12 +683,18 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 
 // relinquishes the underlying connection for all storage disks.
 func closeStorageDisks(storageDisks []StorageAPI) {
+	var wg sync.WaitGroup
 	for _, disk := range storageDisks {
 		if disk == nil {
 			continue
 		}
-		disk.Close()
+		wg.Add(1)
+		go func(disk StorageAPI) {
+			defer wg.Done()
+			disk.Close()
+		}(disk)
 	}
+	wg.Wait()
 }
 
 func initStorageDisksWithErrorsWithoutHealthCheck(endpoints Endpoints) ([]StorageAPI, []error) {
@@ -896,8 +867,14 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error {
 	if disk == nil {
 		return errDiskNotFound
 	}
+	volumes := []string{
+		minioMetaTmpDeletedBucket, // creates .minio.sys/tmp as well as .minio.sys/tmp/.trash
+		minioMetaMultipartBucket,  // creates .minio.sys/multipart
+		dataUsageBucket,           // creates .minio.sys/buckets
+		minioConfigBucket,         // creates .minio.sys/config
+	}
 	// Attempt to create MinIO internal buckets.
-	return disk.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaTmpDeletedBucket, dataUsageBucket, minioMetaTmpBucket+"-old")
+	return disk.MakeVolBulk(context.TODO(), volumes...)
 }
 
 // Initialize a new set of set formats which will be written to all disks.

cmd/format-erasure_test.go

@@ -132,11 +132,12 @@ func TestFormatErasureMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = formatErasureMigrate(rootPath); err != nil {
+	formatData, _, err := formatErasureMigrate(rootPath)
+	if err != nil {
 		t.Fatal(err)
 	}
 
-	migratedVersion, err := formatGetBackendErasureVersion(pathJoin(rootPath, minioMetaBucket, formatConfigFile))
+	migratedVersion, err := formatGetBackendErasureVersion(formatData)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -179,7 +180,7 @@ func TestFormatErasureMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = formatErasureMigrate(rootPath); err == nil {
+	if _, _, err = formatErasureMigrate(rootPath); err == nil {
 		t.Fatal("Expected to fail with unexpected backend format")
 	}
 
@@ -199,7 +200,7 @@ func TestFormatErasureMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err = formatErasureMigrate(rootPath); err == nil {
+	if _, _, err = formatErasureMigrate(rootPath); err == nil {
 		t.Fatal("Expected to fail with unexpected backend format version number")
 	}
 }

cmd/fs-v1.go

@@ -352,7 +352,7 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 	}
 
 	// Load bucket info.
-	cache, err = scanDataFolder(ctx, fs.fsPath, cache, func(item scannerItem) (sizeSummary, error) {
+	cache, err = scanDataFolder(ctx, -1, -1, fs.fsPath, cache, func(item scannerItem) (sizeSummary, error) {
 		bucket, object := item.bucket, item.objectPath()
 		fsMetaBytes, err := xioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile))
 		if err != nil && !osIsNotExist(err) {

cmd/gateway/azure/gateway-azure.go

@@ -1416,6 +1416,7 @@ func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (*pol
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
 			policy.NewStatement(
+				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(

cmd/gateway/gcs/gateway-gcs.go

@@ -1470,6 +1470,7 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
 			policy.NewStatement(
+				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				actionSet,

cmd/global-heal.go

@@ -219,7 +219,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 
 		healEntry := func(entry metaCacheEntry) {
+			if entry.name == "" && len(entry.metadata) == 0 {
+				// ignore entries that don't have metadata.
+				return
+			}
 			if entry.isDir() {
+				// ignore healing entry.name's with `/` suffix.
 				return
 			}
 			// We might land at .metacache, .trash, .multipart
@@ -246,7 +251,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}, madmin.HealItemObject)
 				if err != nil {
 					tracker.ItemsFailed++
-					logger.LogIf(ctx, err)
+					logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
 					tracker.ItemsHealed++
 				}
@@ -263,7 +268,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					// If not deleted, assume they failed.
 					tracker.ItemsFailed++
 					tracker.BytesFailed += uint64(version.Size)
-					logger.LogIf(ctx, err)
+					if version.VersionID != "" {
+						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
+					} else {
+						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
+					}
 				} else {
 					tracker.ItemsHealed++
 					tracker.BytesDone += uint64(version.Size)

cmd/globals.go

@@ -118,6 +118,9 @@ const (
 
 	// diskMinInodes is the minimum number of inodes we want free on a disk to perform writes.
 	diskMinInodes = 1000
+
+	// tlsClientSessionCacheSize is the cache size for client sessions.
+	tlsClientSessionCacheSize = 100
 )
 
 var globalCLIContext = struct {
@@ -335,6 +338,13 @@ var (
 	globalServiceFreezeCnt int32
 	globalServiceFreezeMu  sync.Mutex // Updates.
 
+	// List of local drives to this node, this is only set during server startup.
+	globalLocalDrives []StorageAPI
+
+	// Is MINIO_CI_CD set?
+	globalIsCICD bool
+
+	globalRootDiskThreshold uint64
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -49,6 +49,7 @@ type apiConfig struct {
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
 	deleteCleanupInterval       time.Duration
+	disableODirect              bool
 }
 
 const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
@@ -150,6 +151,14 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
 	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
+	t.disableODirect = cfg.DisableODirect
+}
+
+func (t *apiConfig) isDisableODirect() bool {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.disableODirect
 }
 
 func (t *apiConfig) getListQuorum() int {

cmd/handler-utils.go

@@ -429,69 +429,8 @@ func extractAPIVersion(r *http.Request) string {
 	return "unknown"
 }
 
-func generateUnexpectedRPCMsg(rpcPath, subsystem string, expectedVersion, gotVersion string) string {
-	var reason string
-	switch {
-	case expectedVersion != gotVersion:
-		reason = fmt.Sprintf("Server expects '%s' API version '%s', instead found '%s'", subsystem, expectedVersion, gotVersion)
-	default:
-		reason = fmt.Sprintf("Unexpected RPC call at this path '%s'", rpcPath)
-	}
-
-	return fmt.Sprintf("%s - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", reason, ReleaseTag)
-}
-
 func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if r.Method == http.MethodOptions {
-			return
-		}
-		version := extractAPIVersion(r)
-		switch {
-		case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
-			desc := generateUnexpectedRPCMsg(r.URL.Path, "peer", peerRESTVersion, version)
-			writeErrorResponseString(r.Context(), w, APIError{
-				Code:           "XMinioPeerVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
-			desc := generateUnexpectedRPCMsg(r.URL.Path, "storage", storageRESTVersion, version)
-			writeErrorResponseString(r.Context(), w, APIError{
-				Code:           "XMinioStorageVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
-			desc := generateUnexpectedRPCMsg(r.URL.Path, "lock", lockRESTVersion, version)
-			writeErrorResponseString(r.Context(), w, APIError{
-				Code:           "XMinioLockVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		case strings.HasPrefix(r.URL.Path, adminPathPrefix):
-			var desc string
-			if version == "v1" {
-				desc = fmt.Sprintf("Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases", madmin.AdminAPIVersion, version)
-			} else if version == madmin.AdminAPIVersion {
-				desc = fmt.Sprintf("This 'admin' API is not supported by server in '%s'", getMinioMode())
-			} else {
-				desc = fmt.Sprintf("Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases", version, madmin.AdminAPIVersion)
-			}
-			writeErrorResponseJSON(r.Context(), w, APIError{
-				Code:           "XMinioAdminVersionMismatch",
-				Description:    desc,
-				HTTPStatusCode: http.StatusUpgradeRequired,
-			}, r.URL)
-		default:
-			writeErrorResponse(r.Context(), w, APIError{
-				Code: "BadRequest",
-				Description: fmt.Sprintf("An error occurred when parsing the HTTP request %s at '%s'",
-					r.Method, r.URL.Path),
-				HTTPStatusCode: http.StatusBadRequest,
-			}, r.URL)
-		}
-	}
+	return errorResponseHandler
 }
 
 // If none of the http routes match respond with appropriate errors
@@ -499,24 +438,21 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method == http.MethodOptions {
 		return
 	}
-	version := extractAPIVersion(r)
+	desc := "Do not upgrade one server at a time - please follow the recommended guidelines mentioned here https://github.com/minio/minio#upgrading-minio for your environment"
 	switch {
 	case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
-		desc := fmt.Sprintf("Server expects 'peer' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", peerRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",
 			Description:    desc,
 			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
-		desc := fmt.Sprintf("Server expects 'storage' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", storageRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioStorageVersionMismatch",
 			Description:    desc,
 			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
-		desc := fmt.Sprintf("Server expects 'lock' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", lockRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioLockVersionMismatch",
 			Description:    desc,
@@ -524,11 +460,13 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, adminPathPrefix):
 		var desc string
-		if version == "v1" {
+		version := extractAPIVersion(r)
+		switch version {
+		case "v1", madmin.AdminAPIVersionV2:
 			desc = fmt.Sprintf("Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases", madmin.AdminAPIVersion, version)
-		} else if version == madmin.AdminAPIVersion {
+		case madmin.AdminAPIVersion:
 			desc = fmt.Sprintf("This 'admin' API is not supported by server in '%s'", getMinioMode())
-		} else {
+		default:
 			desc = fmt.Sprintf("Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases", version, madmin.AdminAPIVersion)
 		}
 		writeErrorResponseJSON(r.Context(), w, APIError{

cmd/iam-store.go

@@ -1084,11 +1084,12 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 			continue
 		}
 		p, found := cache.iamPolicyDocsMap[policy]
-		if found {
-			if bucketName == "" || p.Policy.MatchResource(bucketName) {
-				policies = append(policies, policy)
-				combinedPolicy = combinedPolicy.Merge(p.Policy)
-			}
+		if !found {
+			continue
+		}
+		if bucketName == "" || p.Policy.MatchResource(bucketName) {
+			policies = append(policies, policy)
+			combinedPolicy = combinedPolicy.Merge(p.Policy)
 		}
 	}
 	return strings.Join(policies, ","), combinedPolicy
@@ -1456,7 +1457,20 @@ func (store *IAMStoreSys) GetAllParentUsers() []string {
 	res := set.NewStringSet()
 	for _, cred := range cache.iamUsersMap {
 		if cred.IsServiceAccount() || cred.IsTemp() {
-			res.Add(cred.ParentUser)
+			parentUser := cred.ParentUser
+			if cred.SessionToken != "" {
+				claims, err := getClaimsFromToken(cred.SessionToken)
+				if err != nil {
+					continue
+				}
+				if v, ok := claims[subClaim]; ok {
+					subFromToken, ok := v.(string)
+					if ok {
+						parentUser = subFromToken
+					}
+				}
+			}
+			res.Add(parentUser)
 		}
 	}
 
@@ -1485,7 +1499,8 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
 		AccessKey: accessKey,
 		SecretKey: cred.SecretKey,
 		Status: func() string {
-			if status == madmin.AccountEnabled {
+			switch string(status) {
+			case string(madmin.AccountEnabled), string(auth.AccountOn):
 				return auth.AccountOn
 			}
 			return auth.AccountOff
@@ -1511,13 +1526,16 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
 	// Found newly requested service account, to be an existing account -
 	// reject such operation (updates to the service account are handled in
 	// a different API).
-	if _, found := cache.iamUsersMap[accessKey]; found {
-		return errIAMActionNotAllowed
+	if scred, found := cache.iamUsersMap[accessKey]; found {
+		if scred.ParentUser != parentUser {
+			return errIAMServiceAccountUsed
+		}
+		return errIAMServiceAccount
 	}
 
 	// Parent user must not be a service account.
 	if cr, found := cache.iamUsersMap[parentUser]; found && cr.IsServiceAccount() {
-		return errIAMActionNotAllowed
+		return errIAMServiceAccount
 	}
 
 	u := newUserIdentity(cred)
@@ -1551,6 +1569,10 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 	switch opts.status {
 	// The caller did not ask to update status account, do nothing
 	case "":
+	case string(madmin.AccountEnabled):
+		cr.Status = auth.AccountOn
+	case string(madmin.AccountDisabled):
+		cr.Status = auth.AccountOff
 	// Update account status
 	case auth.AccountOn, auth.AccountOff:
 		cr.Status = opts.status
@@ -1629,7 +1651,8 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
 		AccessKey: accessKey,
 		SecretKey: ureq.SecretKey,
 		Status: func() string {
-			if ureq.Status == madmin.AccountEnabled {
+			switch string(ureq.Status) {
+			case string(madmin.AccountEnabled), string(auth.AccountOn):
 				return auth.AccountOn
 			}
 			return auth.AccountOff

cmd/iam.go

@@ -636,6 +636,7 @@ func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.
 	}
 
 	sys.notifyForUser(ctx, cred.AccessKey, true)
+
 	return nil
 }
 
@@ -882,18 +883,19 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut
 	var embeddedPolicy *iampolicy.Policy
 
 	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
-	if err == nil {
-		pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
-		sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
-		if ptok && spok && pt == "embedded-policy" {
-			policyBytes, err := base64.StdEncoding.DecodeString(sp)
-			if err == nil {
-				p, err := iampolicy.ParseConfig(bytes.NewReader(policyBytes))
-				if err == nil {
-					policy := iampolicy.Policy{}.Merge(*p)
-					embeddedPolicy = &policy
-				}
-			}
+	if err != nil {
+		return auth.Credentials{}, nil, err
+	}
+	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
+	sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
+	if ptok && spok && pt == "embedded-policy" {
+		policyBytes, err := base64.StdEncoding.DecodeString(sp)
+		if err != nil {
+			return auth.Credentials{}, nil, err
+		}
+		embeddedPolicy, err = iampolicy.ParseConfig(bytes.NewReader(policyBytes))
+		if err != nil {
+			return auth.Credentials{}, nil, err
 		}
 	}
 
@@ -1004,11 +1006,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var expiredUsers []string
 	for _, parentUser := range parentUsers {
-		userid, err := parseOpenIDParentUser(parentUser)
-		if err == errSkipFile {
-			continue
-		}
-		u, err := globalOpenIDConfig.LookupUser(userid)
+		u, err := globalOpenIDConfig.LookupUser(parentUser)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 			continue

cmd/jwt.go

@@ -132,13 +132,13 @@ func authenticateURL(accessKey, secretKey string) (string, error) {
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
-func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, bool, error) {
+func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
 	token, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req)
 	if err != nil {
 		if err == jwtreq.ErrNoTokenInRequest {
-			return nil, false, errNoAuthToken
+			return nil, nil, false, errNoAuthToken
 		}
-		return nil, false, err
+		return nil, nil, false, err
 	}
 	claims := xjwt.NewMapClaims()
 	if err := xjwt.ParseWithClaims(token, claims, func(claims *xjwt.MapClaims) ([]byte, error) {
@@ -151,20 +151,21 @@ func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, bool, error) {
 		}
 		return []byte(cred.SecretKey), nil
 	}); err != nil {
-		return claims, false, errAuthentication
+		return claims, nil, false, errAuthentication
 	}
 	owner := true
+	var groups []string
 	if globalActiveCred.AccessKey != claims.AccessKey {
 		// Check if the access key is part of users credentials.
 		ucred, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
 		if !ok {
-			return nil, false, errInvalidAccessKeyID
+			return nil, nil, false, errInvalidAccessKeyID
 		}
 
 		// get embedded claims
 		eclaims, s3Err := checkClaimsFromToken(req, ucred)
 		if s3Err != ErrNone {
-			return nil, false, errAuthentication
+			return nil, nil, false, errAuthentication
 		}
 
 		for k, v := range eclaims {
@@ -177,9 +178,11 @@ func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, bool, error) {
 		} else {
 			owner = globalActiveCred.AccessKey == ucred.ParentUser
 		}
+
+		groups = ucred.Groups
 	}
 
-	return claims, owner, nil
+	return claims, groups, owner, nil
 }
 
 // newCachedAuthToken returns a token that is cached up to 15 seconds.

cmd/jwt_test.go

@@ -149,7 +149,7 @@ func TestWebRequestAuthenticate(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, _, gotErr := webRequestAuthenticate(testCase.req)
+		_, _, _, gotErr := webRequestAuthenticate(testCase.req)
 		if testCase.expectedErr != gotErr {
 			t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr)
 		}

cmd/last-minute.go

@@ -15,6 +15,8 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+//go:generate msgp -file=$GOFILE -unexported
+
 package cmd
 
 import (
@@ -76,34 +78,34 @@ type AccElem struct {
 	N     int64
 }
 
-// add dur to a as a single element.
+// Add a duration to a single element.
 func (a *AccElem) add(dur time.Duration) {
 	a.Total += int64(dur)
 	a.N++
 }
 
-// merge b into a.
+// Merge b into a.
 func (a *AccElem) merge(b AccElem) {
 	a.N += b.N
 	a.Total += b.Total
 }
 
-// avg converts total to average.
-func (a *AccElem) avg() uint64 {
+// Avg converts total to average.
+func (a AccElem) avg() uint64 {
 	if a.N >= 1 && a.Total > 0 {
 		return uint64(a.Total / a.N)
 	}
 	return 0
 }
 
-// LastMinuteLatencies keeps track of last minute latencies.
-type LastMinuteLatencies struct {
-	Totals  [60][sizeLastElemMarker]AccElem
+// lastMinuteLatency keeps track of last minute latency.
+type lastMinuteLatency struct {
+	Totals  [60]AccElem
 	LastSec int64
 }
 
-// Merge safely merges two LastMinuteLatencies structures into one
-func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLatencies) {
+// Merge data of two lastMinuteLatency structure
+func (l lastMinuteLatency) merge(o lastMinuteLatency) (merged lastMinuteLatency) {
 	if l.LastSec > o.LastSec {
 		o.forwardTo(l.LastSec)
 		merged.LastSec = l.LastSec
@@ -113,57 +115,73 @@ func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLate
 	}
 
 	for i := range merged.Totals {
-		for j := range merged.Totals[i] {
-			merged.Totals[i][j] = AccElem{
-				Total: l.Totals[i][j].Total + o.Totals[i][j].Total,
-				N:     l.Totals[i][j].N + o.Totals[i][j].N,
-			}
+		merged.Totals[i] = AccElem{
+			Total: l.Totals[i].Total + o.Totals[i].Total,
+			N:     l.Totals[i].N + o.Totals[i].N,
 		}
 	}
 	return merged
 }
 
+// Add  a new duration data
+func (l *lastMinuteLatency) add(t time.Duration) {
+	sec := time.Now().Unix()
+	l.forwardTo(sec)
+	winIdx := sec % 60
+	l.Totals[winIdx].add(t)
+	l.LastSec = sec
+}
+
+// Merge all recorded latencies of last minute into one
+func (l *lastMinuteLatency) getAvgData() AccElem {
+	var res AccElem
+	sec := time.Now().Unix()
+	l.forwardTo(sec)
+	for _, elem := range l.Totals[:] {
+		res.merge(elem)
+	}
+	return res
+}
+
+// forwardTo time t, clearing any entries in between.
+func (l *lastMinuteLatency) forwardTo(t int64) {
+	if l.LastSec >= t {
+		return
+	}
+	if t-l.LastSec >= 60 {
+		l.Totals = [60]AccElem{}
+		return
+	}
+	for l.LastSec != t {
+		// Clear next element.
+		idx := (l.LastSec + 1) % 60
+		l.Totals[idx] = AccElem{}
+		l.LastSec++
+	}
+}
+
+// LastMinuteLatencies keeps track of last minute latencies.
+type LastMinuteLatencies [sizeLastElemMarker]lastMinuteLatency
+
+// Merge safely merges two LastMinuteLatencies structures into one
+func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLatencies) {
+	for i := range l {
+		merged[i] = l[i].merge(o[i])
+	}
+	return merged
+}
+
 // Add latency t from object with the specified size.
 func (l *LastMinuteLatencies) Add(size int64, t time.Duration) {
-	tag := sizeToTag(size)
-
-	// Update...
-	sec := time.Now().Unix()
-	l.forwardTo(sec)
-
-	winIdx := sec % 60
-	l.Totals[winIdx][tag].add(t)
-
-	l.LastSec = sec
+	l[sizeToTag(size)].add(t)
 }
 
-// GetAvg will return the average for each bucket from the last time minute.
+// GetAvgData will return the average for each bucket from the last time minute.
 // The number of objects is also included.
-func (l *LastMinuteLatencies) GetAvg() [sizeLastElemMarker]AccElem {
+func (l *LastMinuteLatencies) GetAvgData() [sizeLastElemMarker]AccElem {
 	var res [sizeLastElemMarker]AccElem
-	sec := time.Now().Unix()
-	l.forwardTo(sec)
-	for _, elems := range l.Totals[:] {
-		for j := range elems {
-			res[j].merge(elems[j])
-		}
+	for i, elem := range l[:] {
+		res[i] = elem.getAvgData()
 	}
 	return res
 }
-
-// forwardTo time t, clearing any entries in between.
-func (l *LastMinuteLatencies) forwardTo(t int64) {
-	if l.LastSec >= t {
-		return
-	}
-	if t-l.LastSec >= 60 {
-		l.Totals = [60][sizeLastElemMarker]AccElem{}
-		return
-	}
-	for l.LastSec != t {
-		// Clear next element.
-		idx := (l.LastSec + 1) % 60
-		l.Totals[idx] = [sizeLastElemMarker]AccElem{}
-		l.LastSec++
-	}
-}

cmd/last-minute_gen.go

@@ -136,6 +136,282 @@ func (z AccElem) Msgsize() (s int) {
 
 // DecodeMsg implements msgp.Decodable
 func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(sizeLastElemMarker) {
+		err = msgp.ArrayError{Wanted: uint32(sizeLastElemMarker), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		var field []byte
+		_ = field
+		var zb0002 uint32
+		zb0002, err = dc.ReadMapHeader()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+		for zb0002 > 0 {
+			zb0002--
+			field, err = dc.ReadMapKeyPtr()
+			if err != nil {
+				err = msgp.WrapError(err, za0001)
+				return
+			}
+			switch msgp.UnsafeString(field) {
+			case "Totals":
+				var zb0003 uint32
+				zb0003, err = dc.ReadArrayHeader()
+				if err != nil {
+					err = msgp.WrapError(err, za0001, "Totals")
+					return
+				}
+				if zb0003 != uint32(60) {
+					err = msgp.ArrayError{Wanted: uint32(60), Got: zb0003}
+					return
+				}
+				for za0002 := range z[za0001].Totals {
+					var zb0004 uint32
+					zb0004, err = dc.ReadMapHeader()
+					if err != nil {
+						err = msgp.WrapError(err, za0001, "Totals", za0002)
+						return
+					}
+					for zb0004 > 0 {
+						zb0004--
+						field, err = dc.ReadMapKeyPtr()
+						if err != nil {
+							err = msgp.WrapError(err, za0001, "Totals", za0002)
+							return
+						}
+						switch msgp.UnsafeString(field) {
+						case "Total":
+							z[za0001].Totals[za0002].Total, err = dc.ReadInt64()
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
+								return
+							}
+						case "N":
+							z[za0001].Totals[za0002].N, err = dc.ReadInt64()
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "N")
+								return
+							}
+						default:
+							err = dc.Skip()
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002)
+								return
+							}
+						}
+					}
+				}
+			case "LastSec":
+				z[za0001].LastSec, err = dc.ReadInt64()
+				if err != nil {
+					err = msgp.WrapError(err, za0001, "LastSec")
+					return
+				}
+			default:
+				err = dc.Skip()
+				if err != nil {
+					err = msgp.WrapError(err, za0001)
+					return
+				}
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(sizeLastElemMarker))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		// map header, size 2
+		// write "Totals"
+		err = en.Append(0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
+		if err != nil {
+			return
+		}
+		err = en.WriteArrayHeader(uint32(60))
+		if err != nil {
+			err = msgp.WrapError(err, za0001, "Totals")
+			return
+		}
+		for za0002 := range z[za0001].Totals {
+			// map header, size 2
+			// write "Total"
+			err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+			if err != nil {
+				return
+			}
+			err = en.WriteInt64(z[za0001].Totals[za0002].Total)
+			if err != nil {
+				err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
+				return
+			}
+			// write "N"
+			err = en.Append(0xa1, 0x4e)
+			if err != nil {
+				return
+			}
+			err = en.WriteInt64(z[za0001].Totals[za0002].N)
+			if err != nil {
+				err = msgp.WrapError(err, za0001, "Totals", za0002, "N")
+				return
+			}
+		}
+		// write "LastSec"
+		err = en.Append(0xa7, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x63)
+		if err != nil {
+			return
+		}
+		err = en.WriteInt64(z[za0001].LastSec)
+		if err != nil {
+			err = msgp.WrapError(err, za0001, "LastSec")
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(sizeLastElemMarker))
+	for za0001 := range z {
+		// map header, size 2
+		// string "Totals"
+		o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
+		o = msgp.AppendArrayHeader(o, uint32(60))
+		for za0002 := range z[za0001].Totals {
+			// map header, size 2
+			// string "Total"
+			o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].Total)
+			// string "N"
+			o = append(o, 0xa1, 0x4e)
+			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].N)
+		}
+		// string "LastSec"
+		o = append(o, 0xa7, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x63)
+		o = msgp.AppendInt64(o, z[za0001].LastSec)
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(sizeLastElemMarker) {
+		err = msgp.ArrayError{Wanted: uint32(sizeLastElemMarker), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		var field []byte
+		_ = field
+		var zb0002 uint32
+		zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+		for zb0002 > 0 {
+			zb0002--
+			field, bts, err = msgp.ReadMapKeyZC(bts)
+			if err != nil {
+				err = msgp.WrapError(err, za0001)
+				return
+			}
+			switch msgp.UnsafeString(field) {
+			case "Totals":
+				var zb0003 uint32
+				zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, za0001, "Totals")
+					return
+				}
+				if zb0003 != uint32(60) {
+					err = msgp.ArrayError{Wanted: uint32(60), Got: zb0003}
+					return
+				}
+				for za0002 := range z[za0001].Totals {
+					var zb0004 uint32
+					zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
+					if err != nil {
+						err = msgp.WrapError(err, za0001, "Totals", za0002)
+						return
+					}
+					for zb0004 > 0 {
+						zb0004--
+						field, bts, err = msgp.ReadMapKeyZC(bts)
+						if err != nil {
+							err = msgp.WrapError(err, za0001, "Totals", za0002)
+							return
+						}
+						switch msgp.UnsafeString(field) {
+						case "Total":
+							z[za0001].Totals[za0002].Total, bts, err = msgp.ReadInt64Bytes(bts)
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
+								return
+							}
+						case "N":
+							z[za0001].Totals[za0002].N, bts, err = msgp.ReadInt64Bytes(bts)
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "N")
+								return
+							}
+						default:
+							bts, err = msgp.Skip(bts)
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002)
+								return
+							}
+						}
+					}
+				}
+			case "LastSec":
+				z[za0001].LastSec, bts, err = msgp.ReadInt64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, za0001, "LastSec")
+					return
+				}
+			default:
+				bts, err = msgp.Skip(bts)
+				if err != nil {
+					err = msgp.WrapError(err, za0001)
+					return
+				}
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *LastMinuteLatencies) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (sizeLastElemMarker * (16 + (60 * (9 + msgp.Int64Size + msgp.Int64Size)) + msgp.Int64Size))
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *lastMinuteLatency) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
 	_ = field
 	var zb0001 uint32
@@ -165,48 +441,36 @@ func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
 			}
 			for za0001 := range z.Totals {
 				var zb0003 uint32
-				zb0003, err = dc.ReadArrayHeader()
+				zb0003, err = dc.ReadMapHeader()
 				if err != nil {
 					err = msgp.WrapError(err, "Totals", za0001)
 					return
 				}
-				if zb0003 != uint32(sizeLastElemMarker) {
-					err = msgp.ArrayError{Wanted: uint32(sizeLastElemMarker), Got: zb0003}
-					return
-				}
-				for za0002 := range z.Totals[za0001] {
-					var zb0004 uint32
-					zb0004, err = dc.ReadMapHeader()
+				for zb0003 > 0 {
+					zb0003--
+					field, err = dc.ReadMapKeyPtr()
 					if err != nil {
-						err = msgp.WrapError(err, "Totals", za0001, za0002)
+						err = msgp.WrapError(err, "Totals", za0001)
 						return
 					}
-					for zb0004 > 0 {
-						zb0004--
-						field, err = dc.ReadMapKeyPtr()
+					switch msgp.UnsafeString(field) {
+					case "Total":
+						z.Totals[za0001].Total, err = dc.ReadInt64()
 						if err != nil {
-							err = msgp.WrapError(err, "Totals", za0001, za0002)
+							err = msgp.WrapError(err, "Totals", za0001, "Total")
 							return
 						}
-						switch msgp.UnsafeString(field) {
-						case "Total":
-							z.Totals[za0001][za0002].Total, err = dc.ReadInt64()
-							if err != nil {
-								err = msgp.WrapError(err, "Totals", za0001, za0002, "Total")
-								return
-							}
-						case "N":
-							z.Totals[za0001][za0002].N, err = dc.ReadInt64()
-							if err != nil {
-								err = msgp.WrapError(err, "Totals", za0001, za0002, "N")
-								return
-							}
-						default:
-							err = dc.Skip()
-							if err != nil {
-								err = msgp.WrapError(err, "Totals", za0001, za0002)
-								return
-							}
+					case "N":
+						z.Totals[za0001].N, err = dc.ReadInt64()
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001, "N")
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001)
+							return
 						}
 					}
 				}
@@ -229,7 +493,7 @@ func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
+func (z *lastMinuteLatency) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 2
 	// write "Totals"
 	err = en.Append(0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
@@ -242,33 +506,26 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 		return
 	}
 	for za0001 := range z.Totals {
-		err = en.WriteArrayHeader(uint32(sizeLastElemMarker))
+		// map header, size 2
+		// write "Total"
+		err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 		if err != nil {
-			err = msgp.WrapError(err, "Totals", za0001)
 			return
 		}
-		for za0002 := range z.Totals[za0001] {
-			// map header, size 2
-			// write "Total"
-			err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
-			if err != nil {
-				return
-			}
-			err = en.WriteInt64(z.Totals[za0001][za0002].Total)
-			if err != nil {
-				err = msgp.WrapError(err, "Totals", za0001, za0002, "Total")
-				return
-			}
-			// write "N"
-			err = en.Append(0xa1, 0x4e)
-			if err != nil {
-				return
-			}
-			err = en.WriteInt64(z.Totals[za0001][za0002].N)
-			if err != nil {
-				err = msgp.WrapError(err, "Totals", za0001, za0002, "N")
-				return
-			}
+		err = en.WriteInt64(z.Totals[za0001].Total)
+		if err != nil {
+			err = msgp.WrapError(err, "Totals", za0001, "Total")
+			return
+		}
+		// write "N"
+		err = en.Append(0xa1, 0x4e)
+		if err != nil {
+			return
+		}
+		err = en.WriteInt64(z.Totals[za0001].N)
+		if err != nil {
+			err = msgp.WrapError(err, "Totals", za0001, "N")
+			return
 		}
 	}
 	// write "LastSec"
@@ -285,23 +542,20 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *lastMinuteLatency) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 2
 	// string "Totals"
 	o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(60))
 	for za0001 := range z.Totals {
-		o = msgp.AppendArrayHeader(o, uint32(sizeLastElemMarker))
-		for za0002 := range z.Totals[za0001] {
-			// map header, size 2
-			// string "Total"
-			o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
-			o = msgp.AppendInt64(o, z.Totals[za0001][za0002].Total)
-			// string "N"
-			o = append(o, 0xa1, 0x4e)
-			o = msgp.AppendInt64(o, z.Totals[za0001][za0002].N)
-		}
+		// map header, size 2
+		// string "Total"
+		o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+		o = msgp.AppendInt64(o, z.Totals[za0001].Total)
+		// string "N"
+		o = append(o, 0xa1, 0x4e)
+		o = msgp.AppendInt64(o, z.Totals[za0001].N)
 	}
 	// string "LastSec"
 	o = append(o, 0xa7, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x63)
@@ -310,7 +564,7 @@ func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
+func (z *lastMinuteLatency) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var field []byte
 	_ = field
 	var zb0001 uint32
@@ -340,48 +594,36 @@ func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			}
 			for za0001 := range z.Totals {
 				var zb0003 uint32
-				zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Totals", za0001)
 					return
 				}
-				if zb0003 != uint32(sizeLastElemMarker) {
-					err = msgp.ArrayError{Wanted: uint32(sizeLastElemMarker), Got: zb0003}
-					return
-				}
-				for za0002 := range z.Totals[za0001] {
-					var zb0004 uint32
-					zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
+				for zb0003 > 0 {
+					zb0003--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
 					if err != nil {
-						err = msgp.WrapError(err, "Totals", za0001, za0002)
+						err = msgp.WrapError(err, "Totals", za0001)
 						return
 					}
-					for zb0004 > 0 {
-						zb0004--
-						field, bts, err = msgp.ReadMapKeyZC(bts)
+					switch msgp.UnsafeString(field) {
+					case "Total":
+						z.Totals[za0001].Total, bts, err = msgp.ReadInt64Bytes(bts)
 						if err != nil {
-							err = msgp.WrapError(err, "Totals", za0001, za0002)
+							err = msgp.WrapError(err, "Totals", za0001, "Total")
 							return
 						}
-						switch msgp.UnsafeString(field) {
-						case "Total":
-							z.Totals[za0001][za0002].Total, bts, err = msgp.ReadInt64Bytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "Totals", za0001, za0002, "Total")
-								return
-							}
-						case "N":
-							z.Totals[za0001][za0002].N, bts, err = msgp.ReadInt64Bytes(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "Totals", za0001, za0002, "N")
-								return
-							}
-						default:
-							bts, err = msgp.Skip(bts)
-							if err != nil {
-								err = msgp.WrapError(err, "Totals", za0001, za0002)
-								return
-							}
+					case "N":
+						z.Totals[za0001].N, bts, err = msgp.ReadInt64Bytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001, "N")
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001)
+							return
 						}
 					}
 				}
@@ -405,7 +647,7 @@ func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *LastMinuteLatencies) Msgsize() (s int) {
-	s = 1 + 7 + msgp.ArrayHeaderSize + (60 * (sizeLastElemMarker * (9 + msgp.Int64Size + msgp.Int64Size))) + 8 + msgp.Int64Size
+func (z *lastMinuteLatency) Msgsize() (s int) {
+	s = 1 + 7 + msgp.ArrayHeaderSize + (60 * (9 + msgp.Int64Size + msgp.Int64Size)) + 8 + msgp.Int64Size
 	return
 }

cmd/last-minute_gen_test.go

@@ -234,3 +234,116 @@ func BenchmarkDecodeLastMinuteLatencies(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshallastMinuteLatency(t *testing.T) {
+	v := lastMinuteLatency{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsglastMinuteLatency(b *testing.B) {
+	v := lastMinuteLatency{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsglastMinuteLatency(b *testing.B) {
+	v := lastMinuteLatency{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshallastMinuteLatency(b *testing.B) {
+	v := lastMinuteLatency{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodelastMinuteLatency(t *testing.T) {
+	v := lastMinuteLatency{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodelastMinuteLatency Msgsize() is inaccurate")
+	}
+
+	vn := lastMinuteLatency{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodelastMinuteLatency(b *testing.B) {
+	v := lastMinuteLatency{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodelastMinuteLatency(b *testing.B) {
+	v := lastMinuteLatency{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/local-locker.go

@@ -322,11 +322,10 @@ func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refresh
 		}
 		idx := 0
 		for {
-			mapID := formatUUID(args.UID, idx)
 			lris, ok := l.lockMap[resource]
 			if !ok {
 				// Inconsistent. Delete UID.
-				delete(l.lockUID, mapID)
+				delete(l.lockUID, formatUUID(args.UID, idx))
 				return idx > 0, nil
 			}
 			for i := range lris {
@@ -335,7 +334,7 @@ func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refresh
 				}
 			}
 			idx++
-			resource, ok = l.lockUID[mapID]
+			resource, ok = l.lockUID[formatUUID(args.UID, idx)]
 			if !ok {
 				// No more resources for UID, but we did update at least one.
 				return true, nil
@@ -350,25 +349,29 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 
-	for k := range l.lockMap {
-		found := false
-		// Since we mutate the value, remove one per loop.
-		for {
-			lris, ok := l.lockMap[k]
-			if !ok {
-				break
-			}
-			for _, lri := range lris {
-				if time.Since(lri.TimeLastRefresh) > interval {
-					l.removeEntry(lri.Name, dsync.LockArgs{Owner: lri.Owner, UID: lri.UID}, &lris)
-					found = true
+	for k, lris := range l.lockMap {
+		modified := false
+		for i := 0; i < len(lris); {
+			lri := &lris[i]
+			if time.Since(lri.TimeLastRefresh) > interval {
+				delete(l.lockUID, formatUUID(lri.UID, lri.idx))
+				if len(lris) == 1 {
+					// Remove the write lock.
+					delete(l.lockMap, lri.Name)
+					modified = false
 					break
 				}
+				modified = true
+				// Remove the appropriate lock.
+				lris = append(lris[:i], lris[i+1:]...)
+				// Check same i
+			} else {
+				// Move to next
+				i++
 			}
-			// We did not find any more to expire.
-			if !found {
-				break
-			}
+		}
+		if modified {
+			l.lockMap[k] = lris
 		}
 	}
 }

cmd/local-locker_test.go

@@ -19,9 +19,13 @@ package cmd
 
 import (
 	"context"
+	"encoding/hex"
+	"fmt"
+	"math/rand"
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/minio/minio/internal/dsync"
 )
 
@@ -254,3 +258,89 @@ func TestLocalLockerUnlock(t *testing.T) {
 		t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), 0, 0)
 	}
 }
+
+func Test_localLocker_expireOldLocksExpire(t *testing.T) {
+	rng := rand.New(rand.NewSource(0))
+	// Numbers of unique locks
+	for _, locks := range []int{100, 1000, 1e6} {
+		if testing.Short() && locks > 100 {
+			continue
+		}
+		t.Run(fmt.Sprintf("%d-locks", locks), func(t *testing.T) {
+			// Number of readers per lock...
+			for _, readers := range []int{1, 10, 100} {
+				if locks > 1000 && readers > 1 {
+					continue
+				}
+				if testing.Short() && readers > 10 {
+					continue
+				}
+				t.Run(fmt.Sprintf("%d-read", readers), func(t *testing.T) {
+					l := newLocker()
+					for i := 0; i < locks; i++ {
+						var tmp [16]byte
+						rng.Read(tmp[:])
+						res := []string{hex.EncodeToString(tmp[:])}
+
+						for i := 0; i < readers; i++ {
+							rng.Read(tmp[:])
+							ok, err := l.RLock(context.Background(), dsync.LockArgs{
+								UID:       uuid.NewString(),
+								Resources: res,
+								Source:    hex.EncodeToString(tmp[:8]),
+								Owner:     hex.EncodeToString(tmp[8:]),
+								Quorum:    0,
+							})
+							if !ok || err != nil {
+								t.Fatal("failed:", err, ok)
+							}
+						}
+					}
+					start := time.Now()
+					l.expireOldLocks(time.Hour)
+					t.Logf("Scan Took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+					if len(l.lockMap) != locks {
+						t.Fatalf("objects deleted, want %d != got %d", locks, len(l.lockMap))
+					}
+					if len(l.lockUID) != locks*readers {
+						t.Fatalf("objects deleted, want %d != got %d", locks*readers, len(l.lockUID))
+					}
+
+					// Expire 50%
+					expired := time.Now().Add(-time.Hour * 2)
+					for _, v := range l.lockMap {
+						for i := range v {
+							if rng.Intn(2) == 0 {
+								v[i].TimeLastRefresh = expired
+							}
+						}
+					}
+					start = time.Now()
+					l.expireOldLocks(time.Hour)
+					t.Logf("Expire 50%% took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+
+					if len(l.lockUID) == locks*readers {
+						t.Fatalf("objects uids all remain, unlikely")
+					}
+					if len(l.lockMap) == 0 {
+						t.Fatalf("objects all deleted, 0 remains")
+					}
+					if len(l.lockUID) == 0 {
+						t.Fatalf("objects uids all deleted, 0 remains")
+					}
+
+					start = time.Now()
+					l.expireOldLocks(-time.Minute)
+					t.Logf("Expire rest took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+
+					if len(l.lockMap) != 0 {
+						t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockMap))
+					}
+					if len(l.lockUID) != 0 {
+						t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockUID))
+					}
+				})
+			}
+		})
+	}
+}

cmd/metacache-set.go

@@ -542,13 +542,9 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 	var fallbackDisks []StorageAPI
 
 	// Special case: ask all disks if the drive count is 4
-	if askDisks == -1 || er.setDriveCount == 4 {
-		askDisks = len(disks) // with 'strict' quorum list on all online disks.
-		listingQuorum = er.defaultRQuorum()
-	}
-	if askDisks == 0 {
-		askDisks = globalAPIConfig.getListQuorum()
-		listingQuorum = askDisks
+	if askDisks <= 0 || er.setDriveCount == 4 {
+		askDisks = len(disks)          // with 'strict' quorum list on all online disks.
+		listingQuorum = len(disks) / 2 // keep this such that we can list all objects with different quorum ratio.
 	}
 	if askDisks > 0 && len(disks) > askDisks {
 		rand.Shuffle(len(disks), func(i, j int) {

cmd/metrics-v2.go

@@ -70,6 +70,7 @@ func init() {
 
 	nodeCollector = newMinioCollectorNode([]*MetricsGroup{
 		getNodeHealthMetrics(),
+		getLocalDiskStorageMetrics(),
 		getCacheMetrics(),
 		getHTTPMetrics(),
 		getNetworkMetrics(),
@@ -157,6 +158,8 @@ const (
 	writeBytes      MetricName = "write_bytes"
 	wcharBytes      MetricName = "wchar_bytes"
 
+	apiLatencyMicroSec MetricName = "latency_us"
+
 	usagePercent MetricName = "update_percent"
 
 	commitInfo  MetricName = "commit_info"
@@ -226,10 +229,31 @@ func (g *MetricsGroup) RegisterRead(read func(ctx context.Context) []Metric) {
 	})
 }
 
+func (m *Metric) copyMetric() Metric {
+	metric := Metric{
+		Description:          m.Description,
+		Value:                m.Value,
+		HistogramBucketLabel: m.HistogramBucketLabel,
+		StaticLabels:         make(map[string]string),
+		VariableLabels:       make(map[string]string),
+		Histogram:            make(map[string]uint64),
+	}
+	for k, v := range m.StaticLabels {
+		metric.StaticLabels[k] = v
+	}
+	for k, v := range m.VariableLabels {
+		metric.VariableLabels[k] = v
+	}
+	for k, v := range m.Histogram {
+		metric.Histogram[k] = v
+	}
+	return metric
+}
+
 // Get - returns cached value always upton the configured TTL,
 // once the TTL expires "read()" registered function is called
 // to return the new values and updated.
-func (g *MetricsGroup) Get() []Metric {
+func (g *MetricsGroup) Get() (metrics []Metric) {
 	var c interface{}
 	var err error
 	if g.cacheInterval <= 0 {
@@ -247,7 +271,11 @@ func (g *MetricsGroup) Get() []Metric {
 		return []Metric{}
 	}
 
-	return m
+	metrics = make([]Metric, 0, len(m))
+	for i := range m {
+		metrics = append(metrics, m[i].copyMetric())
+	}
+	return metrics
 }
 
 func getClusterCapacityTotalBytesMD() MetricDescription {
@@ -290,6 +318,16 @@ func getClusterCapacityUsageFreeBytesMD() MetricDescription {
 	}
 }
 
+func getNodeDiskAPILatencyMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: diskSubsystem,
+		Name:      apiLatencyMicroSec,
+		Help:      "Average last minute latency in µs for disk API storage operations.",
+		Type:      gaugeMetric,
+	}
+}
+
 func getNodeDiskUsedBytesMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
@@ -1558,6 +1596,35 @@ func getLocalStorageMetrics() *MetricsGroup {
 				Value:          float64(disk.FreeInodes),
 				VariableLabels: map[string]string{"disk": disk.DrivePath},
 			})
+
+		}
+		return
+	})
+	return mg
+}
+
+func getLocalDiskStorageMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 3 * time.Second,
+	}
+	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
+		objLayer := newObjectLayerFn()
+		// Service not initialized yet
+		if objLayer == nil || globalIsGateway {
+			return
+		}
+
+		metrics = make([]Metric, 0, 50)
+		storageInfo, _ := objLayer.LocalStorageInfo(ctx)
+		for _, disk := range storageInfo.Disks {
+			for apiName, latency := range disk.Metrics.APILatencies {
+				val := latency.(uint64)
+				metrics = append(metrics, Metric{
+					Description:    getNodeDiskAPILatencyMD(),
+					Value:          float64(val / 1000),
+					VariableLabels: map[string]string{"disk": disk.DrivePath, "api": "storage." + apiName},
+				})
+			}
 		}
 		return
 	})

cmd/metrics.go

@@ -674,7 +674,7 @@ func metricsHandler() http.Handler {
 // AuthMiddleware checks if the bearer token is valid and authorized.
 func AuthMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		claims, owner, authErr := webRequestAuthenticate(r)
+		claims, groups, owner, authErr := webRequestAuthenticate(r)
 		if authErr != nil || !claims.VerifyIssuer("prometheus", true) {
 			w.WriteHeader(http.StatusForbidden)
 			return
@@ -682,6 +682,7 @@ func AuthMiddleware(h http.Handler) http.Handler {
 		// For authenticated users apply IAM policy.
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     claims.AccessKey,
+			Groups:          groups,
 			Action:          iampolicy.PrometheusAdminAction,
 			ConditionValues: getConditionValues(r, "", claims.AccessKey, claims.Map()),
 			IsOwner:         owner,

cmd/net.go

@@ -102,15 +102,14 @@ func mustGetLocalIP6() (ipList set.StringSet) {
 
 // getHostIP returns IP address of given host.
 func getHostIP(host string) (ipList set.StringSet, err error) {
-	var ips []net.IP
-
-	if ips, err = net.LookupIP(host); err != nil {
+	addrs, err := globalDNSCache.LookupHost(GlobalContext, host)
+	if err != nil {
 		return ipList, err
 	}
 
 	ipList = set.NewStringSet()
-	for _, ip := range ips {
-		ipList.Add(ip.String())
+	for _, addr := range addrs {
+		ipList.Add(addr)
 	}
 
 	return ipList, err

cmd/notification.go

@@ -616,6 +616,26 @@ func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketNam
 	return bucketStats
 }
 
+// ReloadPoolMeta reloads on disk updates on pool metadata
+func (sys *NotificationSys) ReloadPoolMeta(ctx context.Context) {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(ctx, func() error {
+			return client.ReloadPoolMeta(ctx)
+		}, idx, *client.host)
+	}
+	for _, nErr := range ng.Wait() {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
+		if nErr.Err != nil {
+			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+		}
+	}
+}
+
 // LoadTransitionTierConfig notifies remote peers to load their remote tier
 // configs from config store.
 func (sys *NotificationSys) LoadTransitionTierConfig(ctx context.Context) {
@@ -1610,3 +1630,31 @@ func (sys *NotificationSys) ReloadSiteReplicationConfig(ctx context.Context) []e
 	wg.Wait()
 	return errs
 }
+
+// GetLastDayTierStats fetches per-tier stats of the last 24hrs from all peers
+func (sys *NotificationSys) GetLastDayTierStats(ctx context.Context) dailyAllTierStats {
+	errs := make([]error, len(sys.allPeerClients))
+	lastDayStats := make([]dailyAllTierStats, len(sys.allPeerClients))
+	var wg sync.WaitGroup
+	for index := range sys.peerClients {
+		if sys.peerClients[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			lastDayStats[index], errs[index] = sys.peerClients[index].GetLastDayTierStats(ctx)
+		}(index)
+	}
+
+	wg.Wait()
+	merged := globalTransitionState.getDailyAllTierStats()
+	for i, stat := range lastDayStats {
+		if errs[i] != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to fetch last day tier stats: %w", errs[i]))
+			continue
+		}
+		merged.merge(stat)
+	}
+	return merged
+}

cmd/object-api-common.go

@@ -164,6 +164,8 @@ func listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter
 		result.IsTruncated = true
 		if len(objInfos) > 0 {
 			result.NextMarker = objInfos[len(objInfos)-1].Name
+		} else if len(result.Prefixes) > 0 {
+			result.NextMarker = result.Prefixes[len(result.Prefixes)-1]
 		}
 	}
 
@@ -399,6 +401,8 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 		result.IsTruncated = true
 		if len(objInfos) > 0 {
 			result.NextMarker = objInfos[len(objInfos)-1].Name
+		} else if len(result.Prefixes) > 0 {
+			result.NextMarker = result.Prefixes[len(result.Prefixes)-1]
 		}
 	}
 

cmd/object-api-errors.go

@@ -159,10 +159,15 @@ func toObjectErr(err error, params ...string) error {
 			apiErr.Object = decodeDirObject(params[1])
 		}
 		return apiErr
-	case io.ErrUnexpectedEOF.Error(), io.ErrShortWrite.Error():
-		return IncompleteBody{}
-	case context.Canceled.Error(), context.DeadlineExceeded.Error():
-		return IncompleteBody{}
+	case io.ErrUnexpectedEOF.Error(), io.ErrShortWrite.Error(), context.Canceled.Error(), context.DeadlineExceeded.Error():
+		apiErr := IncompleteBody{}
+		if len(params) >= 1 {
+			apiErr.Bucket = params[0]
+		}
+		if len(params) >= 2 {
+			apiErr.Object = decodeDirObject(params[1])
+		}
+		return apiErr
 	}
 	return err
 }

cmd/object-api-interface.go

@@ -71,6 +71,9 @@ type ObjectOptions struct {
 
 	// Use the maximum parity (N/2), used when saving server configuration files
 	MaxParity bool
+
+	// Mutate set to 'true' if the call is namespace mutation call
+	Mutate bool
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.

cmd/object-api-utils.go

@@ -61,10 +61,7 @@ const (
 	// MinIO tmp meta prefix.
 	minioMetaTmpBucket = minioMetaBucket + "/tmp"
 	// MinIO tmp meta prefix for deleted objects.
-	minioMetaTmpDeletedBucket      = minioMetaTmpBucket + "/.trash"
-	minioMetaSpeedTestBucket       = minioMetaBucket + "/speedtest"
-	minioMetaSpeedTestBucketPrefix = "objects/"
-
+	minioMetaTmpDeletedBucket = minioMetaTmpBucket + "/.trash"
 	// DNS separator (period), used for bucket name validation.
 	dnsDelimiter = "."
 	// On compressed files bigger than this;

cmd/peer-rest-client.go

@@ -422,12 +422,21 @@ func (client *peerRESTClient) GetSELinuxInfo(ctx context.Context) (info madmin.S
 
 // GetSysConfig - fetch sys config for a remote node.
 func (client *peerRESTClient) GetSysConfig(ctx context.Context) (info madmin.SysConfig, err error) {
+	sent := time.Now()
 	respBody, err := client.callWithContext(ctx, peerRESTMethodSysConfig, nil, nil, -1)
 	if err != nil {
 		return
 	}
+	roundtrip := int32(time.Since(sent).Milliseconds())
 	defer http.DrainBody(respBody)
+
 	err = gob.NewDecoder(respBody).Decode(&info)
+	cfg := info.Config["time-info"]
+	if cfg != nil {
+		ti := cfg.(madmin.TimeInfo)
+		ti.RoundtripDuration = roundtrip
+		info.Config["time-info"] = ti
+	}
 	return info, err
 }
 
@@ -758,6 +767,16 @@ func (client *peerRESTClient) UpdateMetacacheListing(ctx context.Context, m meta
 	return resp, msgp.Decode(respBody, &resp)
 }
 
+func (client *peerRESTClient) ReloadPoolMeta(ctx context.Context) error {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodReloadPoolMeta, nil, nil, 0)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
 func (client *peerRESTClient) LoadTransitionTierConfig(ctx context.Context) error {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodLoadTransitionTierConfig, nil, nil, 0)
 	if err != nil {
@@ -1056,3 +1075,18 @@ func (client *peerRESTClient) ReloadSiteReplicationConfig(ctx context.Context) e
 	defer http.DrainBody(respBody)
 	return nil
 }
+
+func (client *peerRESTClient) GetLastDayTierStats(ctx context.Context) (dailyAllTierStats, error) {
+	var result map[string]lastDayTierStats
+	respBody, err := client.callWithContext(context.Background(), peerRESTMethodGetLastDayTierStats, nil, nil, -1)
+	if err != nil {
+		return result, err
+	}
+	defer http.DrainBody(respBody)
+
+	err = gob.NewDecoder(respBody).Decode(&result)
+	if err != nil {
+		return dailyAllTierStats{}, err
+	}
+	return dailyAllTierStats(result), nil
+}

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v17" // Add "storage-class" option for SpeedTest
+	peerRESTVersion       = "v19" // Add getlastdaytierstats
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -67,6 +67,8 @@ const (
 	peerRESTMethodLoadTransitionTierConfig    = "/loadtransitiontierconfig"
 	peerRESTMethodSpeedtest                   = "/speedtest"
 	peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
+	peerRESTMethodReloadPoolMeta              = "/reloadpoolmeta"
+	peerRESTMethodGetLastDayTierStats         = "/getlastdaytierstats"
 )
 
 const (

cmd/peer-rest-server.go

@@ -1023,6 +1023,27 @@ func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *h
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(state))
 }
 
+func (s *peerRESTServer) ReloadPoolMetaHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	pools, ok := objAPI.(*erasureServerPools)
+	if !ok {
+		return
+	}
+	if err := pools.ReloadPoolMeta(r.Context()); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+}
+
 func (s *peerRESTServer) LoadTransitionTierConfigHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
@@ -1147,7 +1168,6 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 	var totalBytesWritten uint64
 	var totalBytesRead uint64
 
-	bucket := minioMetaSpeedTestBucket
 	objCountPerThread := make([]uint64, concurrent)
 	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
 	defer uploadsCancel()
@@ -1157,7 +1177,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 		uploadsCancel()
 	}()
 
-	objNamePrefix := minioMetaSpeedTestBucketPrefix + uuid.New().String()
+	objNamePrefix := "speedtest/objects/" + uuid.New().String()
 
 	wg.Add(concurrent)
 	for i := 0; i < concurrent; i++ {
@@ -1177,7 +1197,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 					return
 				}
 				reader := NewPutObjReader(hashReader)
-				objInfo, err := objAPI.PutObject(uploadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
+				objInfo, err := objAPI.PutObject(uploadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
 					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{
 					UserDefined: map[string]string{
 						xhttp.AmzStorageClass: storageClass,
@@ -1224,7 +1244,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 				if objCountPerThread[i] == j {
 					j = 0
 				}
-				r, err := objAPI.GetObjectNInfo(downloadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
+				r, err := objAPI.GetObjectNInfo(downloadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
 					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
 				if err != nil {
 					if !contextCanceled(downloadsCtx) {
@@ -1306,6 +1326,23 @@ func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request
 	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }
 
+// GetLastDayTierStatsHandler - returns per-tier stats in the last 24hrs for this server
+func (s *peerRESTServer) GetLastDayTierStatsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "GetLastDayTierStats")
+	if objAPI := newObjectLayerFn(); objAPI == nil || globalTransitionState == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	result := globalTransitionState.getDailyAllTierStats()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(result))
+}
+
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
 	server := &peerRESTServer{}
@@ -1353,4 +1390,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(httpTraceHdrs(server.GetLastDayTierStatsHandler))
 }

cmd/policy_test.go

@@ -31,14 +31,14 @@ func TestPolicySysIsAllowed(t *testing.T) {
 	p := &policy.Policy{
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetBucketLocationAction),
 				policy.NewResourceSet(policy.NewResource("mybucket", "")),
 				condition.NewFunctions(),
 			),
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.PutObjectAction),
@@ -164,14 +164,14 @@ func TestPolicyToBucketAccessPolicy(t *testing.T) {
 	case1Policy := &policy.Policy{
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
 				policy.NewResourceSet(policy.NewResource("mybucket", "")),
 				condition.NewFunctions(),
 			),
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetObjectAction),
@@ -199,7 +199,7 @@ func TestPolicyToBucketAccessPolicy(t *testing.T) {
 	case3Policy := &policy.Policy{
 		Version: "12-10-2012",
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.PutObjectAction),
@@ -244,14 +244,14 @@ func TestBucketAccessPolicyToPolicy(t *testing.T) {
 	case1Result := &policy.Policy{
 		Version: policy.DefaultVersion,
 		Statements: []policy.Statement{
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
 				policy.NewResourceSet(policy.NewResource("mybucket", "")),
 				condition.NewFunctions(),
 			),
-			policy.NewStatement(
+			policy.NewStatement("",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetObjectAction),

cmd/prepare-storage.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -70,7 +71,7 @@ var printEndpointError = func() func(Endpoint, error, bool) {
 }()
 
 // Cleans up tmp directory of the local disk.
-func formatErasureCleanupTmp(diskPath string) error {
+func formatErasureCleanupTmp(diskPath string) {
 	// Need to move temporary objects left behind from previous run of minio
 	// server to a unique directory under `minioMetaTmpBucket-old` to clean
 	// up `minioMetaTmpBucket` for the current run.
@@ -81,9 +82,23 @@ func formatErasureCleanupTmp(diskPath string) error {
 	//
 	// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
 	// temporary objects from one of the previous runs of minio server.
-	tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", mustGetUUID())
+	tmpID := mustGetUUID()
+	tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", tmpID)
 	if err := renameAll(pathJoin(diskPath, minioMetaTmpBucket),
-		tmpOld); err != nil && err != errFileNotFound {
+		tmpOld); err != nil && !errors.Is(err, errFileNotFound) {
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
+			pathJoin(diskPath, minioMetaTmpBucket),
+			tmpOld,
+			osErrToFileErr(err)))
+	}
+
+	if err := mkdirAll(pathJoin(diskPath, minioMetaTmpDeletedBucket), 0o777); err != nil {
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate",
+			pathJoin(diskPath, minioMetaTmpBucket),
+			err))
+	}
+
+	if err := renameAll(tmpOld, pathJoin(diskPath, minioMetaTmpDeletedBucket, tmpID)); err != nil && !errors.Is(err, errFileNotFound) {
 		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
 			pathJoin(diskPath, minioMetaTmpBucket),
 			tmpOld,
@@ -92,16 +107,6 @@ func formatErasureCleanupTmp(diskPath string) error {
 
 	// Renames and schedules for purging all bucket metacache.
 	renameAllBucketMetacache(diskPath)
-
-	// Removal of tmp-old folder is backgrounded completely.
-	go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old"))
-
-	if err := mkdirAll(pathJoin(diskPath, minioMetaTmpDeletedBucket), 0o777); err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty please investigate",
-			pathJoin(diskPath, minioMetaTmpBucket),
-			err))
-	}
-	return nil
 }
 
 // Following error message is added to fix a regression in release
@@ -178,13 +183,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		}
 	}(storageDisks)
 
-	// Sanitize all local disks during server startup.
-	for _, disk := range storageDisks {
-		if disk != nil && disk.IsLocal() {
-			disk.(*xlStorageDiskIDCheck).storage.(*xlStorage).Sanitize()
-		}
-	}
-
 	for i, err := range errs {
 		if err != nil {
 			if err == errDiskNotFound && retryCount >= 5 {
@@ -239,13 +237,14 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 
 	// Return error when quorum unformatted disks - indicating we are
 	// waiting for first server to be online.
-	if quorumUnformattedDisks(sErrs) && !firstDisk {
+	unformattedDisks := quorumUnformattedDisks(sErrs)
+	if unformattedDisks && !firstDisk {
 		return nil, nil, errNotFirstDisk
 	}
 
 	// Return error when quorum unformatted disks but waiting for rest
 	// of the servers to be online.
-	if quorumUnformattedDisks(sErrs) && firstDisk {
+	if unformattedDisks && firstDisk {
 		return nil, nil, errFirstDiskWait
 	}
 
@@ -291,10 +290,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		return nil, nil, err
 	}
 
-	// The will always recreate some directories inside .minio.sys of
-	// the local disk such as tmp, multipart and background-ops
-	initErasureMetaVolumesInLocalDisks(storageDisks, formatConfigs)
-
 	return storageDisks, format, nil
 }
 
@@ -311,10 +306,18 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCou
 		return time.Now().Round(time.Second).Sub(formatStartTime).String()
 	}
 
-	// Wait on each try for an update.
-	ticker := time.NewTicker(500 * time.Millisecond)
-	defer ticker.Stop()
 	var tries int
+	storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID, distributionAlgo)
+	if err == nil {
+		return storageDisks, format, nil
+	}
+
+	tries++ // tried already once
+
+	// Wait on each try for an update.
+	ticker := time.NewTicker(250 * time.Millisecond)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-ticker.C:

cmd/server-main.go

@@ -51,23 +51,27 @@ import (
 // ServerFlags - server command specific flags
 var ServerFlags = []cli.Flag{
 	cli.StringFlag{
-		Name:  "address",
-		Value: ":" + GlobalMinioDefaultPort,
-		Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
+		Name:   "address",
+		Value:  ":" + GlobalMinioDefaultPort,
+		Usage:  "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
+		EnvVar: "MINIO_ADDRESS",
 	},
 	cli.IntFlag{
-		Name:  "listeners",
-		Value: 1,
-		Usage: "bind N number of listeners per ADDRESS:PORT",
+		Name:   "listeners",
+		Value:  1,
+		Usage:  "bind N number of listeners per ADDRESS:PORT",
+		EnvVar: "MINIO_LISTENERS",
 	},
 	cli.StringFlag{
-		Name:  "console-address",
-		Usage: "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname",
+		Name:   "console-address",
+		Usage:  "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname",
+		EnvVar: "MINIO_CONSOLE_ADDRESS",
 	},
 	cli.DurationFlag{
 		Name:   "shutdown-timeout",
 		Value:  xhttp.DefaultShutdownTimeout,
 		Usage:  "shutdown timeout to gracefully shutdown server",
+		EnvVar: "MINIO_SHUTDOWN_TIMEOUT",
 		Hidden: true,
 	},
 }
@@ -181,15 +185,17 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 
 	// allow transport to be HTTP/1.1 for proxying.
 	globalProxyTransport = newCustomHTTPProxyTransport(&tls.Config{
-		RootCAs:          globalRootCAs,
-		CipherSuites:     fips.CipherSuitesTLS(),
-		CurvePreferences: fips.EllipticCurvesTLS(),
+		RootCAs:            globalRootCAs,
+		CipherSuites:       fips.CipherSuitesTLS(),
+		CurvePreferences:   fips.EllipticCurvesTLS(),
+		ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize),
 	}, rest.DefaultTimeout)()
 	globalProxyEndpoints = GetProxyEndpoints(globalEndpoints)
 	globalInternodeTransport = newInternodeHTTPTransport(&tls.Config{
-		RootCAs:          globalRootCAs,
-		CipherSuites:     fips.CipherSuitesTLS(),
-		CurvePreferences: fips.EllipticCurvesTLS(),
+		RootCAs:            globalRootCAs,
+		CipherSuites:       fips.CipherSuitesTLS(),
+		CurvePreferences:   fips.EllipticCurvesTLS(),
+		ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize),
 	}, rest.DefaultTimeout)()
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
@@ -289,6 +295,7 @@ func configRetriableErrors(err error) bool {
 		errors.Is(err, io.ErrUnexpectedEOF) ||
 		errors.As(err, &rquorum) ||
 		errors.As(err, &wquorum) ||
+		isErrObjectNotFound(err) ||
 		isErrBucketNotFound(err) ||
 		errors.Is(err, os.ErrDeadlineExceeded)
 }
@@ -573,22 +580,23 @@ func serverMain(ctx *cli.Context) {
 	// Initialize users credentials and policies in background right after config has initialized.
 	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
 
-	// Initialize transition tier configuration manager
-	if globalIsErasure {
-		if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
-			logger.LogIf(GlobalContext, err)
-		}
-	}
-
 	initDataScanner(GlobalContext, newObject)
 
-	if globalIsErasure { // to be done after config init
+	// Initialize transition tier configuration manager
+	if globalIsErasure {
 		initBackgroundReplication(GlobalContext, newObject)
 		initBackgroundTransition(GlobalContext, newObject)
-		globalTierJournal, err = initTierDeletionJournal(GlobalContext)
-		if err != nil {
-			logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
-		}
+
+		go func() {
+			if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil {
+				logger.LogIf(GlobalContext, err)
+			}
+
+			globalTierJournal, err = initTierDeletionJournal(GlobalContext)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
+			}
+		}()
 	}
 
 	// initialize the new disk cache objects.
@@ -602,7 +610,7 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	// Prints the formatted startup message, if err is not nil then it prints additional information as well.
-	printStartupMessage(getAPIEndpoints(), err)
+	go printStartupMessage(getAPIEndpoints(), err)
 
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
 		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", globalActiveCred)
@@ -629,7 +637,17 @@ func serverMain(ctx *cli.Context) {
 	if serverDebugLog {
 		logger.Info("== DEBUG Mode enabled ==")
 		logger.Info("Currently set environment settings:")
+		ks := []string{
+			config.EnvAccessKey,
+			config.EnvSecretKey,
+			config.EnvRootUser,
+			config.EnvRootPassword,
+		}
 		for _, v := range os.Environ() {
+			// Do not print sensitive creds in debug.
+			if contains(ks, strings.Split(v, "=")[0]) {
+				continue
+			}
 			logger.Info(v)
 		}
 		logger.Info("======")

cmd/storage-datatypes.go

@@ -47,7 +47,7 @@ type DiskInfo struct {
 // the number of calls of each API and the moving average of
 // the duration of each API.
 type DiskMetrics struct {
-	APILatencies map[string]string `json:"apiLatencies,omitempty"`
+	APILatencies map[string]uint64 `json:"apiLatencies,omitempty"`
 	APICalls     map[string]uint64 `json:"apiCalls,omitempty"`
 }
 
@@ -72,11 +72,12 @@ type FilesInfo struct {
 	IsTruncated bool
 }
 
-// FilesInfoVersions represents a list of file versions,
-// additionally indicates if the list is last.
-type FilesInfoVersions struct {
-	FilesVersions []FileInfoVersions
-	IsTruncated   bool
+// Size returns size of all versions for the object 'Name'
+func (f FileInfoVersions) Size() (size int64) {
+	for _, v := range f.Versions {
+		size += v.Size
+	}
+	return size
 }
 
 // FileInfoVersions represent a list of versions for a given file.
@@ -235,7 +236,9 @@ func (fi *FileInfo) SetInlineData() {
 }
 
 // VersionPurgeStatusKey denotes purge status in metadata
-const VersionPurgeStatusKey = "purgestatus"
+const (
+	VersionPurgeStatusKey = ReservedMetadataPrefixLower + "purgestatus"
+)
 
 // newFileInfo - initializes new FileInfo, allocates a fresh erasure info.
 func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {

cmd/storage-datatypes_gen.go

@@ -299,7 +299,7 @@ func (z *DiskMetrics) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 			if z.APILatencies == nil {
-				z.APILatencies = make(map[string]string, zb0002)
+				z.APILatencies = make(map[string]uint64, zb0002)
 			} else if len(z.APILatencies) > 0 {
 				for key := range z.APILatencies {
 					delete(z.APILatencies, key)
@@ -308,13 +308,13 @@ func (z *DiskMetrics) DecodeMsg(dc *msgp.Reader) (err error) {
 			for zb0002 > 0 {
 				zb0002--
 				var za0001 string
-				var za0002 string
+				var za0002 uint64
 				za0001, err = dc.ReadString()
 				if err != nil {
 					err = msgp.WrapError(err, "APILatencies")
 					return
 				}
-				za0002, err = dc.ReadString()
+				za0002, err = dc.ReadUint64()
 				if err != nil {
 					err = msgp.WrapError(err, "APILatencies", za0001)
 					return
@@ -381,7 +381,7 @@ func (z *DiskMetrics) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "APILatencies")
 			return
 		}
-		err = en.WriteString(za0002)
+		err = en.WriteUint64(za0002)
 		if err != nil {
 			err = msgp.WrapError(err, "APILatencies", za0001)
 			return
@@ -421,7 +421,7 @@ func (z *DiskMetrics) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendMapHeader(o, uint32(len(z.APILatencies)))
 	for za0001, za0002 := range z.APILatencies {
 		o = msgp.AppendString(o, za0001)
-		o = msgp.AppendString(o, za0002)
+		o = msgp.AppendUint64(o, za0002)
 	}
 	// string "APICalls"
 	o = append(o, 0xa8, 0x41, 0x50, 0x49, 0x43, 0x61, 0x6c, 0x6c, 0x73)
@@ -459,7 +459,7 @@ func (z *DiskMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 			if z.APILatencies == nil {
-				z.APILatencies = make(map[string]string, zb0002)
+				z.APILatencies = make(map[string]uint64, zb0002)
 			} else if len(z.APILatencies) > 0 {
 				for key := range z.APILatencies {
 					delete(z.APILatencies, key)
@@ -467,14 +467,14 @@ func (z *DiskMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			}
 			for zb0002 > 0 {
 				var za0001 string
-				var za0002 string
+				var za0002 uint64
 				zb0002--
 				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "APILatencies")
 					return
 				}
-				za0002, bts, err = msgp.ReadStringBytes(bts)
+				za0002, bts, err = msgp.ReadUint64Bytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "APILatencies", za0001)
 					return
@@ -529,7 +529,7 @@ func (z *DiskMetrics) Msgsize() (s int) {
 	if z.APILatencies != nil {
 		for za0001, za0002 := range z.APILatencies {
 			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002)
+			s += msgp.StringPrefixSize + len(za0001) + msgp.Uint64Size
 		}
 	}
 	s += 9 + msgp.MapHeaderSize
@@ -1530,178 +1530,6 @@ func (z *FilesInfo) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *FilesInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "FilesVersions":
-			var zb0002 uint32
-			zb0002, err = dc.ReadArrayHeader()
-			if err != nil {
-				err = msgp.WrapError(err, "FilesVersions")
-				return
-			}
-			if cap(z.FilesVersions) >= int(zb0002) {
-				z.FilesVersions = (z.FilesVersions)[:zb0002]
-			} else {
-				z.FilesVersions = make([]FileInfoVersions, zb0002)
-			}
-			for za0001 := range z.FilesVersions {
-				err = z.FilesVersions[za0001].DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, "FilesVersions", za0001)
-					return
-				}
-			}
-		case "IsTruncated":
-			z.IsTruncated, err = dc.ReadBool()
-			if err != nil {
-				err = msgp.WrapError(err, "IsTruncated")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z *FilesInfoVersions) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
-	// write "FilesVersions"
-	err = en.Append(0x82, 0xad, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
-	if err != nil {
-		return
-	}
-	err = en.WriteArrayHeader(uint32(len(z.FilesVersions)))
-	if err != nil {
-		err = msgp.WrapError(err, "FilesVersions")
-		return
-	}
-	for za0001 := range z.FilesVersions {
-		err = z.FilesVersions[za0001].EncodeMsg(en)
-		if err != nil {
-			err = msgp.WrapError(err, "FilesVersions", za0001)
-			return
-		}
-	}
-	// write "IsTruncated"
-	err = en.Append(0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.IsTruncated)
-	if err != nil {
-		err = msgp.WrapError(err, "IsTruncated")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z *FilesInfoVersions) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
-	// string "FilesVersions"
-	o = append(o, 0x82, 0xad, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
-	o = msgp.AppendArrayHeader(o, uint32(len(z.FilesVersions)))
-	for za0001 := range z.FilesVersions {
-		o, err = z.FilesVersions[za0001].MarshalMsg(o)
-		if err != nil {
-			err = msgp.WrapError(err, "FilesVersions", za0001)
-			return
-		}
-	}
-	// string "IsTruncated"
-	o = append(o, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64)
-	o = msgp.AppendBool(o, z.IsTruncated)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *FilesInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "FilesVersions":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "FilesVersions")
-				return
-			}
-			if cap(z.FilesVersions) >= int(zb0002) {
-				z.FilesVersions = (z.FilesVersions)[:zb0002]
-			} else {
-				z.FilesVersions = make([]FileInfoVersions, zb0002)
-			}
-			for za0001 := range z.FilesVersions {
-				bts, err = z.FilesVersions[za0001].UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "FilesVersions", za0001)
-					return
-				}
-			}
-		case "IsTruncated":
-			z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "IsTruncated")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *FilesInfoVersions) Msgsize() (s int) {
-	s = 1 + 14 + msgp.ArrayHeaderSize
-	for za0001 := range z.FilesVersions {
-		s += z.FilesVersions[za0001].Msgsize()
-	}
-	s += 12 + msgp.BoolSize
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *VolInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 	var zb0001 uint32

cmd/storage-datatypes_gen_test.go

@@ -574,119 +574,6 @@ func BenchmarkDecodeFilesInfo(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalFilesInfoVersions(t *testing.T) {
-	v := FilesInfoVersions{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeFilesInfoVersions(t *testing.T) {
-	v := FilesInfoVersions{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeFilesInfoVersions Msgsize() is inaccurate")
-	}
-
-	vn := FilesInfoVersions{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeFilesInfoVersions(b *testing.B) {
-	v := FilesInfoVersions{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
 func TestMarshalUnmarshalVolInfo(t *testing.T) {
 	v := VolInfo{}
 	bts, err := v.MarshalMsg(nil)

cmd/storage-rest-client.go

@@ -611,6 +611,9 @@ func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume stri
 	respBody, err := client.call(ctx, storageRESTMethodDeleteVersions, values, &buffer, -1)
 	defer xhttp.DrainBody(respBody)
 	if err != nil {
+		if contextCanceled(ctx) {
+			err = ctx.Err()
+		}
 		for i := range errs {
 			errs[i] = err
 		}

cmd/sts-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+/// Copyright (c) 2015-2021 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -80,16 +80,6 @@ const (
 	roleArnClaim = "roleArn"
 )
 
-func parseOpenIDParentUser(parentUser string) (userID string, err error) {
-	if strings.HasPrefix(parentUser, "openid:") {
-		tokens := strings.SplitN(strings.TrimPrefix(parentUser, "openid:"), ":", 2)
-		if len(tokens) == 2 {
-			return tokens[0], nil
-		}
-	}
-	return "", errSkipFile
-}
-
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
 type stsAPIHandlers struct{}
 
@@ -142,24 +132,23 @@ func registerSTSRouter(router *mux.Router) {
 }
 
 func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (user auth.Credentials, isErrCodeSTS bool, stsErr STSErrorCode) {
-	switch getRequestAuthType(r) {
-	default:
+	if !isRequestSignatureV4(r) {
 		return user, true, ErrSTSAccessDenied
-	case authTypeSigned:
-		s3Err := isReqAuthenticated(ctx, r, globalSite.Region, serviceSTS)
-		if s3Err != ErrNone {
-			return user, false, STSErrorCode(s3Err)
-		}
+	}
 
-		user, _, s3Err = getReqAccessKeyV4(r, globalSite.Region, serviceSTS)
-		if s3Err != ErrNone {
-			return user, false, STSErrorCode(s3Err)
-		}
+	s3Err := isReqAuthenticated(ctx, r, globalSite.Region, serviceSTS)
+	if s3Err != ErrNone {
+		return user, false, STSErrorCode(s3Err)
+	}
 
-		// Temporary credentials or Service accounts cannot generate further temporary credentials.
-		if user.IsTemp() || user.IsServiceAccount() {
-			return user, true, ErrSTSAccessDenied
-		}
+	user, _, s3Err = getReqAccessKeyV4(r, globalSite.Region, serviceSTS)
+	if s3Err != ErrNone {
+		return user, false, STSErrorCode(s3Err)
+	}
+
+	// Temporary credentials or Service accounts cannot generate further temporary credentials.
+	if user.IsTemp() || user.IsServiceAccount() {
+		return user, true, ErrSTSAccessDenied
 	}
 
 	// Session tokens are not allowed in STS AssumeRole requests.
@@ -188,11 +177,11 @@ func parseForm(r *http.Request) error {
 func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AssumeRole")
 
+	// Check auth here (otherwise r.Form will have unexpected values from
+	// the call to `parseForm` below), but return failure only after we are
+	// able to validate that it is a valid STS request, so that we are able
+	// to send an appropriate audit log.
 	user, isErrCodeSTS, stsErr := checkAssumeRoleAuth(ctx, r)
-	if stsErr != ErrSTSNone {
-		writeSTSErrorResponse(ctx, w, isErrCodeSTS, stsErr, nil)
-		return
-	}
 
 	if err := parseForm(r); err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
@@ -215,6 +204,13 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	ctx = newContext(r, w, action)
 	defer logger.AuditLog(ctx, w, r, nil)
 
+	// Validate the authentication result here so that failures will be
+	// audit-logged.
+	if stsErr != ErrSTSNone {
+		writeSTSErrorResponse(ctx, w, isErrCodeSTS, stsErr, nil)
+		return
+	}
+
 	sessionPolicyStr := r.Form.Get(stsPolicy)
 	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
 	// The plain text that you use for both inline and managed session
@@ -278,6 +274,22 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Call hook for site replication.
+	if cred.ParentUser != globalActiveCred.AccessKey {
+		if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSTSAcc,
+			STSCredential: &madmin.SRSTSCredential{
+				AccessKey:    cred.AccessKey,
+				SecretKey:    cred.SecretKey,
+				SessionToken: cred.SessionToken,
+				ParentUser:   cred.ParentUser,
+			},
+		}); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
 	assumeRoleResponse := &AssumeRoleResponse{
 		Result: AssumeRoleResult{
 			Credentials: cred,
@@ -497,6 +509,21 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// Call hook for site replication.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemSTSAcc,
+		STSCredential: &madmin.SRSTSCredential{
+			AccessKey:           cred.AccessKey,
+			SecretKey:           cred.SecretKey,
+			SessionToken:        cred.SessionToken,
+			ParentUser:          cred.ParentUser,
+			ParentPolicyMapping: policyName,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	var encodedSuccessResponse []byte
 	switch action {
 	case clientGrants:
@@ -653,13 +680,14 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		return
 	}
 
-	// Call hook for cluster-replication.
+	// Call hook for site replication.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemSTSAcc,
 		STSCredential: &madmin.SRSTSCredential{
 			AccessKey:    cred.AccessKey,
 			SecretKey:    cred.SecretKey,
 			SessionToken: cred.SessionToken,
+			ParentUser:   cred.ParentUser,
 		},
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -805,12 +833,28 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	}
 
 	tmpCredentials.ParentUser = parentUser
-	err = globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, certificate.Subject.CommonName)
+	policyName := certificate.Subject.CommonName
+	err = globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, policyName)
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
 		return
 	}
 
+	// Call hook for site replication.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemSTSAcc,
+		STSCredential: &madmin.SRSTSCredential{
+			AccessKey:           tmpCredentials.AccessKey,
+			SecretKey:           tmpCredentials.SecretKey,
+			SessionToken:        tmpCredentials.SessionToken,
+			ParentUser:          tmpCredentials.ParentUser,
+			ParentPolicyMapping: policyName,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	response := new(AssumeRoleWithCertificateResponse)
 	response.Result.Credentials = tmpCredentials
 	response.Metadata.RequestID = w.Header().Get(xhttp.AmzRequestID)

cmd/sts-handlers_test.go

@@ -20,19 +20,15 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"net/http"
-	"net/url"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/coreos/go-oidc"
 	"github.com/minio/madmin-go"
 	minio "github.com/minio/minio-go/v7"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"golang.org/x/oauth2"
 )
 
 func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
@@ -742,7 +738,7 @@ func (s *TestSuiteIAM) TestOpenIDSTS(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -817,7 +813,7 @@ func (s *TestSuiteIAM) TestOpenIDSTSAddUser(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -907,7 +903,7 @@ func (s *TestSuiteIAM) TestOpenIDServiceAcc(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -985,132 +981,11 @@ func (s *TestSuiteIAM) TestOpenIDServiceAcc(c *check) {
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket)
 }
 
-type providerParams struct {
-	clientID, clientSecret, providerURL, redirectURL string
-}
-
-var testProvider = providerParams{
-	clientID:     "minio-client-app",
-	clientSecret: "minio-client-app-secret",
-	providerURL:  "http://127.0.0.1:5556/dex",
-	redirectURL:  "http://127.0.0.1:10000/oauth_callback",
-}
-
-// mockTestUserInteraction - tries to login to dex using provided credentials.
-// It performs the user's browser interaction to login and retrieves the auth
-// code from dex and exchanges it for a JWT.
-func mockTestUserInteraction(ctx context.Context, pro providerParams, username, password string) (string, error) {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
-	provider, err := oidc.NewProvider(ctx, pro.providerURL)
-	if err != nil {
-		return "", fmt.Errorf("unable to create provider: %v", err)
-	}
-
-	// Configure an OpenID Connect aware OAuth2 client.
-	oauth2Config := oauth2.Config{
-		ClientID:     pro.clientID,
-		ClientSecret: pro.clientSecret,
-		RedirectURL:  pro.redirectURL,
-
-		// Discovery returns the OAuth2 endpoints.
-		Endpoint: provider.Endpoint(),
-
-		// "openid" is a required scope for OpenID Connect flows.
-		Scopes: []string{oidc.ScopeOpenID, "groups"},
-	}
-
-	state := "xxx"
-	authCodeURL := oauth2Config.AuthCodeURL(state)
-	// fmt.Printf("authcodeurl: %s\n", authCodeURL)
-
-	var lastReq *http.Request
-	checkRedirect := func(req *http.Request, via []*http.Request) error {
-		// fmt.Printf("CheckRedirect:\n")
-		// fmt.Printf("Upcoming: %s %#v\n", req.URL.String(), req)
-		// for _, c := range via {
-		// 	fmt.Printf("Sofar: %s %#v\n", c.URL.String(), c)
-		// }
-		// Save the last request in a redirect chain.
-		lastReq = req
-		// We do not follow redirect back to client application.
-		if req.URL.Path == "/oauth_callback" {
-			return http.ErrUseLastResponse
-		}
-		return nil
-	}
-
-	dexClient := http.Client{
-		CheckRedirect: checkRedirect,
-	}
-
-	u, err := url.Parse(authCodeURL)
-	if err != nil {
-		return "", fmt.Errorf("url parse err: %v", err)
-	}
-
-	// Start the user auth flow. This page would present the login with
-	// email or LDAP option.
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return "", fmt.Errorf("new request err: %v", err)
-	}
-	_, err = dexClient.Do(req)
-	// fmt.Printf("Do: %#v %#v\n", resp, err)
-	if err != nil {
-		return "", fmt.Errorf("auth url request err: %v", err)
-	}
-
-	// Modify u to choose the ldap option
-	u.Path += "/ldap"
-	// fmt.Println(u)
-
-	// Pick the LDAP login option. This would return a form page after
-	// following some redirects. `lastReq` would be the URL of the form
-	// page, where we need to POST (submit) the form.
-	req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
-	if err != nil {
-		return "", fmt.Errorf("new request err (/ldap): %v", err)
-	}
-	_, err = dexClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("request err: %v", err)
-	}
-
-	// Fill the login form with our test creds:
-	// fmt.Printf("login form url: %s\n", lastReq.URL.String())
-	formData := url.Values{}
-	formData.Set("login", username)
-	formData.Set("password", password)
-	req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode()))
-	if err != nil {
-		return "", fmt.Errorf("new request err (/login): %v", err)
-	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-	_, err = dexClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("post form err: %v", err)
-	}
-	// fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header)
-	// fmt.Printf("lastReq: %#v\n", lastReq.URL.String())
-
-	// On form submission, the last redirect response contains the auth
-	// code, which we now have in `lastReq`. Exchange it for a JWT id_token.
-	q := lastReq.URL.Query()
-	code := q.Get("code")
-	oauth2Token, err := oauth2Config.Exchange(ctx, code)
-	if err != nil {
-		return "", fmt.Errorf("unable to exchange code for id token: %v", err)
-	}
-
-	rawIDToken, ok := oauth2Token.Extra("id_token").(string)
-	if !ok {
-		return "", fmt.Errorf("id_token not found!")
-	}
-
-	// fmt.Printf("TOKEN: %s\n", rawIDToken)
-	return rawIDToken, nil
+var testAppParams = OpenIDClientAppParams{
+	ClientID:     "minio-client-app",
+	ClientSecret: "minio-client-app-secret",
+	ProviderURL:  "http://127.0.0.1:5556/dex",
+	RedirectURL:  "http://127.0.0.1:10000/oauth_callback",
 }
 
 const (
@@ -1248,7 +1123,7 @@ func (s *TestSuiteIAM) TestOpenIDSTSWithRolePolicy(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}
@@ -1294,7 +1169,7 @@ func (s *TestSuiteIAM) TestOpenIDServiceAccWithRolePolicy(c *check) {
 	}
 
 	// Generate web identity STS token by interacting with OpenID IDP.
-	token, err := mockTestUserInteraction(ctx, testProvider, "dillon@example.io", "dillon")
+	token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon")
 	if err != nil {
 		c.Fatalf("mock user err: %v", err)
 	}

cmd/test-utils_test.go

@@ -256,8 +256,10 @@ const (
 // Random number state.
 // We generate random temporary file names so that there's a good
 // chance the file doesn't exist yet.
-var randN uint32
-var randmu sync.Mutex
+var (
+	randN  uint32
+	randmu sync.Mutex
+)
 
 // Temp files created in default Tmp dir
 var globalTestTmpDir = os.TempDir()
@@ -2155,6 +2157,7 @@ func mustGetPoolEndpoints(args ...string) EndpointServerPools {
 		SetCount:     setCount,
 		DrivesPerSet: drivesPerSet,
 		Endpoints:    endpoints,
+		CmdLine:      strings.Join(args, " "),
 	}}
 }
 

cmd/tier-handlers.go

@@ -227,7 +227,11 @@ func (api adminAPIHandlers) TierStatsHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	data, err := json.Marshal(dui.tierStats())
+	tierStats := dui.tierStats()
+	dailyStats := globalNotificationSys.GetLastDayTierStats(ctx)
+	tierStats = dailyStats.addToTierInfo(tierStats)
+
+	data, err := json.Marshal(tierStats)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/tier-last-day-stats.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"time"
+
+	"github.com/minio/madmin-go"
+)
+
+type lastDayTierStats struct {
+	Bins      [24]tierStats
+	UpdatedAt time.Time
+}
+
+func (l *lastDayTierStats) addStats(ts tierStats) {
+	now := time.Now()
+	l.forwardTo(now)
+
+	nowIdx := now.Hour()
+	l.Bins[nowIdx] = l.Bins[nowIdx].add(ts)
+}
+
+// forwardTo moves time to t, clearing entries between last update and t.
+func (l *lastDayTierStats) forwardTo(t time.Time) {
+	since := t.Sub(l.UpdatedAt).Hours()
+	// within the hour since l.UpdatedAt
+	if since < 1 {
+		return
+	}
+
+	idx, lastIdx := t.Hour(), l.UpdatedAt.Hour()
+	l.UpdatedAt = t
+
+	if since >= 24 {
+		l.Bins = [24]tierStats{}
+		return
+	}
+
+	for ; lastIdx != idx; lastIdx++ {
+		l.Bins[(lastIdx+1)%24] = tierStats{}
+	}
+}
+
+func (l *lastDayTierStats) clone() lastDayTierStats {
+	clone := lastDayTierStats{
+		UpdatedAt: l.UpdatedAt,
+	}
+	copy(clone.Bins[:], l.Bins[:])
+	return clone
+}
+
+func (l lastDayTierStats) merge(m lastDayTierStats) (merged lastDayTierStats) {
+	cl := l.clone()
+	cm := m.clone()
+
+	if cl.UpdatedAt.After(cm.UpdatedAt) {
+		cm.forwardTo(cl.UpdatedAt)
+		merged.UpdatedAt = cl.UpdatedAt
+	} else {
+		cl.forwardTo(cm.UpdatedAt)
+		merged.UpdatedAt = cm.UpdatedAt
+	}
+
+	for i := range cl.Bins {
+		merged.Bins[i] = cl.Bins[i].add(cm.Bins[i])
+	}
+
+	return merged
+}
+
+// dailyAllTierStats is used to aggregate last day tier stats across MinIO servers
+type dailyAllTierStats map[string]lastDayTierStats
+
+func (l dailyAllTierStats) merge(m dailyAllTierStats) {
+	for tier, st := range m {
+		l[tier] = l[tier].merge(st)
+	}
+}
+
+func (l dailyAllTierStats) addToTierInfo(tierInfos []madmin.TierInfo) []madmin.TierInfo {
+	for i := range tierInfos {
+		var lst lastDayTierStats
+		var ok bool
+		if lst, ok = l[tierInfos[i].Name]; !ok {
+			continue
+		}
+		for hr, st := range lst.Bins {
+			tierInfos[i].DailyStats.Bins[hr] = madmin.TierStats{
+				TotalSize:   st.TotalSize,
+				NumVersions: st.NumVersions,
+				NumObjects:  st.NumObjects,
+			}
+		}
+		tierInfos[i].DailyStats.UpdatedAt = lst.UpdatedAt
+	}
+	return tierInfos
+}