Better reporting of total/free usable capacity of the cluster (#15230)

The current code uses approximation using a ratio. The approximation 
can skew if we have multiple pools with different disk capacities.

Replace the algorithm with a simpler one which counts data 
disks and ignore parity disks.

.github/workflows/replication.yaml

@@ -1,4 +1,4 @@
-name: Multi-site replication tests
+name: MinIO advanced tests
 
 on:
   pull_request:
@@ -16,7 +16,7 @@ permissions:
 
 jobs:
   replication-test:
-    name: Replication Tests with Go ${{ matrix.go-version }}
+    name: Advanced Tests with Go ${{ matrix.go-version }}
     runs-on: ubuntu-latest
 
     strategy:
@@ -37,11 +37,18 @@ jobs:
           key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Test Decom
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-decom
+
       - name: Test Replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-replication
+
       - name: Test MinIO IDP for automatic site replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

Dockerfile.hotfix

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG RELEASE
 

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG TARGETARCH
 

Dockerfile.release.fips

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG TARGETARCH
 

Makefile

@@ -41,6 +41,10 @@ test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@CGO_ENABLED=0 go test -tags kqueue ./...
 
+test-decom: install
+	@echo "Running minio decom tests"
+	@env bash $(PWD)/docs/distributed/decom.sh
+
 test-upgrade: build
 	@echo "Running minio upgrade tests"
 	@(env bash $(PWD)/buildscripts/minio-upgrade.sh)

cmd/admin-bucket-handlers.go

@@ -92,15 +92,17 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	bucketMeta := madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeQuotaConfig,
-		Bucket: bucket,
-		Quota:  data,
+		Type:      madmin.SRBucketMetaTypeQuotaConfig,
+		Bucket:    bucket,
+		Quota:     data,
+		UpdatedAt: updatedAt,
 	}
 	if quotaConfig.Quota == 0 {
 		bucketMeta.Quota = nil
@@ -267,7 +269,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -362,7 +364,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -707,7 +709,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -721,6 +724,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 				Bucket:           bucket,
 				ObjectLockConfig: &cfgStr,
+				UpdatedAt:        updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -794,7 +798,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -846,7 +850,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -883,15 +887,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
 			// Call site replication hook.
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypePolicy,
-				Bucket: bucket,
-				Policy: bucketPolicyBytes,
+				Type:      madmin.SRBucketMetaTypePolicy,
+				Bucket:    bucket,
+				Policy:    bucketPolicyBytes,
+				UpdatedAt: updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -922,7 +928,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -965,7 +971,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 
 			// Store the bucket encryption configuration in the object layer
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -979,6 +986,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Type:      madmin.SRBucketMetaTypeSSEConfig,
 				Bucket:    bucket,
 				SSEConfig: &cfgStr,
+				UpdatedAt: updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -998,7 +1006,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -1008,9 +1017,10 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			// errors.
 			cfgStr := base64.StdEncoding.EncodeToString(configData)
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypeTags,
-				Bucket: bucket,
-				Tags:   &cfgStr,
+				Type:      madmin.SRBucketMetaTypeTags,
+				Bucket:    bucket,
+				Tags:      &cfgStr,
+				UpdatedAt: updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -1033,15 +1043,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
 
 			bucketMeta := madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypeQuotaConfig,
-				Bucket: bucket,
-				Quota:  data,
+				Type:      madmin.SRBucketMetaTypeQuotaConfig,
+				Bucket:    bucket,
+				Quota:     data,
+				UpdatedAt: updatedAt,
 			}
 			if quotaConfig.Quota == 0 {
 				bucketMeta.Quota = nil

cmd/admin-handlers-idp-config.go

@@ -0,0 +1,321 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+// List of implemented ID config types.
+var idCfgTypes = set.CreateStringSet("openid")
+
+// SetIdentityProviderCfg:
+//
+// PUT <admin-prefix>/id-cfg?type=openid&name=dex1
+func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetIdentityCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err, logger.Application)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	var cfgDataBuilder strings.Builder
+	switch cfgType {
+	case "openid":
+		fmt.Fprintf(&cfgDataBuilder, "identity_openid")
+	}
+
+	// Ensure body content type is opaque.
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/octet-stream" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+		return
+	}
+
+	// Subsystem configuration name could be empty.
+	cfgName := mux.Vars(r)["name"]
+	if cfgName != "" {
+		fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
+	}
+
+	fmt.Fprintf(&cfgDataBuilder, "%s%s", config.KvSpaceSeparator, string(reqBytes))
+
+	cfgData := cfgDataBuilder.String()
+	subSys, _, _, err := config.GetSubSys(cfgData)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	cfg, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	dynamic, err := cfg.ReadConfig(strings.NewReader(cfgData))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// IDP config is not dynamic. Sanity check.
+	if dynamic {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		return
+	}
+
+	if err = validateConfig(cfg, subSys); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+
+	// Update the actual server config on disk.
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write to the config input KV to history.
+	if err = saveServerConfigHistory(ctx, objectAPI, []byte(cfgData)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetIdentityProviderCfg:
+//
+// GET <admin-prefix>/id-cfg?type=openid&name=dex_test
+//
+// GetIdentityProviderCfg returns a list of configured IDPs on the server if
+// name is empty. If name is non-empty, returns the configuration details for
+// the IDP of the given type and configuration name. The configuration name for
+// the default ("un-named") configuration target is `_`.
+func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetIdentityProviderCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	cfgName := r.Form.Get("name")
+	password := cred.SecretKey
+
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// If no cfgName is provided, we list.
+	if cfgName == "" {
+		a.listIdentityProviders(ctx, w, r, cfgType, password)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+
+	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	if err != nil {
+		if err == openid.ErrProviderConfigNotFound {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			return
+		}
+
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	res := madmin.IDPConfig{
+		Type: cfgType,
+		Name: cfgName,
+		Info: cfgInfos,
+	}
+	data, err := json.Marshal(res)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
+	// var subSys string
+	switch cfgType {
+	case "openid":
+		// subSys = config.IdentityOpenIDSubSys
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+	cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(cfgList)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// DeleteIdentityProviderCfg:
+//
+// DELETE <admin-prefix>/id-cfg?type=openid&name=dex_test
+func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteIdentityProviderCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	cfgName := mux.Vars(r)["name"]
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+
+	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	if err != nil {
+		if err == openid.ErrProviderConfigNotFound {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			return
+		}
+
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	hasEnv := false
+	for _, ci := range cfgInfos {
+		if ci.IsCfg && ci.IsEnv {
+			hasEnv = true
+			break
+		}
+	}
+
+	if hasEnv {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
+		return
+	}
+
+	var subSys string
+	switch cfgType {
+	case "openid":
+		subSys = config.IdentityOpenIDSubSys
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg, err = readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if err = validateConfig(cfg, subSys); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	dynamic := config.SubSystemsDynamic.Contains(subSys)
+	if dynamic {
+		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
+	}
+}

cmd/admin-handlers-pools.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -149,8 +150,10 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 
 	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
+		apiErr := toAdminAPIErr(ctx, errInvalidArgument)
+		apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
 		// We didn't find any matching pools, invalid input
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
 		return
 	}
 

cmd/admin-handlers-site-replication.go

@@ -160,7 +160,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRIAMItemPolicy:
 		if item.Policy == nil {
-			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
 		} else {
 			policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
 			if perr != nil {
@@ -168,21 +168,21 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 				return
 			}
 			if policy.IsEmpty() {
-				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
 			} else {
-				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy, item.UpdatedAt)
 			}
 		}
 	case madmin.SRIAMItemSvcAcc:
-		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange)
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
 	case madmin.SRIAMItemPolicyMapping:
-		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping)
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
 	case madmin.SRIAMItemSTSAcc:
-		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential)
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
 	case madmin.SRIAMItemIAMUser:
-		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser)
+		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
 	case madmin.SRIAMItemGroupInfo:
-		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo)
+		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -214,7 +214,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRBucketMetaTypePolicy:
 		if item.Policy == nil {
-			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 		} else {
 			bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
 			if berr != nil {
@@ -222,33 +222,33 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				return
 			}
 			if bktPolicy.IsEmpty() {
-				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 			} else {
-				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy, item.UpdatedAt)
 			}
 		}
 	case madmin.SRBucketMetaTypeQuotaConfig:
 		if item.Quota == nil {
-			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil)
+			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 		} else {
 			quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
-			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig); err != nil {
+			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 		}
 	case madmin.SRBucketMetaTypeVersionConfig:
-		err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning)
+		err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeTags:
-		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
+		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeObjectLockConfig:
-		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
+		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeSSEConfig:
-		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
+		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)

cmd/admin-handlers-users.go

@@ -72,6 +72,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 			AccessKey:   accessKey,
 			IsDeleteReq: true,
 		},
+		UpdatedAt: UTCNow(),
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -240,9 +241,9 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-
+	var updatedAt time.Time
 	if updReq.IsRemove {
-		err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
+		updatedAt, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
 	} else {
 		// Check if group already exists
 		if _, gerr := globalIAMSys.GetGroupDescription(updReq.Group); gerr != nil {
@@ -253,7 +254,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 				return
 			}
 		}
-		err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+		updatedAt, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
 	}
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -265,6 +266,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: updReq,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -341,12 +343,15 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 	group := vars["group"]
 	status := vars["status"]
 
-	var err error
+	var (
+		err       error
+		updatedAt time.Time
+	)
 	switch status {
 	case statusEnabled:
-		err = globalIAMSys.SetGroupStatus(ctx, group, true)
+		updatedAt, err = globalIAMSys.SetGroupStatus(ctx, group, true)
 	case statusDisabled:
-		err = globalIAMSys.SetGroupStatus(ctx, group, false)
+		updatedAt, err = globalIAMSys.SetGroupStatus(ctx, group, false)
 	default:
 		err = errInvalidArgument
 	}
@@ -364,6 +369,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 				IsRemove: false,
 			},
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -391,7 +397,8 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if err := globalIAMSys.SetUserStatus(ctx, accessKey, madmin.AccountStatus(status)); err != nil {
+	updatedAt, err := globalIAMSys.SetUserStatus(ctx, accessKey, madmin.AccountStatus(status))
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -405,6 +412,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 				Status: madmin.AccountStatus(status),
 			},
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -439,8 +447,8 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	userCred, exists := globalIAMSys.GetUser(ctx, accessKey)
-	if exists && (userCred.IsTemp() || userCred.IsServiceAccount()) {
+	user, exists := globalIAMSys.GetUser(ctx, accessKey)
+	if exists && (user.Credentials.IsTemp() || user.Credentials.IsServiceAccount()) {
 		// Updating STS credential is not allowed, and this API does not
 		// support updating service accounts.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
@@ -501,7 +509,8 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
+	updatedAt, err := globalIAMSys.CreateUser(ctx, accessKey, ureq)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -513,6 +522,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 			IsDeleteReq: false,
 			UserReq:     &ureq,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -674,7 +684,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	opts.sessionPolicy = sp
-	newCred, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
+	newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -717,6 +727,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 					Status:        auth.AccountOn,
 				},
 			},
+			UpdatedAt: updatedAt,
 		})
 		if err != nil {
 			logger.LogIf(ctx, err)
@@ -800,7 +811,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		status:        updateReq.NewStatus,
 		sessionPolicy: sp,
 	}
-	err = globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
+	updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -818,6 +829,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 					SessionPolicy: updateReq.NewPolicy,
 				},
 			},
+			UpdatedAt: updatedAt,
 		})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1057,6 +1069,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 					AccessKey: serviceAccount,
 				},
 			},
+			UpdatedAt: UTCNow(),
 		}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1395,8 +1408,9 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	// Call cluster-replication policy creation hook to replicate policy deletion to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-		Type: madmin.SRIAMItemPolicy,
-		Name: policyName,
+		Type:      madmin.SRIAMItemPolicy,
+		Name:      policyName,
+		UpdatedAt: UTCNow(),
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1453,7 +1467,8 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	if err = globalIAMSys.SetPolicy(ctx, policyName, *iamPolicy); err != nil {
+	updatedAt, err := globalIAMSys.SetPolicy(ctx, policyName, *iamPolicy)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1461,9 +1476,10 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	// Call cluster-replication policy creation hook to replicate policy to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-		Type:   madmin.SRIAMItemPolicy,
-		Name:   policyName,
-		Policy: iamPolicyBytes,
+		Type:      madmin.SRIAMItemPolicy,
+		Name:      policyName,
+		Policy:    iamPolicyBytes,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1498,7 +1514,8 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		}
 	}
 
-	if err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup); err != nil {
+	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1510,6 +1527,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 			IsGroup:     isGroup,
 			Policy:      policyName,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1597,22 +1615,22 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case allUsersFile:
-			userCreds := make(map[string]auth.Credentials)
+			userIdentities := make(map[string]UserIdentity)
 			globalIAMSys.store.rlock()
-			err := globalIAMSys.store.loadUsers(ctx, regUser, userCreds)
+			err := globalIAMSys.store.loadUsers(ctx, regUser, userIdentities)
 			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
 			userAccounts := make(map[string]madmin.AddOrUpdateUserReq)
-			for u, cred := range userCreds {
+			for u, uid := range userIdentities {
 				status := madmin.AccountDisabled
-				if cred.IsValid() {
+				if uid.Credentials.IsValid() {
 					status = madmin.AccountEnabled
 				}
 				userAccounts[u] = madmin.AddOrUpdateUserReq{
-					SecretKey: cred.SecretKey,
+					SecretKey: uid.Credentials.SecretKey,
 					Status:    status,
 				}
 			}
@@ -1646,7 +1664,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case allSvcAcctsFile:
-			serviceAccounts := make(map[string]auth.Credentials)
+			serviceAccounts := make(map[string]UserIdentity)
 			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts)
 			globalIAMSys.store.runlock()
@@ -1661,12 +1679,12 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 					// site replication is enabled.
 					continue
 				}
-				claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.AccessKey)
+				claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 					return
 				}
-				_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.AccessKey)
+				_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 					return
@@ -1681,13 +1699,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 					}
 				}
 				svcAccts[user] = madmin.SRSvcAccCreate{
-					Parent:        acc.ParentUser,
+					Parent:        acc.Credentials.ParentUser,
 					AccessKey:     user,
-					SecretKey:     acc.SecretKey,
-					Groups:        acc.Groups,
+					SecretKey:     acc.Credentials.SecretKey,
+					Groups:        acc.Credentials.Groups,
 					Claims:        claims,
 					SessionPolicy: json.RawMessage(policyJSON),
-					Status:        acc.Status,
+					Status:        acc.Credentials.Status,
 				}
 			}
 
@@ -1832,7 +1850,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				if policy.IsEmpty() {
 					err = globalIAMSys.DeletePolicy(ctx, policyName, true)
 				} else {
-					err = globalIAMSys.SetPolicy(ctx, policyName, policy)
+					_, err = globalIAMSys.SetPolicy(ctx, policyName, policy)
 				}
 				if err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allPoliciesFile, policyName), r.URL)
@@ -1870,8 +1888,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					return
 				}
 
-				userCred, exists := globalIAMSys.GetUser(ctx, accessKey)
-				if exists && (userCred.IsTemp() || userCred.IsServiceAccount()) {
+				user, exists := globalIAMSys.GetUser(ctx, accessKey)
+				if exists && (user.Credentials.IsTemp() || user.Credentials.IsServiceAccount()) {
 					// Updating STS credential is not allowed, and this API does not
 					// support updating service accounts.
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
@@ -1910,7 +1928,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allUsersFile, accessKey), r.URL)
 					return
 				}
-				if err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
+				if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, toAdminAPIErrCode(ctx, err), err, allUsersFile, accessKey), r.URL)
 					return
 				}
@@ -1949,7 +1967,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 						return
 					}
 				}
-				if gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil {
+				if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allGroupsFile, group), r.URL)
 					return
 				}
@@ -2018,7 +2036,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 						status:        svcAcctReq.Status,
 						sessionPolicy: sp,
 					}
-					err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
+					_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
 					if err != nil {
 						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 						return
@@ -2044,7 +2062,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					opts.claims[ldapUser] = targetUser // username DN
 				}
 
-				if _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
+				if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 					return
 				}
@@ -2084,7 +2102,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, userPolicyMappingsFile, u), r.URL)
 					return
 				}
-				if err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
 					return
 				}
@@ -2113,7 +2131,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for g, pm := range grpPolicyMap {
-				if err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
 					return
 				}
@@ -2152,7 +2170,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL)
 					return
 				}
-				if err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
 					return
 				}
@@ -2181,7 +2199,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for g, pm := range grpPolicyMap {
-				if err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
 					return
 				}

cmd/admin-handlers.go

@@ -28,6 +28,7 @@ import (
 	"hash/crc32"
 	"io"
 	"io/ioutil"
+	"math"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -50,6 +51,7 @@ import (
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
+	"github.com/minio/minio/internal/pubsub"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
 	"github.com/secure-io/sio-go"
@@ -343,6 +345,92 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// MetricsHandler - GET /minio/admin/v3/metrics
+// ----------
+// Get realtime server metrics
+func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "Metrics")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+	const defaultMetricsInterval = time.Second
+
+	interval, err := time.ParseDuration(r.Form.Get("interval"))
+	if err != nil || interval < time.Second {
+		interval = defaultMetricsInterval
+	}
+	n, err := strconv.Atoi(r.Form.Get("n"))
+	if err != nil || n <= 0 {
+		n = math.MaxInt32
+	}
+	var types madmin.MetricType
+	if t, _ := strconv.ParseUint(r.Form.Get("types"), 10, 64); t != 0 {
+		types = madmin.MetricType(t)
+	} else {
+		types = madmin.MetricsAll
+	}
+	hosts := strings.Split(r.Form.Get("hosts"), ",")
+	byhost := strings.EqualFold(r.Form.Get("by-host"), "true")
+	var hostMap map[string]struct{}
+	if len(hosts) > 0 && hosts[0] != "" {
+		hostMap = make(map[string]struct{}, len(hosts))
+		for _, k := range hosts {
+			if k != "" {
+				hostMap[k] = struct{}{}
+			}
+		}
+	}
+	done := ctx.Done()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	w.Header().Set(xhttp.ContentType, string(mimeJSON))
+
+	for n > 0 {
+		var m madmin.RealtimeMetrics
+		mLocal := collectLocalMetrics(types, hostMap)
+		m.Merge(&mLocal)
+
+		// Allow half the interval for collecting remote...
+		cctx, cancel := context.WithTimeout(ctx, interval/2)
+		mRemote := collectRemoteMetrics(cctx, types, hostMap)
+		cancel()
+		m.Merge(&mRemote)
+		if !byhost {
+			m.ByHost = nil
+		}
+
+		m.Final = n <= 1
+		// Marshal API response
+		jsonBytes, err := json.Marshal(m)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		_, err = w.Write(jsonBytes)
+		if err != nil {
+			n = 0
+		}
+
+		n--
+		if n <= 0 {
+			break
+		}
+
+		// Flush before waiting for next...
+		w.(http.Flusher).Flush()
+
+		select {
+		case <-ticker.C:
+		case <-done:
+			return
+		}
+	}
+}
+
 // DataUsageInfoHandler - GET /minio/admin/v3/datausage
 // ----------
 // Get server/cluster data usage info
@@ -1189,7 +1277,7 @@ func deleteObjectPerfBucket(objectAPI ObjectLayer) {
 func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
 	storageInfo, _ := objectAPI.StorageInfo(ctx)
 	capacityNeeded := uint64(concurrent * size)
-	capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))
+	capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)
 
 	if capacity < capacityNeeded {
 		return false, false, fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s",
@@ -1300,72 +1388,45 @@ const (
 // - input entry is not of the type *madmin.TraceInfo*
 // - errOnly entries are to be traced, not status code 2xx, 3xx.
 // - madmin.TraceInfo type is asked by opts
-func mustTrace(entry interface{}, opts madmin.ServiceTraceOpts) (shouldTrace bool) {
-	trcInfo, ok := entry.(madmin.TraceInfo)
-	if !ok {
+func shouldTrace(trcInfo madmin.TraceInfo, opts madmin.ServiceTraceOpts) (shouldTrace bool) {
+	// Reject all unwanted types.
+	want := opts.TraceTypes()
+	if !want.Contains(trcInfo.TraceType) {
 		return false
 	}
 
-	// Override shouldTrace decision with errOnly filtering
-	defer func() {
-		if shouldTrace && opts.OnlyErrors {
-			shouldTrace = trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
-		}
-	}()
+	isHTTP := trcInfo.TraceType.Overlaps(madmin.TraceInternal|madmin.TraceS3) && trcInfo.HTTP != nil
 
-	if opts.Threshold > 0 {
-		var latency time.Duration
-		switch trcInfo.TraceType {
-		case madmin.TraceOS:
-			latency = trcInfo.OSStats.Duration
-		case madmin.TraceStorage:
-			latency = trcInfo.StorageStats.Duration
-		case madmin.TraceHTTP:
-			latency = trcInfo.CallStats.Latency
-		}
-		if latency < opts.Threshold {
-			return false
-		}
+	// Check latency...
+	if opts.Threshold > 0 && trcInfo.Duration < opts.Threshold {
+		return false
 	}
 
-	if opts.Internal && trcInfo.TraceType == madmin.TraceHTTP && HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) {
-		return true
+	// Check internal path
+	isInternal := isHTTP && HasPrefix(trcInfo.HTTP.ReqInfo.Path, minioReservedBucketPath+SlashSeparator)
+	if isInternal && !opts.Internal {
+		return false
 	}
 
-	if opts.S3 && trcInfo.TraceType == madmin.TraceHTTP && !HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) {
-		return true
+	// Filter non-errors.
+	if isHTTP && opts.OnlyErrors && trcInfo.HTTP.RespInfo.StatusCode < http.StatusBadRequest {
+		return false
 	}
 
-	if opts.Storage && trcInfo.TraceType == madmin.TraceStorage {
-		return true
-	}
-
-	return opts.OS && trcInfo.TraceType == madmin.TraceOS
+	return true
 }
 
 func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err error) {
-	q := r.Form
-
-	opts.OnlyErrors = q.Get("err") == "true"
-	opts.S3 = q.Get("s3") == "true"
-	opts.Internal = q.Get("internal") == "true"
-	opts.Storage = q.Get("storage") == "true"
-	opts.OS = q.Get("os") == "true"
-
+	if err := opts.ParseParams(r); err != nil {
+		return opts, err
+	}
 	// Support deprecated 'all' query
-	if q.Get("all") == "true" {
+	if r.Form.Get("all") == "true" {
 		opts.S3 = true
 		opts.Internal = true
 		opts.Storage = true
 		opts.OS = true
-	}
-
-	if t := q.Get("threshold"); t != "" {
-		d, err := time.ParseDuration(t)
-		if err != nil {
-			return opts, err
-		}
-		opts.Threshold = d
+		// Older mc - cannot deal with more types...
 	}
 	return
 }
@@ -1388,20 +1449,21 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-
 	setEventStreamHeaders(w)
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan interface{}, 4000)
+	traceCh := make(chan pubsub.Maskable, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
+	mask := pubsub.MaskFromMaskable(traceOpts.TraceTypes())
 
-	traceFn := func(entry interface{}) bool {
-		return mustTrace(entry, traceOpts)
-	}
-
-	err = globalTrace.Subscribe(traceCh, ctx.Done(), traceFn)
+	err = globalTrace.Subscribe(mask, traceCh, ctx.Done(), func(entry pubsub.Maskable) bool {
+		if e, ok := entry.(madmin.TraceInfo); ok {
+			return shouldTrace(e, traceOpts)
+		}
+		return false
+	})
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
 		return
@@ -1442,7 +1504,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// The handler sends console logs to the connected HTTP client.
+// The ConsoleLogHandler handler sends console logs to the connected HTTP client.
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
@@ -1460,11 +1522,10 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 		limitLines = 10
 	}
 
-	logKind := r.Form.Get("logType")
-	if logKind == "" {
-		logKind = string(logger.All)
+	logKind := madmin.LogKind(strings.ToUpper(r.Form.Get("logType"))).LogMask()
+	if logKind == 0 {
+		logKind = madmin.LogMaskAll
 	}
-	logKind = strings.ToUpper(logKind)
 
 	// Avoid reusing tcp connection if read timeout is hit
 	// This is needed to make r.Context().Done() work as
@@ -1473,7 +1534,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	setEventStreamHeaders(w)
 
-	logCh := make(chan interface{}, 4000)
+	logCh := make(chan pubsub.Maskable, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 

cmd/admin-router.go

@@ -66,6 +66,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
+		// Metrics operation
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(gz(httpTraceAll(adminAPI.MetricsHandler)))
 
 		if globalIsDistErasure || globalIsErasure {
 			// Heal operations
@@ -176,6 +178,13 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// Import IAM info
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
 
+		// IDentity Provider configuration APIs
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg))).Queries("type", "{type:.*}")
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
+
+		// -- END IAM APIs --
+
 		// GetBucketQuotaConfig
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")

cmd/api-errors.go

@@ -267,6 +267,8 @@ const (
 	ErrAdminConfigNoQuorum
 	ErrAdminConfigTooLarge
 	ErrAdminConfigBadJSON
+	ErrAdminNoSuchConfigTarget
+	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
@@ -1240,11 +1242,21 @@ var errorCodes = errorCodeMap{
 			maxEConfigJSONSize),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoSuchConfigTarget: {
+		Code:           "XMinioAdminNoSuchConfigTarget",
+		Description:    "No such named configuration target exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigBadJSON: {
 		Code:           "XMinioAdminConfigBadJSON",
 		Description:    "JSON configuration provided is of incorrect format",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigEnvOverridden: {
+		Code:           "XMinioAdminConfigEnvOverridden",
+		Description:    "Unable to update config via Admin API due to environment variable override",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigDuplicateKeys: {
 		Code:           "XMinioAdminConfigDuplicateKeys",
 		Description:    "JSON configuration provided has objects with duplicate keys",

cmd/auth-handler.go

@@ -500,11 +500,18 @@ func isSupportedS3AuthType(aType authType) bool {
 func setAuthHandler(h http.Handler) http.Handler {
 	// handler for validating incoming authorization headers.
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		aType := getRequestAuthType(r)
 		if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
 			// Verify if date headers are set, if not reject the request
 			amzDate, errCode := parseAmzDateHeader(r)
 			if errCode != ErrNone {
+				if ok {
+					tc.funcName = "handler.Auth"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				// All our internal APIs are sensitive towards Date
 				// header, for all requests where Date header is not
 				// present we will reject such clients.
@@ -516,6 +523,11 @@ func setAuthHandler(h http.Handler) http.Handler {
 			// or in the future, reject request otherwise.
 			curTime := UTCNow()
 			if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
+				if ok {
+					tc.funcName = "handler.Auth"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -525,6 +537,12 @@ func setAuthHandler(h http.Handler) http.Handler {
 			h.ServeHTTP(w, r)
 			return
 		}
+
+		if ok {
+			tc.funcName = "handler.Auth"
+			tc.responseRecorder.LogErrBody = true
+		}
+
 		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
 		atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
 	})

cmd/background-heal-ops.go

@@ -22,6 +22,7 @@ import (
 	"runtime"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/pubsub"
 )
 
 // healTask represents what to heal along with options
@@ -49,10 +50,10 @@ type healRoutine struct {
 	workers int
 }
 
-func systemIO() int {
+func activeListeners() int {
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	return int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
+	return int(globalHTTPListen.NumSubscribers(pubsub.MaskAll)) + int(globalTrace.NumSubscribers(pubsub.MaskAll))
 }
 
 func waitForLowHTTPReq() {
@@ -61,7 +62,7 @@ func waitForLowHTTPReq() {
 		currentIO = httpServer.GetRequestCount
 	}
 
-	globalHealConfig.Wait(currentIO, systemIO)
+	globalHealConfig.Wait(currentIO, activeListeners)
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {

cmd/bucket-encryption-handlers.go

@@ -108,7 +108,8 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}
 
 	// Store the bucket encryption configuration in the object layer
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -122,6 +123,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: &cfgStr,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -202,7 +204,8 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Delete bucket encryption config from object layer
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -212,6 +215,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: nil,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-handlers.go

@@ -525,7 +525,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			VersionSuspended: vc.Suspended(),
 		}
 
-		if replicateDeletes || hasLockEnabled || !globalTierConfigMgr.Empty() {
+		if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() {
 			if !globalTierConfigMgr.Empty() && object.VersionID == "" && opts.VersionSuspended {
 				opts.VersionID = nullVersionID
 			}
@@ -554,7 +554,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				object.ReplicateDecisionStr = dsc.String()
 			}
 		}
-		if hasLockEnabled {
+		if object.VersionID != "" && hasLockEnabled {
 			if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
 				apiErr := errorCodes.ToAPIErr(apiErrCode)
 				deleteResults[index].errInfo = DeleteError{
@@ -1385,7 +1385,8 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1399,6 +1400,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 		Bucket:           bucket,
 		ObjectLockConfig: &cfgStr,
+		UpdatedAt:        updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1490,7 +1492,8 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1501,9 +1504,10 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
 	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeTags,
-		Bucket: bucket,
-		Tags:   &cfgStr,
+		Type:      madmin.SRBucketMetaTypeTags,
+		Bucket:    bucket,
+		Tags:      &cfgStr,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1572,14 +1576,16 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeTags,
-		Bucket: bucket,
+		Type:      madmin.SRBucketMetaTypeTags,
+		Bucket:    bucket,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-lifecycle-handlers.go

@@ -91,7 +91,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -168,7 +168,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -75,28 +75,28 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 
 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
-func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) error {
+func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if globalIsGateway && globalGatewayName != NASBackendGateway {
 		if configFile == bucketPolicyConfig {
 			if configData == nil {
-				return objAPI.DeleteBucketPolicy(ctx, bucket)
+				return updatedAt, objAPI.DeleteBucketPolicy(ctx, bucket)
 			}
 			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
 			if err != nil {
-				return err
+				return updatedAt, err
 			}
-			return objAPI.SetBucketPolicy(ctx, bucket, config)
+			return updatedAt, objAPI.SetBucketPolicy(ctx, bucket, config)
 		}
-		return NotImplemented{}
+		return updatedAt, NotImplemented{}
 	}
 
 	if bucket == minioMetaBucket {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
@@ -105,56 +105,56 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 			// Only single drive mode needs this fallback.
 			meta = newBucketMetadata(bucket)
 		} else {
-			return err
+			return updatedAt, err
 		}
 	}
-
+	updatedAt = UTCNow()
 	switch configFile {
 	case bucketPolicyConfig:
 		meta.PolicyConfigJSON = configData
-		meta.PolicyConfigUpdatedAt = UTCNow()
+		meta.PolicyConfigUpdatedAt = updatedAt
 	case bucketNotificationConfig:
 		meta.NotificationConfigXML = configData
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
-		meta.EncryptionConfigUpdatedAt = UTCNow()
+		meta.EncryptionConfigUpdatedAt = updatedAt
 	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
-		meta.TaggingConfigUpdatedAt = UTCNow()
+		meta.TaggingConfigUpdatedAt = updatedAt
 	case bucketQuotaConfigFile:
 		meta.QuotaConfigJSON = configData
-		meta.QuotaConfigUpdatedAt = UTCNow()
+		meta.QuotaConfigUpdatedAt = updatedAt
 	case objectLockConfig:
 		meta.ObjectLockConfigXML = configData
-		meta.ObjectLockConfigUpdatedAt = UTCNow()
+		meta.ObjectLockConfigUpdatedAt = updatedAt
 	case bucketVersioningConfig:
 		meta.VersioningConfigXML = configData
-		meta.VersioningConfigUpdatedAt = UTCNow()
+		meta.VersioningConfigUpdatedAt = updatedAt
 	case bucketReplicationConfig:
 		meta.ReplicationConfigXML = configData
-		meta.ReplicationConfigUpdatedAt = UTCNow()
+		meta.ReplicationConfigUpdatedAt = updatedAt
 	case bucketTargetsFile:
 		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, kms.Context{
 			bucket:            meta.Name,
 			bucketTargetsFile: bucketTargetsFile,
 		})
 		if err != nil {
-			return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+			return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
 		}
 	default:
-		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
+		return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
 
 	if err := meta.Save(ctx, objAPI); err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.Set(bucket, meta)
 	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket) // Do not use caller context here
 
-	return nil
+	return updatedAt, nil
 }
 
 // Get metadata for a bucket.

cmd/bucket-notification-handlers.go

@@ -160,7 +160,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers.go

@@ -103,16 +103,18 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Call site replication hook.
 	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypePolicy,
-		Bucket: bucket,
-		Policy: bucketPolicyBytes,
+		Type:      madmin.SRBucketMetaTypePolicy,
+		Bucket:    bucket,
+		Policy:    bucketPolicyBytes,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -148,15 +150,17 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Call site replication hook.
 	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypePolicy,
-		Bucket: bucket,
+		Type:      madmin.SRBucketMetaTypePolicy,
+		Bucket:    bucket,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-replication-handlers.go

@@ -86,7 +86,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -165,7 +165,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
 		return
 	}
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -202,6 +202,12 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+
+	if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	var usageInfo BucketUsageInfo
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err == nil && !dataUsageInfo.LastUpdate.IsZero() {

cmd/bucket-stats.go

@@ -26,7 +26,7 @@ import (
 // ReplicationLatency holds information of bucket operations latency, such us uploads
 type ReplicationLatency struct {
 	// Single & Multipart PUTs latency
-	UploadHistogram LastMinuteLatencies
+	UploadHistogram LastMinuteHistogram
 }
 
 // Merge two replication latency into a new one
@@ -41,7 +41,7 @@ func (rl ReplicationLatency) getUploadLatency() (ret map[string]uint64) {
 	avg := rl.UploadHistogram.GetAvgData()
 	for k, v := range avg {
 		// Convert nanoseconds to milliseconds
-		ret[sizeTagToString(k)] = v.avg() / uint64(time.Millisecond)
+		ret[sizeTagToString(k)] = uint64(v.avg() / time.Millisecond)
 	}
 	return
 }

cmd/bucket-versioning-handler.go

@@ -97,7 +97,8 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -111,6 +112,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,
+		UpdatedAt:  updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/callhome.go

@@ -134,3 +134,17 @@ func performCallhome(ctx context.Context) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
 	}
 }
+
+const (
+	callhomeURL    = "https://subnet.min.io/api/callhome"
+	callhomeURLDev = "http://localhost:9000/api/callhome"
+)
+
+func sendCallhomeInfo(ch CallhomeInfo) error {
+	url := callhomeURL
+	if globalIsCICD {
+		url = callhomeURLDev
+	}
+	_, err := globalSubnetConfig.Post(url, ch)
+	return err
+}

cmd/common-main.go

@@ -102,6 +102,11 @@ func init() {
 		PersistOnFailure: false,
 	}
 
+	t, _ := minioVersionToReleaseTime(Version)
+	if !t.IsZero() {
+		globalVersionUnix = uint64(t.Unix())
+	}
+
 	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
 
 	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile()

cmd/config-current.go

@@ -364,7 +364,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	case config.SubnetSubSys:
-		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default]); err != nil {
+		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], nil); err != nil {
 			return err
 		}
 	case config.CallhomeSubSys:
@@ -599,7 +599,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
 
-	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default])
+	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 	}

cmd/consolelogger.go

@@ -18,10 +18,11 @@
 package cmd
 
 import (
-	ring "container/ring"
+	"container/ring"
 	"context"
 	"sync"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/logger/target/console"
@@ -71,11 +72,11 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
 // HasLogListeners returns true if console log listeners are registered
 // for this node or peers
 func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
-	return sys != nil && sys.pubsub.NumSubscribers() > 0
+	return sys != nil && sys.pubsub.NumSubscribers(madmin.LogMaskAll) > 0
 }
 
 // Subscribe starts console logging for this node.
-func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) error {
+func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry pubsub.Maskable) bool) error {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
 		logger.AddSystemTarget(sys)
@@ -115,8 +116,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
 			}
 		}
 	}
-
-	return sys.pubsub.Subscribe(subCh, doneCh, filter)
+	return sys.pubsub.Subscribe(pubsub.MaskFromMaskable(madmin.LogMaskAll), subCh, doneCh, filter)
 }
 
 // Init if HTTPConsoleLoggerSys is valid, always returns nil right now
@@ -163,9 +163,9 @@ func (sys *HTTPConsoleLoggerSys) Type() types.TargetType {
 
 // Send log message 'e' to console and publish to console
 // log pubsub system
-func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
+func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
 	var lg log.Info
-	switch e := e.(type) {
+	switch e := entry.(type) {
 	case log.Entry:
 		lg = log.Info{Entry: e, NodeName: sys.nodeName}
 	case string:
@@ -179,5 +179,5 @@ func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
 	sys.logBuf = sys.logBuf.Next()
 	sys.Unlock()
 
-	return sys.console.Send(e, string(logger.All))
+	return sys.console.Send(entry, string(logger.All))
 }

cmd/data-scanner-metric.go

@@ -0,0 +1,315 @@
+package cmd
+
+import (
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+)
+
+//go:generate stringer -type=scannerMetric -trimprefix=scannerMetric $GOFILE
+
+type scannerMetric uint8
+
+type scannerMetrics struct {
+	// All fields must be accessed atomically and aligned.
+	operations [scannerMetricLast]uint64
+	latency    [scannerMetricLastRealtime]lockedLastMinuteLatency
+
+	// actions records actions performed.
+	actions        [lifecycle.ActionCount]uint64
+	actionsLatency [lifecycle.ActionCount]lockedLastMinuteLatency
+
+	// currentPaths contains (string,*currentPathTracker) for each disk processing.
+	// Alignment not required.
+	currentPaths sync.Map
+
+	cycleInfoMu sync.Mutex
+	cycleInfo   *currentScannerCycle
+}
+
+var globalScannerMetrics scannerMetrics
+
+const (
+	// START Realtime metrics, that only to records
+	// last minute latencies and total operation count.
+	scannerMetricReadMetadata scannerMetric = iota
+	scannerMetricCheckMissing
+	scannerMetricSaveUsage
+	scannerMetricApplyAll
+	scannerMetricApplyVersion
+	scannerMetricTierObjSweep
+	scannerMetricHealCheck
+	scannerMetricILM
+	scannerMetricCheckReplication
+	scannerMetricYield
+
+	// START Trace metrics:
+	scannerMetricStartTrace
+	scannerMetricScanObject // Scan object. All operations included.
+
+	// END realtime metrics:
+	scannerMetricLastRealtime
+
+	// Trace only metrics:
+	scannerMetricScanFolder     // Scan a folder on disk, recursively.
+	scannerMetricScanCycle      // Full cycle, cluster global
+	scannerMetricScanBucketDisk // Single bucket on one disk
+
+	// Must be last:
+	scannerMetricLast
+)
+
+// log scanner action.
+// Use for s > scannerMetricStartTrace
+func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].add(duration)
+		}
+
+		if s > scannerMetricStartTrace && globalTrace.NumSubscribers(madmin.TraceScanner) > 0 {
+			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " ")))
+		}
+	}
+}
+
+// time a scanner action.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) time(s scannerMetric) func() {
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].add(duration)
+		}
+	}
+}
+
+// timeSize add time and size of a scanner action.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) timeSize(s scannerMetric) func(sz int) {
+	startTime := time.Now()
+	return func(sz int) {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].addSize(duration, int64(sz))
+		}
+	}
+}
+
+// incTime will increment time on metric s with a specific duration.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
+	atomic.AddUint64(&p.operations[s], 1)
+	if s < scannerMetricLastRealtime {
+		p.latency[s].add(d)
+	}
+}
+
+func (p *scannerMetrics) incNoTime(s scannerMetric) {
+	atomic.AddUint64(&p.operations[s], 1)
+	if s < scannerMetricLastRealtime {
+		p.latency[s].add(0)
+	}
+}
+
+// timeILM times an ILM action.
+// lifecycle.NoneAction is ignored.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) timeILM(a lifecycle.Action) func() {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return func() {}
+	}
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+		atomic.AddUint64(&p.actions[a], 1)
+		p.actionsLatency[a].add(duration)
+	}
+}
+
+type currentPathTracker struct {
+	name *unsafe.Pointer // contains atomically accessed *string
+}
+
+// currentPathUpdater provides a lightweight update function for keeping track of
+// current objects for each disk.
+// Returns a function that can be used to update the current object
+// and a function to call to when processing finished.
+func (p *scannerMetrics) currentPathUpdater(disk, initial string) (update func(path string), done func()) {
+	initialPtr := unsafe.Pointer(&initial)
+	tracker := &currentPathTracker{
+		name: &initialPtr,
+	}
+
+	p.currentPaths.Store(disk, tracker)
+	return func(path string) {
+			atomic.StorePointer(tracker.name, unsafe.Pointer(&path))
+		}, func() {
+			p.currentPaths.Delete(disk)
+		}
+}
+
+// getCurrentPaths returns the paths currently being processed.
+func (p *scannerMetrics) getCurrentPaths() []string {
+	var res []string
+	prefix := globalLocalNodeName + "/"
+	p.currentPaths.Range(func(key, value interface{}) bool {
+		// We are a bit paranoid, but better miss an entry than crash.
+		name, ok := key.(string)
+		if !ok {
+			return true
+		}
+		obj, ok := value.(*currentPathTracker)
+		if !ok {
+			return true
+		}
+		strptr := (*string)(atomic.LoadPointer(obj.name))
+		if strptr != nil {
+			res = append(res, pathJoin(prefix, name, *strptr))
+		}
+		return true
+	})
+	return res
+}
+
+// activeDisks returns the number of currently active disks.
+// (since this is concurrent it may not be 100% reliable)
+func (p *scannerMetrics) activeDisks() int {
+	var i int
+	p.currentPaths.Range(func(k, v interface{}) bool {
+		i++
+		return true
+	})
+	return i
+}
+
+// lifetime returns the lifetime count of the specified metric.
+func (p *scannerMetrics) lifetime(m scannerMetric) uint64 {
+	if m < 0 || m >= scannerMetricLast {
+		return 0
+	}
+	val := atomic.LoadUint64(&p.operations[m])
+	return val
+}
+
+// lastMinute returns the last minute statistics of a metric.
+// m should be < scannerMetricLastRealtime
+func (p *scannerMetrics) lastMinute(m scannerMetric) AccElem {
+	if m < 0 || m >= scannerMetricLastRealtime {
+		return AccElem{}
+	}
+	val := p.latency[m].total()
+	return val
+}
+
+// lifetimeActions returns the lifetime count of the specified ilm metric.
+func (p *scannerMetrics) lifetimeActions(a lifecycle.Action) uint64 {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return 0
+	}
+	val := atomic.LoadUint64(&p.actions[a])
+	return val
+}
+
+// lastMinuteActions returns the last minute statistics of an ilm metric.
+func (p *scannerMetrics) lastMinuteActions(a lifecycle.Action) AccElem {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return AccElem{}
+	}
+	val := p.actionsLatency[a].total()
+	return val
+}
+
+// setCycle updates the current cycle metrics.
+func (p *scannerMetrics) setCycle(c *currentScannerCycle) {
+	if c != nil {
+		c2 := c.clone()
+		c = &c2
+	}
+	p.cycleInfoMu.Lock()
+	p.cycleInfo = c
+	p.cycleInfoMu.Unlock()
+}
+
+// getCycle returns the current cycle metrics.
+// If not nil, the returned value can safely be modified.
+func (p *scannerMetrics) getCycle() *currentScannerCycle {
+	p.cycleInfoMu.Lock()
+	defer p.cycleInfoMu.Unlock()
+	if p.cycleInfo == nil {
+		return nil
+	}
+	c := p.cycleInfo.clone()
+	return &c
+}
+
+func (p *scannerMetrics) report() madmin.ScannerMetrics {
+	var m madmin.ScannerMetrics
+	cycle := p.getCycle()
+	if cycle != nil {
+		m.CurrentCycle = cycle.current
+		m.CyclesCompletedAt = cycle.cycleCompleted
+		m.CurrentStarted = cycle.started
+	}
+	m.CollectedAt = time.Now()
+	m.ActivePaths = p.getCurrentPaths()
+	m.LifeTimeOps = make(map[string]uint64, scannerMetricLast)
+	for i := scannerMetric(0); i < scannerMetricLast; i++ {
+		if n := atomic.LoadUint64(&p.operations[i]); n > 0 {
+			m.LifeTimeOps[i.String()] = n
+		}
+	}
+	if len(m.LifeTimeOps) == 0 {
+		m.LifeTimeOps = nil
+	}
+
+	m.LastMinute.Actions = make(map[string]madmin.TimedAction, scannerMetricLastRealtime)
+	for i := scannerMetric(0); i < scannerMetricLastRealtime; i++ {
+		lm := p.lastMinute(i)
+		if lm.N > 0 {
+			m.LastMinute.Actions[i.String()] = lm.asTimedAction()
+		}
+	}
+	if len(m.LastMinute.Actions) == 0 {
+		m.LastMinute.Actions = nil
+	}
+
+	// ILM
+	m.LifeTimeILM = make(map[string]uint64)
+	for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ {
+		if n := atomic.LoadUint64(&p.actions[i]); n > 0 {
+			m.LifeTimeILM[i.String()] = n
+		}
+	}
+	if len(m.LifeTimeILM) == 0 {
+		m.LifeTimeILM = nil
+	}
+
+	if len(m.LifeTimeILM) > 0 {
+		m.LastMinute.ILM = make(map[string]madmin.TimedAction, len(m.LifeTimeILM))
+		for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ {
+			lm := p.lastMinuteActions(i)
+			if lm.N > 0 {
+				m.LastMinute.ILM[i.String()] = madmin.TimedAction{Count: uint64(lm.N), AccTime: uint64(lm.Total)}
+			}
+		}
+		if len(m.LastMinute.ILM) == 0 {
+			m.LastMinute.ILM = nil
+		}
+	}
+	return m
+}

cmd/data-scanner.go

@@ -31,7 +31,6 @@ import (
 	"path"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/bits-and-blooms/bloom/v3"
@@ -66,7 +65,7 @@ var (
 
 	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper = newDynamicSleeper(10, 10*time.Second)
+	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
 	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
 )
 
@@ -162,35 +161,45 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 	// No unlock for "leader" lock.
 
 	// Load current bloom cycle
-	nextBloomCycle := intDataUpdateTracker.current() + 1
+	var cycleInfo currentScannerCycle
+	cycleInfo.next = intDataUpdateTracker.current() + 1
 
 	buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
-	if len(buf) >= 8 {
-		if err = binary.Read(bytes.NewReader(buf), binary.LittleEndian, &nextBloomCycle); err != nil {
-			logger.LogIf(ctx, err)
-		}
+	if len(buf) == 8 {
+		cycleInfo.next = binary.LittleEndian.Uint64(buf)
+	} else if len(buf) > 8 {
+		cycleInfo.next = binary.LittleEndian.Uint64(buf[:8])
+		buf = buf[8:]
+		_, err = cycleInfo.UnmarshalMsg(buf)
+		logger.LogIf(ctx, err)
 	}
 
 	scannerTimer := time.NewTimer(scannerCycle.Load())
 	defer scannerTimer.Stop()
+	defer globalScannerMetrics.setCycle(nil)
 
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-scannerTimer.C:
-			if intDataUpdateTracker.debug {
-				console.Debugln("starting scanner cycle")
-			}
+			// Reset the timer for next cycle.
+			// If scanner takes longer we start at once.
+			scannerTimer.Reset(scannerCycle.Load())
+
+			stopFn := globalScannerMetrics.log(scannerMetricScanCycle)
+			cycleInfo.current = cycleInfo.next
+			cycleInfo.started = time.Now()
+			globalScannerMetrics.setCycle(&cycleInfo)
 
 			bgHealInfo := readBackgroundHealInfo(ctx, objAPI)
-			scanMode := getCycleScanMode(nextBloomCycle, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
+			scanMode := getCycleScanMode(cycleInfo.current, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
 			if bgHealInfo.CurrentScanMode != scanMode {
 				newHealInfo := bgHealInfo
 				newHealInfo.CurrentScanMode = scanMode
 				if scanMode == madmin.HealDeepScan {
 					newHealInfo.BitrotStartTime = time.Now().UTC()
-					newHealInfo.BitrotStartCycle = nextBloomCycle
+					newHealInfo.BitrotStartCycle = cycleInfo.current
 				}
 				saveBackgroundHealInfo(ctx, objAPI, newHealInfo)
 			}
@@ -198,22 +207,27 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
+			bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle), scanMode)
+			err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
+			stopFn()
 			if err == nil {
 				// Store new cycle...
-				nextBloomCycle++
-				var tmp [8]byte
-				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
-				if err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp[:]); err != nil {
-					logger.LogIf(ctx, err)
+				cycleInfo.next++
+				cycleInfo.current = 0
+				cycleInfo.cycleCompleted = append(cycleInfo.cycleCompleted, time.Now())
+				if len(cycleInfo.cycleCompleted) > dataUsageUpdateDirCycles {
+					cycleInfo.cycleCompleted = cycleInfo.cycleCompleted[len(cycleInfo.cycleCompleted)-dataUsageUpdateDirCycles:]
 				}
+				globalScannerMetrics.setCycle(&cycleInfo)
+				tmp := make([]byte, 8, 8+cycleInfo.Msgsize())
+				// Cycle for backward compat.
+				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
+				tmp, _ = cycleInfo.MarshalMsg(tmp)
+				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
+				logger.LogIf(ctx, err)
 			}
-
-			// Reset the timer for next cycle.
-			scannerTimer.Reset(scannerCycle.Load())
 		}
 	}
 }
@@ -244,24 +258,11 @@ type folderScanner struct {
 	// Will not be closed when returned.
 	updates    chan<- dataUsageEntry
 	lastUpdate time.Time
+
+	// updateCurrentPath should be called whenever a new path is scanned.
+	updateCurrentPath func(string)
 }
 
-type scannerStats struct {
-	// All fields must be accessed atomically and aligned.
-
-	accTotalObjects  uint64
-	accTotalVersions uint64
-	accFolders       uint64
-	bucketsStarted   uint64
-	bucketsFinished  uint64
-	ilmChecks        uint64
-
-	// actions records actions performed.
-	actions [lifecycle.ActionCount]uint64
-}
-
-var globalScannerStats scannerStats
-
 // Cache structure and compaction:
 //
 // A cache structure will be kept with a tree of usages.
@@ -305,24 +306,14 @@ var globalScannerStats scannerStats
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
 func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
-	t := UTCNow()
-
 	logPrefix := color.Green("data-usage: ")
-	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
-	atomic.AddUint64(&globalScannerStats.bucketsStarted, 1)
-	defer func() {
-		atomic.AddUint64(&globalScannerStats.bucketsFinished, 1)
-	}()
-	if intDataUpdateTracker.debug {
-		defer func() {
-			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
-		}()
-	}
 
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
 	}
+	updatePath, closeDisk := globalScannerMetrics.currentPathUpdater(basePath, cache.Info.Name)
+	defer closeDisk()
 
 	s := folderScanner{
 		root:                  basePath,
@@ -335,6 +326,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		healObjectSelect:      0,
 		scanMode:              scanMode,
 		updates:               cache.Info.updates,
+		updateCurrentPath:     updatePath,
 	}
 
 	// Add disks for set healing.
@@ -366,14 +358,8 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 			s.withFilter = nil
 		}
 	}
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Start scanning. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
-	}
 
 	done := ctx.Done()
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
-	}
 
 	// Read top level in bucket.
 	select {
@@ -389,9 +375,6 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		return cache, err
 	}
 
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Finished scanner, %v entries (%+v) %s \n", len(s.newCache.Cache), *s.newCache.sizeRecursive(s.newCache.Info.Name), logSuffix)
-	}
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle = cache.Info.NextCycle
 	return s.newCache, nil
@@ -420,10 +403,10 @@ func (f *folderScanner) sendUpdate() {
 func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, into *dataUsageEntry) error {
 	done := ctx.Done()
 	scannerLogPrefix := color.Green("folder-scanner:")
+
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
-	atomic.AddUint64(&globalScannerStats.accFolders, 1)
 
 	for {
 		select {
@@ -648,7 +631,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					f.updateCache.replaceHashed(h, &thisHash, dataUsageEntry{})
 				}
 			}
+			f.updateCurrentPath(folder.name)
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
+			stopFn()
 			// Add new folders if this is new and we don't have existing.
 			if !into.Compacted {
 				parent := f.updateCache.find(thisHash.Key())
@@ -676,7 +662,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
+			f.updateCurrentPath(folder.name)
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name, "EXISTING")
 			scanFolder(folder)
+			stopFn()
 		}
 
 		// Scan for healing
@@ -717,9 +706,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		healObjectsPrefix := color.Green("healObjects:")
 		for k := range abandonedChildren {
 			bucket, prefix := path2BucketObject(k)
-			if f.dataUsageScannerDebug {
-				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
-			}
+			stopFn := globalScannerMetrics.time(scannerMetricCheckMissing)
+			f.updateCurrentPath(k)
 
 			if bucket != resolver.bucket {
 				// Bucket might be missing as well with abandoned children.
@@ -807,6 +795,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				},
 			})
 
+			stopFn()
 			if f.dataUsageScannerDebug && err != nil && err != errFileNotFound {
 				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
@@ -814,7 +803,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// Add unless healing returned an error.
 			if foundObjs {
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
+				stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, this.name, "HEALED")
 				scanFolder(this)
+				stopFn()
 			}
 		}
 		break
@@ -964,7 +955,6 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 		return false, size
 	}
 
-	atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
 	action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
@@ -975,7 +965,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
 		}
 	}
-	atomic.AddUint64(&globalScannerStats.actions[action], 1)
+	defer globalScannerMetrics.timeILM(action)
 
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
@@ -1092,16 +1082,23 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 // The metadata will be compared to consensus on the object layer before any changes are applied.
 // If no metadata is supplied, -1 is returned if no action is taken.
 func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) int64 {
+	done := globalScannerMetrics.time(scannerMetricILM)
 	applied, size := i.applyLifecycle(ctx, o, oi)
+	done()
+
 	// For instance, an applied lifecycle means we remove/transitioned an object
 	// from the current deployment, which means we don't have to call healing
 	// routine even if we are asked to do via heal flag.
 	if !applied {
 		if i.heal.enabled {
+			done := globalScannerMetrics.time(scannerMetricHealCheck)
 			size = i.applyHealing(ctx, o, oi)
+			done()
 		}
 		// replicate only if lifecycle rules are not applied.
+		done := globalScannerMetrics.time(scannerMetricCheckReplication)
 		i.healReplication(ctx, o, oi.Clone(), sizeS)
+		done()
 	}
 	return size
 }
@@ -1341,15 +1338,20 @@ type dynamicSleeper struct {
 
 	// cycle will be closed
 	cycle chan struct{}
+
+	// isScanner should be set when this is used by the scanner
+	// to record metrics.
+	isScanner bool
 }
 
 // newDynamicSleeper
-func newDynamicSleeper(factor float64, maxWait time.Duration) *dynamicSleeper {
+func newDynamicSleeper(factor float64, maxWait time.Duration, isScanner bool) *dynamicSleeper {
 	return &dynamicSleeper{
-		factor:   factor,
-		cycle:    make(chan struct{}),
-		maxSleep: maxWait,
-		minSleep: 100 * time.Microsecond,
+		factor:    factor,
+		cycle:     make(chan struct{}),
+		maxSleep:  maxWait,
+		minSleep:  100 * time.Microsecond,
+		isScanner: isScanner,
 	}
 }
 
@@ -1379,15 +1381,24 @@ func (d *dynamicSleeper) Timer(ctx context.Context) func() {
 			select {
 			case <-ctx.Done():
 				if !timer.Stop() {
+					if d.isScanner {
+						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+					}
 					<-timer.C
 				}
 				return
 			case <-timer.C:
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 				return
 			case <-cycle:
 				if !timer.Stop() {
 					// We expired.
 					<-timer.C
+					if d.isScanner {
+						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+					}
 					return
 				}
 			}
@@ -1418,14 +1429,23 @@ func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) {
 		case <-ctx.Done():
 			if !timer.Stop() {
 				<-timer.C
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 			}
 			return
 		case <-timer.C:
+			if d.isScanner {
+				globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+			}
 			return
 		case <-cycle:
 			if !timer.Stop() {
 				// We expired.
 				<-timer.C
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 				return
 			}
 		}

cmd/data-update-tracker_test.go

@@ -63,7 +63,7 @@ func (t *testingLogger) Type() types.TargetType {
 	return types.TargetHTTP
 }
 
-func (t *testingLogger) Send(entry interface{}, errKind string) error {
+func (t *testingLogger) Send(entry interface{}) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.t == nil {
@@ -75,7 +75,7 @@ func (t *testingLogger) Send(entry interface{}, errKind string) error {
 	}
 
 	t.t.Helper()
-	t.t.Log(e.Level, ":", errKind, e.Message)
+	t.t.Log(e.Level, ":", e.Message)
 	return nil
 }
 

cmd/data-usage-cache.go

@@ -1323,3 +1323,19 @@ func (z dataUsageHashMap) Msgsize() (s int) {
 	}
 	return
 }
+
+//msgp:encode ignore currentScannerCycle
+//msgp:decode ignore currentScannerCycle
+
+type currentScannerCycle struct {
+	current        uint64
+	next           uint64
+	started        time.Time
+	cycleCompleted []time.Time
+}
+
+// clone returns a clone.
+func (z currentScannerCycle) clone() currentScannerCycle {
+	z.cycleCompleted = append(make([]time.Time, 0, len(z.cycleCompleted)), z.cycleCompleted...)
+	return z
+}

cmd/data-usage-cache_gen.go

@@ -3,6 +3,8 @@ package cmd
 // Code generated by github.com/tinylib/msgp DO NOT EDIT.
 
 import (
+	"time"
+
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -284,6 +286,101 @@ func (z *allTierStats) Msgsize() (s int) {
 	return
 }
 
+// MarshalMsg implements msgp.Marshaler
+func (z *currentScannerCycle) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 4
+	// string "current"
+	o = append(o, 0x84, 0xa7, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74)
+	o = msgp.AppendUint64(o, z.current)
+	// string "next"
+	o = append(o, 0xa4, 0x6e, 0x65, 0x78, 0x74)
+	o = msgp.AppendUint64(o, z.next)
+	// string "started"
+	o = append(o, 0xa7, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
+	o = msgp.AppendTime(o, z.started)
+	// string "cycleCompleted"
+	o = append(o, 0xae, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.cycleCompleted)))
+	for za0001 := range z.cycleCompleted {
+		o = msgp.AppendTime(o, z.cycleCompleted[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *currentScannerCycle) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "current":
+			z.current, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "current")
+				return
+			}
+		case "next":
+			z.next, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "next")
+				return
+			}
+		case "started":
+			z.started, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "started")
+				return
+			}
+		case "cycleCompleted":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "cycleCompleted")
+				return
+			}
+			if cap(z.cycleCompleted) >= int(zb0002) {
+				z.cycleCompleted = (z.cycleCompleted)[:zb0002]
+			} else {
+				z.cycleCompleted = make([]time.Time, zb0002)
+			}
+			for za0001 := range z.cycleCompleted {
+				z.cycleCompleted[za0001], bts, err = msgp.ReadTimeBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "cycleCompleted", za0001)
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *currentScannerCycle) Msgsize() (s int) {
+	s = 1 + 8 + msgp.Uint64Size + 5 + msgp.Uint64Size + 8 + msgp.TimeSize + 15 + msgp.ArrayHeaderSize + (len(z.cycleCompleted) * (msgp.TimeSize))
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/data-usage-cache_gen_test.go

@@ -122,6 +122,64 @@ func BenchmarkDecodeallTierStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalcurrentScannerCycle(t *testing.T) {
+	v := currentScannerCycle{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgcurrentScannerCycle(b *testing.B) {
+	v := currentScannerCycle{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgcurrentScannerCycle(b *testing.B) {
+	v := currentScannerCycle{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalcurrentScannerCycle(b *testing.B) {
+	v := currentScannerCycle{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaldataUsageCache(t *testing.T) {
 	v := dataUsageCache{}
 	bts, err := v.MarshalMsg(nil)

cmd/endpoint-ellipses.go

@@ -42,7 +42,7 @@ type endpointSet struct {
 
 // Supported set sizes this is used to find the optimal
 // single set size.
-var setSizes = []uint64{4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+var setSizes = []uint64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
 
 // getDivisibleSize - returns a greatest common divisor of
 // all the ellipses sizes.

cmd/endpoint-ellipses_test.go

@@ -201,18 +201,6 @@ func TestGetSetIndexes(t *testing.T) {
 		success    bool
 	}{
 		// Invalid inputs.
-		{
-			[]string{"data{1...3}"},
-			[]uint64{3},
-			nil,
-			false,
-		},
-		{
-			[]string{"data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"},
-			[]uint64{2, 4, 8},
-			nil,
-			false,
-		},
 		{
 			[]string{"data{1...17}/export{1...52}"},
 			[]uint64{14144},
@@ -220,6 +208,18 @@ func TestGetSetIndexes(t *testing.T) {
 			false,
 		},
 		// Valid inputs.
+		{
+			[]string{"data{1...3}"},
+			[]uint64{3},
+			[][]uint64{{3}},
+			true,
+		},
+		{
+			[]string{"data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"},
+			[]uint64{2, 4, 8},
+			[][]uint64{{2}, {2, 2}, {2, 2, 2, 2}},
+			true,
+		},
 		{
 			[]string{"data{1...27}"},
 			[]uint64{27},

cmd/erasure-metadata.go

@@ -417,7 +417,7 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 	}
 
 	parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass])
-	if parityBlocks <= 0 {
+	if parityBlocks < 0 {
 		parityBlocks = defaultParityCount
 	}
 

cmd/erasure-multipart.go

@@ -290,7 +290,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 
 	onlineDisks := er.getDisks()
 	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
-	if parityDrives <= 0 {
+	if parityDrives < 0 {
 		parityDrives = er.defaultParityCount
 	}
 

cmd/erasure-object.go

@@ -736,7 +736,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	storageDisks := er.getDisks()
 	// Get parity and data drive count based on storage class metadata
 	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
-	if parityDrives <= 0 {
+	if parityDrives < 0 {
 		parityDrives = er.defaultParityCount
 	}
 	dataDrives := len(storageDisks) - parityDrives
@@ -885,7 +885,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	if !opts.MaxParity {
 		// Get parity and data drive count based on storage class metadata
 		parityDrives = globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
-		if parityDrives <= 0 {
+		if parityDrives < 0 {
 			parityDrives = er.defaultParityCount
 		}
 

cmd/erasure-object_test.go

@@ -893,6 +893,14 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 
 	// Object for test case 1 - No StorageClass defined, no MetaData in PutObject
 	object1 := "object1"
+	globalStorageClass = storageclass.Config{
+		RRS: storageclass.StorageClass{
+			Parity: 2,
+		},
+		Standard: storageclass.StorageClass{
+			Parity: 4,
+		},
+	}
 	_, err = obj.PutObject(ctx, bucket, object1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
@@ -964,17 +972,16 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "", false)
-	parts5SC := storageclass.Config{
-		RRS: storageclass.StorageClass{
-			Parity: 2,
-		},
-	}
+	parts5SC := globalStorageClass
 
 	// Object for test case 6 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting Standard Storage Class
 	object6 := "object6"
 	metadata6 := make(map[string]string)
 	metadata6["x-amz-storage-class"] = storageclass.STANDARD
 	globalStorageClass = storageclass.Config{
+		Standard: storageclass.StorageClass{
+			Parity: 4,
+		},
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
@@ -1035,7 +1042,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		tt := tt
 		t.(*testing.T).Run("", func(t *testing.T) {
 			globalStorageClass = tt.storageClassCfg
-			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, getDefaultParityBlocks(len(erasureDisks)))
+			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, storageclass.DefaultParityBlocks(len(erasureDisks)))
 			if tt.expectedError != nil && err == nil {
 				t.Errorf("Expected %s, got %s", tt.expectedError, err)
 			}

cmd/erasure-server-pool-decom.go

@@ -53,6 +53,9 @@ type PoolDecommissionInfo struct {
 
 	// Last bucket/object decommissioned.
 	Bucket string `json:"-" msg:"bkt"`
+	// Captures prefix that is currently being
+	// decommissioned inside the 'Bucket'
+	Prefix string `json:"-" msg:"pfx"`
 	Object string `json:"-" msg:"obj"`
 
 	// Verbose information
@@ -73,6 +76,7 @@ func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
 			// Clear tracker info.
 			if pd.Bucket == bucket {
 				pd.Bucket = "" // empty this out for next bucket
+				pd.Prefix = "" // empty this out for the next bucket
 				pd.Object = "" // empty this out for next object
 			}
 			return
@@ -80,12 +84,6 @@ func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
 	}
 }
 
-func (pd *PoolDecommissionInfo) bucketsToDecommission() []string {
-	queuedBuckets := make([]string, len(pd.QueuedBuckets))
-	copy(queuedBuckets, pd.QueuedBuckets)
-	return queuedBuckets
-}
-
 func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
 	for _, b := range pd.DecommissionedBuckets {
 		if b == bucket {
@@ -95,17 +93,18 @@ func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
 	return false
 }
 
-func (pd *PoolDecommissionInfo) bucketPush(bucket string) {
+func (pd *PoolDecommissionInfo) bucketPush(bucket decomBucketInfo) {
 	for _, b := range pd.QueuedBuckets {
 		if pd.isBucketDecommissioned(b) {
 			return
 		}
-		if b == bucket {
+		if b == bucket.String() {
 			return
 		}
 	}
-	pd.QueuedBuckets = append(pd.QueuedBuckets, bucket)
-	pd.Bucket = bucket
+	pd.QueuedBuckets = append(pd.QueuedBuckets, bucket.String())
+	pd.Bucket = bucket.Name
+	pd.Prefix = bucket.Prefix
 }
 
 // PoolStatus captures current pool status
@@ -183,12 +182,12 @@ func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool {
 	return p.Pools[idx].Decommission.isBucketDecommissioned(bucket)
 }
 
-func (p *poolMeta) BucketDone(idx int, bucket string) {
+func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) {
 	if p.Pools[idx].Decommission == nil {
 		// Decommission not in progress.
 		return
 	}
-	p.Pools[idx].Decommission.bucketPop(bucket)
+	p.Pools[idx].Decommission.bucketPop(bucket.String())
 }
 
 func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) {
@@ -208,19 +207,38 @@ func (p *poolMeta) TrackCurrentBucketObject(idx int, bucket string, object strin
 	p.Pools[idx].Decommission.Object = object
 }
 
-func (p *poolMeta) PendingBuckets(idx int) []string {
+func (p *poolMeta) PendingBuckets(idx int) []decomBucketInfo {
 	if p.Pools[idx].Decommission == nil {
 		// Decommission not in progress.
 		return nil
 	}
 
-	return p.Pools[idx].Decommission.bucketsToDecommission()
+	decomBuckets := make([]decomBucketInfo, len(p.Pools[idx].Decommission.QueuedBuckets))
+	for i := range decomBuckets {
+		bucket, prefix := path2BucketObject(p.Pools[idx].Decommission.QueuedBuckets[i])
+		decomBuckets[i] = decomBucketInfo{
+			Name:   bucket,
+			Prefix: prefix,
+		}
+	}
+
+	return decomBuckets
 }
 
-func (p *poolMeta) QueueBuckets(idx int, buckets []BucketInfo) {
+//msgp:ignore decomBucketInfo
+type decomBucketInfo struct {
+	Name   string
+	Prefix string
+}
+
+func (db decomBucketInfo) String() string {
+	return pathJoin(db.Name, db.Prefix)
+}
+
+func (p *poolMeta) QueueBuckets(idx int, buckets []decomBucketInfo) {
 	// add new queued buckets
 	for _, bucket := range buckets {
-		p.Pools[idx].Decommission.bucketPush(bucket.Name)
+		p.Pools[idx].Decommission.bucketPush(bucket)
 	}
 }
 
@@ -607,7 +625,7 @@ func (v versionsSorter) reverse() {
 	})
 }
 
-func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bName string) error {
+func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
 
 	var wg sync.WaitGroup
@@ -628,7 +646,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			continue
 		}
 
-		vc, _ := globalBucketVersioningSys.Get(bName)
+		vc, _ := globalBucketVersioningSys.Get(bi.Name)
 		decommissionEntry := func(entry metaCacheEntry) {
 			defer func() {
 				<-parallelWorkers
@@ -639,7 +657,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				return
 			}
 
-			fivs, err := entry.fileInfoVersions(bName)
+			fivs, err := entry.fileInfoVersions(bi.Name)
 			if err != nil {
 				return
 			}
@@ -652,25 +670,26 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			for _, version := range fivs.Versions {
 				// TODO: Skip transitioned objects for now.
 				if version.IsRemote() {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bName, version.Name))
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bi.Name, version.Name))
 					continue
 				}
 				// We will skip decommissioning delete markers
 				// with single version, its as good as there
 				// is no data associated with the object.
 				if version.Deleted && len(fivs.Versions) == 1 {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no content left", bName, version.Name))
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no content left", bi.Name, version.Name))
 					continue
 				}
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
-						bName,
+						bi.Name,
 						version.Name,
 						ObjectOptions{
 							Versioned:         vc.PrefixEnabled(version.Name),
 							VersionID:         version.VersionID,
 							MTime:             version.ModTime,
 							DeleteReplication: version.ReplicationState,
+							DeleteMarker:      true, // make sure we create a delete marker
 						})
 					var failure bool
 					if err != nil {
@@ -680,10 +699,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					z.poolMetaMutex.Lock()
 					z.poolMeta.CountItem(idx, 0, failure)
 					z.poolMetaMutex.Unlock()
-					if failure {
-						break // break out on first error
+					if !failure {
+						// Success keep a count.
+						decommissionedCount++
 					}
-					decommissionedCount++
 					continue
 				}
 
@@ -691,7 +710,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
 					gr, err := set.GetObjectNInfo(ctx,
-						bName,
+						bi.Name,
 						encodeDirObject(version.Name),
 						nil,
 						http.Header{},
@@ -708,7 +727,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						logger.LogIf(ctx, err)
 						continue
 					}
-					if err = z.decommissionObject(ctx, bName, gr); err != nil {
+					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
 						failure = true
 						logger.LogIf(ctx, err)
 						continue
@@ -728,19 +747,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// if all versions were decommissioned, then we can delete the object versions.
 			if decommissionedCount == len(fivs.Versions) {
 				_, err := set.DeleteObject(ctx,
-					bName,
+					bi.Name,
 					encodeDirObject(entry.name),
 					ObjectOptions{
 						DeletePrefix: true, // use prefix delete to delete all versions at once.
 					},
 				)
-				auditLogDecom(ctx, "DecomDeleteObject", bName, entry.name, "", err)
+				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 				}
 			}
 			z.poolMetaMutex.Lock()
-			z.poolMeta.TrackCurrentBucketObject(idx, bName, entry.name)
+			z.poolMeta.TrackCurrentBucketObject(idx, bi.Name, entry.name)
 			ok, err := z.poolMeta.updateAfter(ctx, idx, z.serverPools, 30*time.Second)
 			logger.LogIf(ctx, err)
 			if ok {
@@ -753,7 +772,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		resolver := metadataResolutionParams{
 			dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
 			objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
-			bucket:    bName,
+			bucket:    bi.Name,
 		}
 
 		wg.Add(1)
@@ -761,7 +780,8 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			defer wg.Done()
 			err := listPathRaw(ctx, listPathRawOptions{
 				disks:          disks,
-				bucket:         bName,
+				bucket:         bi.Name,
+				path:           bi.Prefix,
 				recursive:      true,
 				forwardTo:      "",
 				minDisks:       len(disks) / 2, // to capture all quorum ratios
@@ -791,7 +811,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
 	pool := z.serverPools[idx]
 	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
-		if z.poolMeta.isBucketDecommissioned(idx, bucket) {
+		if z.poolMeta.isBucketDecommissioned(idx, bucket.String()) {
 			if serverDebugLog {
 				console.Debugln("decommission: already done, moving on", bucket)
 			}
@@ -803,7 +823,7 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 			continue
 		}
 		if serverDebugLog {
-			console.Debugln("decommission: currently on bucket", bucket)
+			console.Debugln("decommission: currently on bucket", bucket.Name)
 		}
 		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
 			return err
@@ -921,7 +941,7 @@ func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, e
 
 	pi, err := z.getDecommissionPoolSpaceInfo(idx)
 	if err != nil {
-		return PoolStatus{}, errInvalidArgument
+		return PoolStatus{}, err
 	}
 
 	poolInfo := z.poolMeta.Pools[idx]
@@ -1031,8 +1051,15 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		return err
 	}
 
+	decomBuckets := make([]decomBucketInfo, len(buckets))
+	for i := range buckets {
+		decomBuckets[i] = decomBucketInfo{
+			Name: buckets[i].Name,
+		}
+	}
+
 	// TODO: Support decommissioning transition tiers.
-	for _, bucket := range buckets {
+	for _, bucket := range decomBuckets {
 		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
 			if lc.HasTransition() {
 				return decomError{
@@ -1059,11 +1086,13 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
-	buckets = append(buckets, BucketInfo{
-		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: minioConfigPrefix,
 	})
-	buckets = append(buckets, BucketInfo{
-		Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: bucketMetaPrefix,
 	})
 
 	var pool *erasureSets
@@ -1089,7 +1118,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 	if err = z.poolMeta.Decommission(idx, pi); err != nil {
 		return err
 	}
-	z.poolMeta.QueueBuckets(idx, buckets)
+	z.poolMeta.QueueBuckets(idx, decomBuckets)
 	if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 		return err
 	}

cmd/erasure-server-pool-decom_gen.go

@@ -110,6 +110,12 @@ func (z *PoolDecommissionInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Bucket")
 				return
 			}
+		case "pfx":
+			z.Prefix, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
 		case "obj":
 			z.Object, err = dc.ReadString()
 			if err != nil {
@@ -153,9 +159,9 @@ func (z *PoolDecommissionInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *PoolDecommissionInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 15
+	// map header, size 16
 	// write "st"
-	err = en.Append(0x8f, 0xa2, 0x73, 0x74)
+	err = en.Append(0xde, 0x0, 0x10, 0xa2, 0x73, 0x74)
 	if err != nil {
 		return
 	}
@@ -268,6 +274,16 @@ func (z *PoolDecommissionInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Bucket")
 		return
 	}
+	// write "pfx"
+	err = en.Append(0xa3, 0x70, 0x66, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Prefix)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
 	// write "obj"
 	err = en.Append(0xa3, 0x6f, 0x62, 0x6a)
 	if err != nil {
@@ -324,9 +340,9 @@ func (z *PoolDecommissionInfo) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *PoolDecommissionInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 15
+	// map header, size 16
 	// string "st"
-	o = append(o, 0x8f, 0xa2, 0x73, 0x74)
+	o = append(o, 0xde, 0x0, 0x10, 0xa2, 0x73, 0x74)
 	o = msgp.AppendTime(o, z.StartTime)
 	// string "ss"
 	o = append(o, 0xa2, 0x73, 0x73)
@@ -361,6 +377,9 @@ func (z *PoolDecommissionInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "bkt"
 	o = append(o, 0xa3, 0x62, 0x6b, 0x74)
 	o = msgp.AppendString(o, z.Bucket)
+	// string "pfx"
+	o = append(o, 0xa3, 0x70, 0x66, 0x78)
+	o = msgp.AppendString(o, z.Prefix)
 	// string "obj"
 	o = append(o, 0xa3, 0x6f, 0x62, 0x6a)
 	o = msgp.AppendString(o, z.Object)
@@ -483,6 +502,12 @@ func (z *PoolDecommissionInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Bucket")
 				return
 			}
+		case "pfx":
+			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
 		case "obj":
 			z.Object, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
@@ -527,7 +552,7 @@ func (z *PoolDecommissionInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *PoolDecommissionInfo) Msgsize() (s int) {
-	s = 1 + 3 + msgp.TimeSize + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.BoolSize + 3 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.ArrayHeaderSize
+	s = 3 + 3 + msgp.TimeSize + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.BoolSize + 3 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.QueuedBuckets {
 		s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001])
 	}
@@ -535,7 +560,7 @@ func (z *PoolDecommissionInfo) Msgsize() (s int) {
 	for za0002 := range z.DecommissionedBuckets {
 		s += msgp.StringPrefixSize + len(z.DecommissionedBuckets[za0002])
 	}
-	s += 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
+	s += 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Prefix) + 4 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
 	return
 }
 

cmd/erasure-server-pool.go

@@ -119,7 +119,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		}
 
 		if deploymentID == "" {
-			// all zones should have same deployment ID
+			// all pools should have same deployment ID
 			deploymentID = formats[i].ID
 		}
 
@@ -238,6 +238,7 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 	return nil
 }
 
+// Return the count of disks in each pool
 func (z *erasureServerPools) SetDriveCounts() []int {
 	setDriveCounts := make([]int, len(z.serverPools))
 	for i := range z.serverPools {
@@ -457,7 +458,7 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 // If the object exists, but the latest version is a delete marker, the index with it is still returned.
 // If the object does not exist ObjectNotFound error is returned.
 // If any other error is found, it is returned.
-// The check is skipped if there is only one zone, and 0, nil is always returned in that case.
+// The check is skipped if there is only one pool, and 0, nil is always returned in that case.
 func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
 	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{
 		NoLock: true,
@@ -525,7 +526,7 @@ func (z *erasureServerPools) BackendInfo() (b madmin.BackendInfo) {
 	b.Type = madmin.Erasure
 
 	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-	if scParity <= 0 {
+	if scParity < 0 {
 		scParity = z.serverPools[0].defaultParityCount
 	}
 	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
@@ -881,7 +882,7 @@ func (z *erasureServerPools) getLatestObjectInfoWithIdx(ctx context.Context, buc
 	sort.Slice(results, func(i, j int) bool {
 		a, b := results[i], results[j]
 		if a.oi.ModTime.Equal(b.oi.ModTime) {
-			// On tiebreak, select the lowest zone index.
+			// On tiebreak, select the lowest pool index.
 			return a.zIdx < b.zIdx
 		}
 		return a.oi.ModTime.After(b.oi.ModTime)
@@ -975,12 +976,12 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 }
 
 func (z *erasureServerPools) deletePrefix(ctx context.Context, bucket string, prefix string) error {
-	for idx, zone := range z.serverPools {
+	for idx, pool := range z.serverPools {
 		if z.IsSuspended(idx) {
 			logger.LogIf(ctx, fmt.Errorf("pool %d is suspended, all writes are suspended", idx+1))
 			continue
 		}
-		_, err := zone.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
+		_, err := pool.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
 		if err != nil {
 			return err
 		}
@@ -1003,7 +1004,6 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return objInfo, err
@@ -1053,7 +1053,9 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		j := j
 		obj := obj
 		eg.Go(func() error {
-			idx, err := z.getPoolIdxExistingNoLock(ctx, bucket, obj.ObjectName)
+			idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, obj.ObjectName, ObjectOptions{
+				NoLock: true,
+			})
 			if err != nil {
 				derrs[j] = err
 				return nil
@@ -2208,7 +2210,6 @@ func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, obje
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2225,7 +2226,6 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2241,7 +2241,6 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2273,7 +2272,7 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 		return z.serverPools[0].TransitionObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
+	opts.Mutate = true // Avoid transitioning an object from a pool being decommissioned.
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
@@ -2289,7 +2288,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
+	opts.Mutate = true // Avoid restoring object from a pool being decommissioned.
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err

cmd/erasure-sets.go

@@ -466,7 +466,7 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 				getDisks:              s.GetDisks(i),
 				getLockers:            s.GetLockers(i),
 				getEndpoints:          s.GetEndpoints(i),
-				deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
+				deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second, false),
 				nsMutex:               mutex,
 				bp:                    bp,
 				bpOld:                 bpOld,

cmd/erasure-single-drive.go

@@ -90,7 +90,7 @@ func newErasureSingle(ctx context.Context, storageDisk StorageAPI, format *forma
 		format:                format,
 		nsMutex:               newNSLock(false),
 		bp:                    bp,
-		deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
+		deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second, false),
 	}
 
 	// start cleanup stale uploads go-routine.
@@ -184,6 +184,10 @@ func (es *erasureSingle) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+func (es *erasureSingle) SetDriveCounts() []int {
+	return []int{1}
+}
+
 func (es *erasureSingle) BackendInfo() (b madmin.BackendInfo) {
 	b.Type = madmin.Erasure
 

cmd/erasure.go

@@ -213,11 +213,13 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				}
 			}
 			di.Metrics = &madmin.DiskMetrics{
-				APILatencies: make(map[string]interface{}),
-				APICalls:     make(map[string]uint64),
+				LastMinute: make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),
+				APICalls:   make(map[string]uint64, len(info.Metrics.APICalls)),
 			}
-			for k, v := range info.Metrics.APILatencies {
-				di.Metrics.APILatencies[k] = v
+			for k, v := range info.Metrics.LastMinute {
+				if v.N > 0 {
+					di.Metrics.LastMinute[k] = v.asTimedAction()
+				}
 			}
 			for k, v := range info.Metrics.APICalls {
 				di.Metrics.APICalls[k] = v

cmd/format-erasure.go

@@ -643,7 +643,7 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 		}, index)
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
+	writeQuorum := (len(storageDisks) + 1/2)
 	// Wait for the routines to finish.
 	return reduceWriteQuorumErrs(ctx, g.Wait(), nil, writeQuorum)
 }
@@ -805,26 +805,13 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 	return getFormatErasureInQuorum(formats)
 }
 
-func getDefaultParityBlocks(drive int) int {
-	switch drive {
-	case 3, 2:
-		return 1
-	case 4, 5:
-		return 2
-	case 6, 7:
-		return 3
-	default:
-		return 4
-	}
-}
-
 // ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
 // It will attempt to read it from env variable and fall back to drives/2.
 func ecDrivesNoConfig(setDriveCount int) int {
 	sc, _ := storageclass.LookupConfig(config.KVS{}, setDriveCount)
 	ecDrives := sc.GetParityForSC(storageclass.STANDARD)
-	if ecDrives <= 0 {
-		ecDrives = getDefaultParityBlocks(setDriveCount)
+	if ecDrives < 0 {
+		ecDrives = storageclass.DefaultParityBlocks(setDriveCount)
 	}
 	return ecDrives
 }

cmd/fs-v1.go

@@ -31,7 +31,6 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -210,7 +209,13 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
 
 // BackendInfo - returns backend information
 func (fs *FSObjects) BackendInfo() madmin.BackendInfo {
-	return madmin.BackendInfo{Type: madmin.FS}
+	return madmin.BackendInfo{
+		Type:             madmin.FS,
+		StandardSCData:   []int{1},
+		StandardSCParity: 0,
+		RRSCData:         []int{1},
+		RRSCParity:       0,
+	}
 }
 
 // LocalStorageInfo - returns underlying storage statistics.
@@ -235,7 +240,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 			},
 		},
 	}
-	storageInfo.Backend.Type = madmin.FS
+	storageInfo.Backend = fs.BackendInfo()
 	return storageInfo, nil
 }
 
@@ -349,6 +354,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 // A partially updated bucket may be returned.
 func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUsageCache) (dataUsageCache, error) {
 	defer close(cache.Info.updates)
+	defer globalScannerMetrics.log(scannerMetricScanBucketDisk, fs.fsPath, bucket)()
 
 	// Get bucket policy
 	// Check if the current bucket has a configured lifecycle policy
@@ -363,6 +369,16 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 	// Load bucket info.
 	cache, err = scanDataFolder(ctx, -1, -1, fs.fsPath, cache, func(item scannerItem) (sizeSummary, error) {
 		bucket, object := item.bucket, item.objectPath()
+		stopFn := globalScannerMetrics.log(scannerMetricScanObject, fs.fsPath, PathJoin(item.bucket, item.objectPath()))
+		defer stopFn()
+
+		var fsMetaBytes []byte
+		done := globalScannerMetrics.timeSize(scannerMetricReadMetadata)
+		defer func() {
+			if done != nil {
+				done(len(fsMetaBytes))
+			}
+		}()
 		fsMetaBytes, err := xioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile))
 		if err != nil && !osIsNotExist(err) {
 			if intDataUpdateTracker.debug {
@@ -391,11 +407,16 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 			}
 			return sizeSummary{}, errSkipFile
 		}
+		done(len(fsMetaBytes))
+		done = nil
+
+		// FS has no "all versions". Increment the counter, though
+		globalScannerMetrics.incNoTime(scannerMetricApplyAll)
 
 		oi := fsMeta.ToObjectInfo(bucket, object, fi)
-		atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
-		atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
+		doneVer := globalScannerMetrics.time(scannerMetricApplyVersion)
 		sz := item.applyActions(ctx, fs, oi, &sizeSummary{})
+		doneVer()
 		if sz >= 0 {
 			return sizeSummary{totalSize: sz, versions: 1}, nil
 		}

cmd/gateway-common.go

@@ -297,6 +297,10 @@ func ErrorRespToObjectError(err error, params ...string) error {
 	}
 
 	switch minioErr.Code {
+	case "PreconditionFailed":
+		err = PreConditionFailed{}
+	case "InvalidRange":
+		err = InvalidRange{}
 	case "BucketAlreadyOwnedByYou":
 		err = BucketAlreadyOwnedByYou{}
 	case "BucketNotEmpty":

cmd/gateway-startup-msg.go

@@ -27,6 +27,12 @@ import (
 
 // Prints the formatted startup message.
 func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
+	if len(globalSubnetConfig.APIKey) == 0 {
+		var builder strings.Builder
+		startupBanner(&builder)
+		logger.Info("\n" + builder.String())
+	}
+
 	strippedAPIEndpoints := stripStandardPorts(apiEndPoints, globalMinioHost)
 	// If cache layer is enabled, print cache capacity.
 	cacheAPI := newCachedObjectLayerFn()

cmd/generic-handlers.go

@@ -97,12 +97,25 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
 // Limits body and header to specific allowed maximum limits as per S3/MinIO API requirements.
 func setRequestLimitHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		// Reject unsupported reserved metadata first before validation.
 		if containsReservedMetadata(r.Header) {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL)
 			return
 		}
+
 		if isHTTPHeaderSizeTooLarge(r.Header) {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1)
 			return
@@ -373,7 +386,14 @@ func hasMultipleAuth(r *http.Request) bool {
 // any malicious requests.
 func setRequestValidityHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		if err := hasBadHost(r.Host); err != nil {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (%s)", invalidReq.Description, err)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -383,6 +403,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 
 		// Check for bad components in URL path.
 		if hasBadPathComponent(r.URL.Path) {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 			return
@@ -391,6 +416,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 		for _, vv := range r.Form {
 			for _, v := range vv {
 				if hasBadPathComponent(v) {
+					if ok {
+						tc.funcName = "handler.ValidRequest"
+						tc.responseRecorder.LogErrBody = true
+					}
+
 					writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 					atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 					return
@@ -398,6 +428,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 			}
 		}
 		if hasMultipleAuth(r) {
+			if ok {
+				tc.funcName = "handler.Auth"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (request has multiple authentication types, please use one)", invalidReq.Description)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -408,6 +443,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 		bucketName, _ := request2BucketObjectName(r)
 		if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
 			if !guessIsRPCReq(r) && !guessIsBrowserReq(r) && !guessIsHealthCheckReq(r) && !guessIsMetricsReq(r) && !isAdminReq(r) {
+				if ok {
+					tc.funcName = "handler.ValidRequest"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
 				return
 			}
@@ -415,8 +455,18 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 		// Deny SSE-C requests if not made over TLS
 		if !globalIsTLS && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
 			if r.Method == http.MethodHead {
+				if ok {
+					tc.funcName = "handler.ValidRequest"
+					tc.responseRecorder.LogErrBody = false
+				}
+
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
 			} else {
+				if ok {
+					tc.funcName = "handler.ValidRequest"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL)
 			}
 			return

cmd/globals.go

@@ -230,6 +230,7 @@ var (
 	globalTrace = pubsub.New(8)
 
 	// global Listen system to send S3 API events to registered listeners
+	// Objects are expected to be event.Event
 	globalHTTPListen = pubsub.New(0)
 
 	// global console system to send console logs to
@@ -373,6 +374,9 @@ var (
 	globalObjectPerfBucket       = "minio-perf-test-tmp-bucket"
 	globalObjectPerfUserMetadata = "X-Amz-Meta-Minio-Object-Perf" // Clients can set this to bypass S3 API service freeze. Used by object pref tests.
 
+	// MinIO version unix timestamp
+	globalVersionUnix uint64
+
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -116,7 +116,9 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
 		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
 		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
-		logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
+		if globalIsDistErasure {
+			logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
+		}
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
 		if len(globalEndpoints.Hostnames()) > 0 {

cmd/http-tracer.go

@@ -97,13 +97,14 @@ func getOpName(name string) (op string) {
 	op = strings.TrimSuffix(op, "Handler-fm")
 	op = strings.Replace(op, "objectAPIHandlers", "s3", 1)
 	op = strings.Replace(op, "adminAPIHandlers", "admin", 1)
-	op = strings.Replace(op, "(*webAPIHandlers)", "web", 1)
-	op = strings.Replace(op, "(*storageRESTServer)", "internal", 1)
-	op = strings.Replace(op, "(*peerRESTServer)", "internal", 1)
-	op = strings.Replace(op, "(*lockRESTServer)", "internal", 1)
+	op = strings.Replace(op, "(*storageRESTServer)", "storageR", 1)
+	op = strings.Replace(op, "(*peerRESTServer)", "peer", 1)
+	op = strings.Replace(op, "(*lockRESTServer)", "lockR", 1)
 	op = strings.Replace(op, "(*stsAPIHandlers)", "sts", 1)
-	op = strings.Replace(op, "LivenessCheckHandler", "healthcheck", 1)
-	op = strings.Replace(op, "ReadinessCheckHandler", "healthcheck", 1)
+	op = strings.Replace(op, "ClusterCheckHandler", "health.Cluster", 1)
+	op = strings.Replace(op, "ClusterReadCheckHandler", "health.ClusterRead", 1)
+	op = strings.Replace(op, "LivenessCheckHandler", "health.Liveness", 1)
+	op = strings.Replace(op, "ReadinessCheckHandler", "health.Readiness", 1)
 	op = strings.Replace(op, "-fm", "", 1)
 	return op
 }
@@ -124,7 +125,7 @@ type traceCtxt struct {
 // otherwise, generate a trace event with request information but no response.
 func httpTracer(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if globalTrace.NumSubscribers() == 0 {
+		if globalTrace.NumSubscribers(madmin.TraceS3|madmin.TraceInternal) == 0 {
 			h.ServeHTTP(w, r)
 			return
 		}
@@ -148,6 +149,14 @@ func httpTracer(h http.Handler) http.Handler {
 		h.ServeHTTP(respRecorder, r)
 		reqEndTime := time.Now().UTC()
 
+		tt := madmin.TraceInternal
+		if strings.HasPrefix(tc.funcName, "s3.") {
+			tt = madmin.TraceS3
+		}
+		// No need to continue if no subscribers for actual type...
+		if globalTrace.NumSubscribers(tt) == 0 {
+			return
+		}
 		// Calculate input body size with headers
 		reqHeaders := r.Header.Clone()
 		reqHeaders.Set("Host", r.Host)
@@ -163,7 +172,7 @@ func httpTracer(h http.Handler) http.Handler {
 
 		// Calculate node name
 		nodeName := r.Host
-		if nodeName == "" || globalIsDistErasure {
+		if globalIsDistErasure {
 			nodeName = globalLocalNodeName
 		}
 		if host, port, err := net.SplitHostPort(nodeName); err == nil {
@@ -184,39 +193,37 @@ func httpTracer(h http.Handler) http.Handler {
 			funcName = "<unknown>"
 		}
 
-		rq := madmin.TraceRequestInfo{
-			Time:     reqStartTime,
-			Proto:    r.Proto,
-			Method:   r.Method,
-			RawQuery: redactLDAPPwd(r.URL.RawQuery),
-			Client:   handlers.GetSourceIP(r),
-			Headers:  reqHeaders,
-			Path:     reqPath,
-			Body:     reqRecorder.Data(),
-		}
-
-		rs := madmin.TraceResponseInfo{
-			Time:       reqEndTime,
-			Headers:    respRecorder.Header().Clone(),
-			StatusCode: respRecorder.StatusCode,
-			Body:       respRecorder.Body(),
-		}
-
-		cs := madmin.TraceCallStats{
-			Latency:         rs.Time.Sub(respRecorder.StartTime),
-			InputBytes:      inputBytes,
-			OutputBytes:     respRecorder.Size(),
-			TimeToFirstByte: respRecorder.TimeToFirstByte,
-		}
-
 		t := madmin.TraceInfo{
-			TraceType: madmin.TraceHTTP,
+			TraceType: tt,
 			FuncName:  funcName,
 			NodeName:  nodeName,
 			Time:      reqStartTime,
-			ReqInfo:   rq,
-			RespInfo:  rs,
-			CallStats: cs,
+			Duration:  reqEndTime.Sub(respRecorder.StartTime),
+			Path:      reqPath,
+			HTTP: &madmin.TraceHTTPStats{
+				ReqInfo: madmin.TraceRequestInfo{
+					Time:     reqStartTime,
+					Proto:    r.Proto,
+					Method:   r.Method,
+					RawQuery: redactLDAPPwd(r.URL.RawQuery),
+					Client:   handlers.GetSourceIP(r),
+					Headers:  reqHeaders,
+					Path:     reqPath,
+					Body:     reqRecorder.Data(),
+				},
+				RespInfo: madmin.TraceResponseInfo{
+					Time:       reqEndTime,
+					Headers:    respRecorder.Header().Clone(),
+					StatusCode: respRecorder.StatusCode,
+					Body:       respRecorder.Body(),
+				},
+				CallStats: madmin.TraceCallStats{
+					Latency:         reqEndTime.Sub(respRecorder.StartTime),
+					InputBytes:      inputBytes,
+					OutputBytes:     respRecorder.Size(),
+					TimeToFirstByte: respRecorder.TimeToFirstByte,
+				},
+			},
 		}
 
 		globalTrace.Publish(t)

cmd/iam-dummy-store.go

@@ -20,8 +20,6 @@ package cmd
 import (
 	"context"
 	"sync"
-
-	"github.com/minio/minio/internal/auth"
 )
 
 type iamDummyStore struct {
@@ -79,7 +77,7 @@ func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]Polic
 	return nil
 }
 
-func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	u, ok := ids.iamUsersMap[user]
 	if !ok {
 		return errNoSuchUser
@@ -88,7 +86,7 @@ func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IA
 	return nil
 }
 
-func (ids *iamDummyStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ids *iamDummyStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
 	for k, v := range ids.iamUsersMap {
 		m[k] = v
 	}

cmd/iam-etcd-store.go

@@ -336,7 +336,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]Policy
 	return nil
 }
 
-func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]auth.Credentials, basePrefix string) error {
+func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
 	var u UserIdentity
 	err := getIAMConfig(&u, userkv.Value, string(userkv.Key))
 	if err != nil {
@@ -349,7 +349,7 @@ func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue,
 	return ies.addUser(ctx, user, userType, u, m)
 }
 
-func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]UserIdentity) error {
 	if u.Credentials.IsExpired() {
 		// Delete expired identity.
 		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
@@ -359,11 +359,11 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
-	m[user] = u.Credentials
+	m[user] = u
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	var u UserIdentity
 	err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
 	if err != nil {
@@ -375,7 +375,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
 	return ies.addUser(ctx, user, userType, u, m)
 }
 
-func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
 	var basePrefix string
 	switch userType {
 	case svcUser:

cmd/iam-object-store.go

@@ -287,7 +287,7 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]Po
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	var u UserIdentity
 	err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
 	if err != nil {
@@ -308,11 +308,11 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 		u.Credentials.AccessKey = user
 	}
 
-	m[user] = u.Credentials
+	m[user] = u
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
 	var basePrefix string
 	switch userType {
 	case svcUser:

cmd/iam-store.go

@@ -249,7 +249,7 @@ type iamCache struct {
 	// map of policy names to policy definitions
 	iamPolicyDocsMap map[string]PolicyDoc
 	// map of usernames to credentials
-	iamUsersMap map[string]auth.Credentials
+	iamUsersMap map[string]UserIdentity
 	// map of group names to group info
 	iamGroupsMap map[string]GroupInfo
 	// map of user names to groups they are a member of
@@ -263,7 +263,7 @@ type iamCache struct {
 func newIamCache() *iamCache {
 	return &iamCache{
 		iamPolicyDocsMap:        map[string]PolicyDoc{},
-		iamUsersMap:             map[string]auth.Credentials{},
+		iamUsersMap:             map[string]UserIdentity{},
 		iamGroupsMap:            map[string]GroupInfo{},
 		iamUserGroupMemberships: map[string]set.StringSet{},
 		iamUserPolicyMap:        map[string]MappedPolicy{},
@@ -346,16 +346,16 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 	var parentName string
 	u, ok := c.iamUsersMap[name]
 	if ok {
-		if !u.IsValid() {
+		if !u.Credentials.IsValid() {
 			return nil, time.Time{}, nil
 		}
-		parentName = u.ParentUser
+		parentName = u.Credentials.ParentUser
 	}
 
 	mp, ok := c.iamUserPolicyMap[name]
 	if !ok {
 		// Service accounts with root credentials, inherit parent permissions
-		if parentName == globalActiveCred.AccessKey && u.IsServiceAccount() {
+		if parentName == globalActiveCred.AccessKey && u.Credentials.IsServiceAccount() {
 			// even if this is set, the claims present in the service
 			// accounts apply the final permissions if any.
 			return []string{"consoleAdmin"}, mp.UpdatedAt, nil
@@ -395,8 +395,8 @@ type IAMStorageAPI interface {
 	getUsersSysType() UsersSysType
 	loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error
 	loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error
-	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error
-	loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error
+	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error
+	loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error
 	loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error
 	loadGroups(ctx context.Context, m map[string]GroupInfo) error
 	loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
@@ -526,12 +526,12 @@ func (store *IAMStoreSys) HasWatcher() bool {
 }
 
 // GetUser - fetches credential from memory.
-func (store *IAMStoreSys) GetUser(user string) (auth.Credentials, bool) {
+func (store *IAMStoreSys) GetUser(user string) (UserIdentity, bool) {
 	cache := store.rlock()
 	defer store.runlock()
 
-	c, ok := cache.iamUsersMap[user]
-	return c, ok
+	u, ok := cache.iamUsersMap[user]
+	return u, ok
 }
 
 // GetMappedPolicy - fetches mapped policy from memory.
@@ -614,9 +614,9 @@ func (store *IAMStoreSys) PolicyDBGet(name string, isGroup bool, groups ...strin
 }
 
 // AddUsersToGroup - adds users to group, creating the group if needed.
-func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, members []string) error {
+func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if group == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -624,12 +624,13 @@ func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, mem
 
 	// Validate that all members exist.
 	for _, member := range members {
-		cr, ok := cache.iamUsersMap[member]
+		u, ok := cache.iamUsersMap[member]
 		if !ok {
-			return errNoSuchUser
+			return updatedAt, errNoSuchUser
 		}
+		cr := u.Credentials
 		if cr.IsTemp() || cr.IsServiceAccount() {
-			return errIAMActionNotAllowed
+			return updatedAt, errIAMActionNotAllowed
 		}
 	}
 
@@ -640,10 +641,11 @@ func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, mem
 		gi = newGroupInfo(members)
 	} else {
 		gi.Members = set.CreateStringSet(append(gi.Members, members...)...).ToSlice()
+		gi.UpdatedAt = UTCNow()
 	}
 
 	if err := store.saveGroupInfo(ctx, group, gi); err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	cache.iamGroupsMap[group] = gi
@@ -660,16 +662,15 @@ func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, mem
 	}
 
 	cache.updatedAt = time.Now()
-
-	return nil
+	return gi.UpdatedAt, nil
 }
 
 // helper function - does not take any locks. Updates only cache if
 // updateCacheOnly is set.
-func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamCache, group string, members []string, updateCacheOnly bool) error {
+func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamCache, group string, members []string, updateCacheOnly bool) (updatedAt time.Time, err error) {
 	gi, ok := cache.iamGroupsMap[group]
 	if !ok {
-		return errNoSuchGroup
+		return updatedAt, errNoSuchGroup
 	}
 
 	s := set.CreateStringSet(gi.Members...)
@@ -679,9 +680,10 @@ func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamC
 	if !updateCacheOnly {
 		err := store.saveGroupInfo(ctx, group, gi)
 		if err != nil {
-			return err
+			return updatedAt, err
 		}
 	}
+	gi.UpdatedAt = UTCNow()
 	cache.iamGroupsMap[group] = gi
 
 	// update user-group membership map
@@ -695,13 +697,13 @@ func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamC
 	}
 
 	cache.updatedAt = time.Now()
-	return nil
+	return gi.UpdatedAt, nil
 }
 
 // RemoveUsersFromGroup - removes users from group, deleting it if it is empty.
-func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) error {
+func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if group == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -709,23 +711,24 @@ func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string
 
 	// Validate that all members exist.
 	for _, member := range members {
-		cr, ok := cache.iamUsersMap[member]
+		u, ok := cache.iamUsersMap[member]
 		if !ok {
-			return errNoSuchUser
+			return updatedAt, errNoSuchUser
 		}
+		cr := u.Credentials
 		if cr.IsTemp() || cr.IsServiceAccount() {
-			return errIAMActionNotAllowed
+			return updatedAt, errIAMActionNotAllowed
 		}
 	}
 
 	gi, ok := cache.iamGroupsMap[group]
 	if !ok {
-		return errNoSuchGroup
+		return updatedAt, errNoSuchGroup
 	}
 
 	// Check if attempting to delete a non-empty group.
 	if len(members) == 0 && len(gi.Members) != 0 {
-		return errGroupNotEmpty
+		return updatedAt, errGroupNotEmpty
 	}
 
 	if len(members) == 0 {
@@ -734,26 +737,26 @@ func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string
 		// Remove the group from storage. First delete the
 		// mapped policy. No-mapped-policy case is ignored.
 		if err := store.deleteMappedPolicy(ctx, group, regUser, true); err != nil && err != errNoSuchPolicy {
-			return err
+			return updatedAt, err
 		}
 		if err := store.deleteGroupInfo(ctx, group); err != nil && err != errNoSuchGroup {
-			return err
+			return updatedAt, err
 		}
 
 		// Delete from server memory
 		delete(cache.iamGroupsMap, group)
 		delete(cache.iamGroupPolicyMap, group)
 		cache.updatedAt = time.Now()
-		return nil
+		return cache.updatedAt, nil
 	}
 
 	return removeMembersFromGroup(ctx, store, cache, group, members, false)
 }
 
 // SetGroupStatus - updates group status
-func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enabled bool) error {
+func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enabled bool) (updatedAt time.Time, err error) {
 	if group == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -761,7 +764,7 @@ func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enab
 
 	gi, ok := cache.iamGroupsMap[group]
 	if !ok {
-		return errNoSuchGroup
+		return updatedAt, errNoSuchGroup
 	}
 
 	if enabled {
@@ -769,15 +772,15 @@ func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enab
 	} else {
 		gi.Status = statusDisabled
 	}
-
+	gi.UpdatedAt = UTCNow()
 	if err := store.saveGroupInfo(ctx, group, gi); err != nil {
-		return err
+		return gi.UpdatedAt, err
 	}
 
 	cache.iamGroupsMap[group] = gi
 	cache.updatedAt = time.Now()
 
-	return nil
+	return gi.UpdatedAt, nil
 }
 
 // GetGroupDescription - builds up group description
@@ -851,9 +854,9 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 
 // PolicyDBSet - update the policy mapping for the given user or group in
 // storage and in cache.
-func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) error {
+func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) (updatedAt time.Time, err error) {
 	if name == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -863,11 +866,11 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 	if store.getUsersSysType() == MinIOUsersSysType {
 		if !isGroup {
 			if _, ok := cache.iamUsersMap[name]; !ok {
-				return errNoSuchUser
+				return updatedAt, errNoSuchUser
 			}
 		} else {
 			if _, ok := cache.iamGroupsMap[name]; !ok {
-				return errNoSuchGroup
+				return updatedAt, errNoSuchGroup
 			}
 		}
 	}
@@ -882,7 +885,7 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 		}
 		err := store.deleteMappedPolicy(ctx, name, userType, isGroup)
 		if err != nil && err != errNoSuchPolicy {
-			return err
+			return updatedAt, err
 		}
 		if !isGroup {
 			delete(cache.iamUserPolicyMap, name)
@@ -890,8 +893,7 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 			delete(cache.iamGroupPolicyMap, name)
 		}
 		cache.updatedAt = time.Now()
-
-		return nil
+		return cache.updatedAt, nil
 	}
 
 	// Handle policy mapping set/update
@@ -899,12 +901,12 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 	for _, p := range mp.toSlice() {
 		if _, found := cache.iamPolicyDocsMap[p]; !found {
 			logger.LogIf(GlobalContext, fmt.Errorf("%w: (%s)", errNoSuchPolicy, p))
-			return errNoSuchPolicy
+			return updatedAt, errNoSuchPolicy
 		}
 	}
 
 	if err := store.saveMappedPolicy(ctx, name, userType, isGroup, mp); err != nil {
-		return err
+		return updatedAt, err
 	}
 	if !isGroup {
 		cache.iamUserPolicyMap[name] = mp
@@ -912,7 +914,7 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 		cache.iamGroupPolicyMap[name] = mp
 	}
 	cache.updatedAt = time.Now()
-	return nil
+	return mp.UpdatedAt, nil
 }
 
 // PolicyNotificationHandler - loads given policy from storage. If not present,
@@ -1068,9 +1070,9 @@ func (store *IAMStoreSys) GetPolicyDoc(name string) (r PolicyDoc, err error) {
 }
 
 // SetPolicy - creates a policy with name.
-func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iampolicy.Policy) error {
+func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iampolicy.Policy) (time.Time, error) {
 	if policy.IsEmpty() || name == "" {
-		return errInvalidArgument
+		return time.Time{}, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -1087,13 +1089,13 @@ func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iam
 	}
 
 	if err := store.savePolicyDoc(ctx, name, d); err != nil {
-		return err
+		return d.UpdateDate, err
 	}
 
 	cache.iamPolicyDocsMap[name] = d
 	cache.updatedAt = time.Now()
 
-	return nil
+	return d.UpdateDate, nil
 }
 
 // ListPolicies - fetches all policies from storage and updates cache as well.
@@ -1198,7 +1200,8 @@ func (store *IAMStoreSys) GetBucketUsers(bucket string) (map[string]madmin.UserI
 
 	result := map[string]madmin.UserInfo{}
 	for k, v := range cache.iamUsersMap {
-		if v.IsTemp() || v.IsServiceAccount() {
+		c := v.Credentials
+		if c.IsTemp() || c.IsServiceAccount() {
 			continue
 		}
 		var policies []string
@@ -1216,7 +1219,7 @@ func (store *IAMStoreSys) GetBucketUsers(bucket string) (map[string]madmin.UserI
 			result[k] = madmin.UserInfo{
 				PolicyName: matchedPolicies,
 				Status: func() madmin.AccountStatus {
-					if v.IsValid() {
+					if c.IsValid() {
 						return madmin.AccountEnabled
 					}
 					return madmin.AccountDisabled
@@ -1235,7 +1238,9 @@ func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo {
 	defer store.runlock()
 
 	result := map[string]madmin.UserInfo{}
-	for k, v := range cache.iamUsersMap {
+	for k, u := range cache.iamUsersMap {
+		v := u.Credentials
+
 		if v.IsTemp() || v.IsServiceAccount() {
 			continue
 		}
@@ -1281,8 +1286,8 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 		// return that info. Otherwise we return error.
 		var groups []string
 		for _, v := range cache.iamUsersMap {
-			if v.ParentUser == name {
-				groups = v.Groups
+			if v.Credentials.ParentUser == name {
+				groups = v.Credentials.Groups
 				break
 			}
 		}
@@ -1297,11 +1302,11 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 		}, nil
 	}
 
-	cred, found := cache.iamUsersMap[name]
+	ui, found := cache.iamUsersMap[name]
 	if !found {
 		return u, errNoSuchUser
 	}
-
+	cred := ui.Credentials
 	if cred.IsTemp() || cred.IsServiceAccount() {
 		return u, errIAMActionNotAllowed
 	}
@@ -1363,7 +1368,7 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		if store.getUsersSysType() == MinIOUsersSysType {
 			memberOf := cache.iamUserGroupMemberships[accessKey].ToSlice()
 			for _, group := range memberOf {
-				removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, true)
+				_, removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, true)
 				if removeErr == errNoSuchGroup {
 					removeErr = nil
 				}
@@ -1376,11 +1381,11 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		// 2. Remove any derived credentials from memory
 		if userType == regUser {
 			for _, u := range cache.iamUsersMap {
-				if u.IsServiceAccount() && u.ParentUser == accessKey {
-					delete(cache.iamUsersMap, u.AccessKey)
+				if u.Credentials.IsServiceAccount() && u.Credentials.ParentUser == accessKey {
+					delete(cache.iamUsersMap, u.Credentials.AccessKey)
 				}
-				if u.IsTemp() && u.ParentUser == accessKey {
-					delete(cache.iamUsersMap, u.AccessKey)
+				if u.Credentials.IsTemp() && u.Credentials.ParentUser == accessKey {
+					delete(cache.iamUsersMap, u.Credentials.AccessKey)
 				}
 			}
 		}
@@ -1412,8 +1417,9 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 	// This mapping is necessary to ensure that valid credentials
 	// have necessary ParentUser present - this is mainly for only
 	// webIdentity based STS tokens.
-	cred, ok := cache.iamUsersMap[accessKey]
+	u, ok := cache.iamUsersMap[accessKey]
 	if ok {
+		cred := u.Credentials
 		if cred.IsTemp() && cred.ParentUser != "" && cred.ParentUser != globalActiveCred.AccessKey {
 			if _, ok := cache.iamUserPolicyMap[cred.ParentUser]; !ok {
 				cache.iamUserPolicyMap[cred.ParentUser] = cache.iamUserPolicyMap[accessKey]
@@ -1439,7 +1445,7 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	if store.getUsersSysType() == MinIOUsersSysType && userType == regUser {
 		memberOf := cache.iamUserGroupMemberships[accessKey].ToSlice()
 		for _, group := range memberOf {
-			removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, false)
+			_, removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, false)
 			if removeErr != nil {
 				return removeErr
 			}
@@ -1451,7 +1457,8 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	// Delete any STS and service account derived from this credential
 	// first.
 	if userType == regUser {
-		for _, u := range cache.iamUsersMap {
+		for _, ui := range cache.iamUsersMap {
+			u := ui.Credentials
 			if u.IsServiceAccount() && u.ParentUser == accessKey {
 				_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
 				delete(cache.iamUsersMap, u.AccessKey)
@@ -1483,9 +1490,9 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 // SetTempUser - saves temporary (STS) credential to storage and cache. If a
 // policy name is given, it is associated with the parent user specified in the
 // credential.
-func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
+func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) (time.Time, error) {
 	if accessKey == "" || !cred.IsTemp() || cred.IsExpired() || cred.ParentUser == "" {
-		return errInvalidArgument
+		return time.Time{}, errInvalidArgument
 	}
 
 	ttl := int64(cred.Expiration.Sub(UTCNow()).Seconds())
@@ -1498,12 +1505,12 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 		_, combinedPolicyStmt := filterPolicies(cache, mp.Policies, "")
 
 		if combinedPolicyStmt.IsEmpty() {
-			return fmt.Errorf("specified policy %s, not found %w", policyName, errNoSuchPolicy)
+			return time.Time{}, fmt.Errorf("specified policy %s, not found %w", policyName, errNoSuchPolicy)
 		}
 
 		err := store.saveMappedPolicy(ctx, cred.ParentUser, stsUser, false, mp, options{ttl: ttl})
 		if err != nil {
-			return err
+			return time.Time{}, err
 		}
 
 		cache.iamUserPolicyMap[cred.ParentUser] = mp
@@ -1512,14 +1519,14 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 	u := newUserIdentity(cred)
 	err := store.saveUserIdentity(ctx, accessKey, stsUser, u, options{ttl: ttl})
 	if err != nil {
-		return err
+		return time.Time{}, err
 	}
 
-	cache.iamUsersMap[accessKey] = cred
+	cache.iamUsersMap[accessKey] = u
 
 	cache.updatedAt = time.Now()
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // DeleteUsers - given a set of users or access keys, deletes them along with
@@ -1531,8 +1538,10 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 
 	var deleted bool
 	usersToDelete := set.CreateStringSet(users...)
-	for user, cred := range cache.iamUsersMap {
+	for user, ui := range cache.iamUsersMap {
 		userType := regUser
+		cred := ui.Credentials
+
 		if cred.IsServiceAccount() {
 			userType = svcUser
 		} else if cred.IsTemp() {
@@ -1575,7 +1584,8 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 	defer store.runlock()
 
 	res := map[string]ParentUserInfo{}
-	for _, cred := range cache.iamUsersMap {
+	for _, ui := range cache.iamUsersMap {
+		cred := ui.Credentials
 		// Only consider service account or STS credentials with
 		// non-empty session tokens.
 		if !(cred.IsServiceAccount() || cred.IsTemp()) ||
@@ -1633,21 +1643,22 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 }
 
 // SetUserStatus - sets current user status.
-func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) error {
+func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) (updatedAt time.Time, err error) {
 	if accessKey != "" && status != madmin.AccountEnabled && status != madmin.AccountDisabled {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
 	defer store.unlock()
 
-	cred, ok := cache.iamUsersMap[accessKey]
+	ui, ok := cache.iamUsersMap[accessKey]
 	if !ok {
-		return errNoSuchUser
+		return updatedAt, errNoSuchUser
 	}
+	cred := ui.Credentials
 
 	if cred.IsTemp() || cred.IsServiceAccount() {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
 	uinfo := newUserIdentity(auth.Credentials{
@@ -1663,17 +1674,17 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
 	})
 
 	if err := store.saveUserIdentity(ctx, accessKey, regUser, uinfo); err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[accessKey] = uinfo.Credentials
+	cache.iamUsersMap[accessKey] = uinfo
 	cache.updatedAt = time.Now()
 
-	return nil
+	return uinfo.UpdatedAt, nil
 }
 
 // AddServiceAccount - add a new service account
-func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Credentials) error {
+func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Credentials) (updatedAt time.Time, err error) {
 	cache := store.lock()
 	defer store.unlock()
 
@@ -1683,44 +1694,45 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
 	// Found newly requested service account, to be an existing account -
 	// reject such operation (updates to the service account are handled in
 	// a different API).
-	if scred, found := cache.iamUsersMap[accessKey]; found {
+	if su, found := cache.iamUsersMap[accessKey]; found {
+		scred := su.Credentials
 		if scred.ParentUser != parentUser {
-			return errIAMServiceAccountUsed
+			return updatedAt, errIAMServiceAccountUsed
 		}
-		return errIAMServiceAccount
+		return updatedAt, errIAMServiceAccount
 	}
 
 	// Parent user must not be a service account.
-	if cr, found := cache.iamUsersMap[parentUser]; found && cr.IsServiceAccount() {
-		return errIAMServiceAccount
+	if u, found := cache.iamUsersMap[parentUser]; found && u.Credentials.IsServiceAccount() {
+		return updatedAt, errIAMServiceAccount
 	}
 
 	u := newUserIdentity(cred)
-	err := store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u)
+	err = store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+	cache.iamUsersMap[u.Credentials.AccessKey] = u
 	cache.updatedAt = time.Now()
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // UpdateServiceAccount - updates a service account on storage.
-func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) error {
+func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) (updatedAt time.Time, err error) {
 	cache := store.lock()
 	defer store.unlock()
 
-	cr, ok := cache.iamUsersMap[accessKey]
-	if !ok || !cr.IsServiceAccount() {
-		return errNoSuchServiceAccount
+	ui, ok := cache.iamUsersMap[accessKey]
+	if !ok || !ui.Credentials.IsServiceAccount() {
+		return updatedAt, errNoSuchServiceAccount
 	}
-
+	cr := ui.Credentials
 	currentSecretKey := cr.SecretKey
 	if opts.secretKey != "" {
 		if !auth.IsSecretKeyValid(opts.secretKey) {
-			return auth.ErrInvalidSecretKeyLength
+			return updatedAt, auth.ErrInvalidSecretKeyLength
 		}
 		cr.SecretKey = opts.secretKey
 	}
@@ -1736,12 +1748,12 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 	case auth.AccountOn, auth.AccountOff:
 		cr.Status = opts.status
 	default:
-		return errors.New("unknown account status value")
+		return updatedAt, errors.New("unknown account status value")
 	}
 
 	m, err := getClaimsFromTokenWithSecret(cr.SessionToken, currentSecretKey)
 	if err != nil {
-		return fmt.Errorf("unable to get svc acc claims: %v", err)
+		return updatedAt, fmt.Errorf("unable to get svc acc claims: %v", err)
 	}
 
 	// Extracted session policy name string can be removed as its not useful
@@ -1757,16 +1769,16 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 
 	if opts.sessionPolicy != nil {
 		if err := opts.sessionPolicy.Validate(); err != nil {
-			return err
+			return updatedAt, err
 		}
 
 		policyBuf, err := json.Marshal(opts.sessionPolicy)
 		if err != nil {
-			return err
+			return updatedAt, err
 		}
 
 		if len(policyBuf) > 16*humanize.KiByte {
-			return fmt.Errorf("Session policy should not exceed 16 KiB characters")
+			return updatedAt, fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
 
 		// Overwrite session policy claims.
@@ -1776,41 +1788,41 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 
 	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	u := newUserIdentity(cr)
 	if err := store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u); err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+	cache.iamUsersMap[u.Credentials.AccessKey] = u
 	cache.updatedAt = time.Now()
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // ListTempAccounts - lists only temporary accounts from the cache.
-func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string) ([]UserIdentity, error) {
 	cache := store.rlock()
 	defer store.runlock()
 
 	userExists := false
-	var tempAccounts []auth.Credentials
+	var tempAccounts []UserIdentity
 	for _, v := range cache.iamUsersMap {
 		isDerived := false
-		if v.IsServiceAccount() || v.IsTemp() {
+		if v.Credentials.IsServiceAccount() || v.Credentials.IsTemp() {
 			isDerived = true
 		}
 
-		if !isDerived && v.AccessKey == accessKey {
+		if !isDerived && v.Credentials.AccessKey == accessKey {
 			userExists = true
-		} else if isDerived && v.ParentUser == accessKey {
+		} else if isDerived && v.Credentials.ParentUser == accessKey {
 			userExists = true
-			if v.IsTemp() {
+			if v.Credentials.IsTemp() {
 				// Hide secret key & session key here
-				v.SecretKey = ""
-				v.SessionToken = ""
+				v.Credentials.SecretKey = ""
+				v.Credentials.SessionToken = ""
 				tempAccounts = append(tempAccounts, v)
 			}
 		}
@@ -1830,8 +1842,9 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 
 	userExists := false
 	var serviceAccounts []auth.Credentials
-	for _, v := range cache.iamUsersMap {
+	for _, u := range cache.iamUsersMap {
 		isDerived := false
+		v := u.Credentials
 		if v.IsServiceAccount() || v.IsTemp() {
 			isDerived = true
 		}
@@ -1857,17 +1870,17 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 }
 
 // AddUser - adds/updates long term user account to storage.
-func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
+func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) (updatedAt time.Time, err error) {
 	cache := store.lock()
 	defer store.unlock()
 
 	cache.updatedAt = time.Now()
 
-	cr, ok := cache.iamUsersMap[accessKey]
+	ui, ok := cache.iamUsersMap[accessKey]
 
 	// It is not possible to update an STS account.
-	if ok && cr.IsTemp() {
-		return errIAMActionNotAllowed
+	if ok && ui.Credentials.IsTemp() {
+		return updatedAt, errIAMActionNotAllowed
 	}
 
 	u := newUserIdentity(auth.Credentials{
@@ -1883,12 +1896,12 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
 	})
 
 	if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[accessKey] = u.Credentials
+	cache.iamUsersMap[accessKey] = u
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // UpdateUserSecretKey - sets user secret key to storage.
@@ -1898,18 +1911,18 @@ func (store *IAMStoreSys) UpdateUserSecretKey(ctx context.Context, accessKey, se
 
 	cache.updatedAt = time.Now()
 
-	cred, ok := cache.iamUsersMap[accessKey]
+	ui, ok := cache.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
 	}
-
+	cred := ui.Credentials
 	cred.SecretKey = secretKey
 	u := newUserIdentity(cred)
 	if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil {
 		return err
 	}
 
-	cache.iamUsersMap[accessKey] = cred
+	cache.iamUsersMap[accessKey] = u
 	return nil
 }
 
@@ -1919,7 +1932,8 @@ func (store *IAMStoreSys) GetSTSAndServiceAccounts() []auth.Credentials {
 	defer store.runlock()
 
 	var res []auth.Credentials
-	for _, cred := range cache.iamUsersMap {
+	for _, u := range cache.iamUsersMap {
+		cred := u.Credentials
 		if cred.IsTemp() || cred.IsServiceAccount() {
 			res = append(res, cred)
 		}
@@ -1940,13 +1954,13 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
 	} else if cred.IsTemp() {
 		userType = stsUser
 	}
-
+	ui := newUserIdentity(cred)
 	// Overwrite the user identity here. As store should be
 	// atomic, it shouldn't cause any corruption.
-	if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, newUserIdentity(cred)); err != nil {
+	if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, ui); err != nil {
 		return err
 	}
-	cache.iamUsersMap[cred.AccessKey] = cred
+	cache.iamUsersMap[cred.AccessKey] = ui
 	return nil
 }
 
@@ -1969,9 +1983,9 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 			if svc, found := cache.iamUsersMap[accessKey]; found {
 				// Load parent user and mapped policies.
 				if store.getUsersSysType() == MinIOUsersSysType {
-					store.loadUser(ctx, svc.ParentUser, regUser, cache.iamUsersMap)
+					store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
 				}
-				store.loadMappedPolicy(ctx, svc.ParentUser, regUser, false, cache.iamUserPolicyMap)
+				store.loadMappedPolicy(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap)
 			} else {
 				// check for STS account
 				store.loadUser(ctx, accessKey, stsUser, cache.iamUsersMap)

cmd/iam.go

@@ -223,8 +223,6 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
-	iamInitStart := time.Now()
-
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -297,8 +295,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
-	iamLoadStart := time.Now()
-
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx); err != nil {
@@ -370,9 +366,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	}
 
 	sys.printIAMRoles()
-
-	now := time.Now()
-	logger.Info("Finished loading IAM sub-system (took %.1fs of %.1fs to load data).", now.Sub(iamLoadStart).Seconds(), now.Sub(iamInitStart).Seconds())
 }
 
 func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[arn.ARN]string) {
@@ -606,14 +599,14 @@ func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[s
 }
 
 // SetPolicy - sets a new named policy.
-func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy.Policy) (time.Time, error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return time.Time{}, errServerNotInitialized
 	}
 
-	err := sys.store.SetPolicy(ctx, policyName, p)
+	updatedAt, err := sys.store.SetPolicy(ctx, policyName, p)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	if !sys.HasWatcher() {
@@ -625,7 +618,7 @@ func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy
 			}
 		}
 	}
-	return nil
+	return updatedAt, nil
 }
 
 // DeleteUser - delete user (only for long-term users not STS users).
@@ -714,9 +707,9 @@ func (sys *IAMSys) notifyForUser(ctx context.Context, accessKey string, isTemp b
 // elsewhere), the AssumeRole case (because the parent user is real and their
 // policy is associated via policy-set API) and the AssumeRoleWithLDAP case
 // (because the policy association is made via policy-set API).
-func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
+func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) (time.Time, error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return time.Time{}, errServerNotInitialized
 	}
 
 	if newGlobalAuthZPluginFn() != nil {
@@ -724,14 +717,14 @@ func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.
 		policyName = ""
 	}
 
-	err := sys.store.SetTempUser(ctx, accessKey, cred, policyName)
+	updatedAt, err := sys.store.SetTempUser(ctx, accessKey, cred, policyName)
 	if err != nil {
-		return err
+		return time.Time{}, err
 	}
 
 	sys.notifyForUser(ctx, cred.AccessKey, true)
 
-	return nil
+	return updatedAt, nil
 }
 
 // ListBucketUsers - list all users who can access this 'bucket'
@@ -789,11 +782,11 @@ func (sys *IAMSys) IsTempUser(name string) (bool, string, error) {
 		return false, "", errServerNotInitialized
 	}
 
-	cred, found := sys.store.GetUser(name)
+	u, found := sys.store.GetUser(name)
 	if !found {
 		return false, "", errNoSuchUser
 	}
-
+	cred := u.Credentials
 	if cred.IsTemp() {
 		return true, cred.ParentUser, nil
 	}
@@ -807,11 +800,11 @@ func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
 		return false, "", errServerNotInitialized
 	}
 
-	cred, found := sys.store.GetUser(name)
+	u, found := sys.store.GetUser(name)
 	if !found {
 		return false, "", errNoSuchUser
 	}
-
+	cred := u.Credentials
 	if cred.IsServiceAccount() {
 		return true, cred.ParentUser, nil
 	}
@@ -835,22 +828,22 @@ func (sys *IAMSys) GetUserInfo(ctx context.Context, name string) (u madmin.UserI
 }
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
-func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) error {
+func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.SetUserStatus(ctx, accessKey, status)
+	updatedAt, err = sys.store.SetUserStatus(ctx, accessKey, status)
 	if err != nil {
-		return err
+		return
 	}
 
 	sys.notifyForUser(ctx, accessKey, false)
-	return nil
+	return updatedAt, nil
 }
 
 func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string) {
@@ -874,34 +867,34 @@ type newServiceAccountOpts struct {
 }
 
 // NewServiceAccount - create a new service account
-func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, groups []string, opts newServiceAccountOpts) (auth.Credentials, error) {
+func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, groups []string, opts newServiceAccountOpts) (auth.Credentials, time.Time, error) {
 	if !sys.Initialized() {
-		return auth.Credentials{}, errServerNotInitialized
+		return auth.Credentials{}, time.Time{}, errServerNotInitialized
 	}
 
 	if parentUser == "" {
-		return auth.Credentials{}, errInvalidArgument
+		return auth.Credentials{}, time.Time{}, errInvalidArgument
 	}
 
 	var policyBuf []byte
 	if opts.sessionPolicy != nil {
 		err := opts.sessionPolicy.Validate()
 		if err != nil {
-			return auth.Credentials{}, err
+			return auth.Credentials{}, time.Time{}, err
 		}
 		policyBuf, err = json.Marshal(opts.sessionPolicy)
 		if err != nil {
-			return auth.Credentials{}, err
+			return auth.Credentials{}, time.Time{}, err
 		}
 		if len(policyBuf) > 16*humanize.KiByte {
-			return auth.Credentials{}, fmt.Errorf("Session policy should not exceed 16 KiB characters")
+			return auth.Credentials{}, time.Time{}, fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
 	}
 
 	// found newly requested service account, to be same as
 	// parentUser, reject such operations.
 	if parentUser == opts.accessKey {
-		return auth.Credentials{}, errIAMActionNotAllowed
+		return auth.Credentials{}, time.Time{}, errIAMActionNotAllowed
 	}
 
 	m := make(map[string]interface{})
@@ -929,24 +922,24 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	} else {
 		accessKey, secretKey, err = auth.GenerateCredentials()
 		if err != nil {
-			return auth.Credentials{}, err
+			return auth.Credentials{}, time.Time{}, err
 		}
 	}
 	cred, err := auth.CreateNewCredentialsWithMetadata(accessKey, secretKey, m, secretKey)
 	if err != nil {
-		return auth.Credentials{}, err
+		return auth.Credentials{}, time.Time{}, err
 	}
 	cred.ParentUser = parentUser
 	cred.Groups = groups
 	cred.Status = string(auth.AccountOn)
 
-	err = sys.store.AddServiceAccount(ctx, cred)
+	updatedAt, err := sys.store.AddServiceAccount(ctx, cred)
 	if err != nil {
-		return auth.Credentials{}, err
+		return auth.Credentials{}, time.Time{}, err
 	}
 
 	sys.notifyForServiceAccount(ctx, cred.AccessKey)
-	return cred, nil
+	return cred, updatedAt, nil
 }
 
 type updateServiceAccountOpts struct {
@@ -956,18 +949,18 @@ type updateServiceAccountOpts struct {
 }
 
 // UpdateServiceAccount - edit a service account
-func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) error {
+func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
-	err := sys.store.UpdateServiceAccount(ctx, accessKey, opts)
+	updatedAt, err = sys.store.UpdateServiceAccount(ctx, accessKey, opts)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForServiceAccount(ctx, accessKey)
-	return nil
+	return updatedAt, nil
 }
 
 // ListServiceAccounts - lists all services accounts associated to a specific user
@@ -985,7 +978,7 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 }
 
 // ListTempAccounts - lists all services accounts associated to a specific user
-func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]UserIdentity, error) {
 	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
@@ -1002,32 +995,32 @@ func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]au
 func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
 	sa, embeddedPolicy, err := sys.getServiceAccount(ctx, accessKey)
 	if err != nil {
-		return sa, embeddedPolicy, err
+		return auth.Credentials{}, embeddedPolicy, err
 	}
 	// Hide secret & session keys
-	sa.SecretKey = ""
-	sa.SessionToken = ""
-	return sa, embeddedPolicy, nil
+	sa.Credentials.SecretKey = ""
+	sa.Credentials.SessionToken = ""
+	return sa.Credentials, embeddedPolicy, nil
 }
 
 // getServiceAccount - gets information about a service account
-func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
+func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (u UserIdentity, p *iampolicy.Policy, err error) {
 	if !sys.Initialized() {
-		return auth.Credentials{}, nil, errServerNotInitialized
+		return u, nil, errServerNotInitialized
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.IsServiceAccount() {
-		return auth.Credentials{}, nil, errNoSuchServiceAccount
+	if !ok || !sa.Credentials.IsServiceAccount() {
+		return u, nil, errNoSuchServiceAccount
 	}
 
 	var embeddedPolicy *iampolicy.Policy
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.Credentials.SessionToken, sa.Credentials.SecretKey)
 	if err != nil {
-		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		jwtClaims, err = auth.ExtractClaims(sa.Credentials.SessionToken, globalActiveCred.SecretKey)
 		if err != nil {
-			return auth.Credentials{}, nil, err
+			return u, nil, err
 		}
 	}
 	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
@@ -1035,11 +1028,11 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut
 	if ptok && spok && pt == embeddedPolicyType {
 		policyBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
-			return auth.Credentials{}, nil, err
+			return u, nil, err
 		}
 		embeddedPolicy, err = iampolicy.ParseConfig(bytes.NewReader(policyBytes))
 		if err != nil {
-			return auth.Credentials{}, nil, err
+			return u, nil, err
 		}
 	}
 
@@ -1057,13 +1050,13 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.IsServiceAccount() {
+	if !ok || !sa.Credentials.IsServiceAccount() {
 		return nil, errNoSuchServiceAccount
 	}
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.Credentials.SessionToken, sa.Credentials.SecretKey)
 	if err != nil {
-		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		jwtClaims, err = auth.ExtractClaims(sa.Credentials.SessionToken, globalActiveCred.SecretKey)
 		if err != nil {
 			return nil, err
 		}
@@ -1078,7 +1071,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.IsServiceAccount() {
+	if !ok || !sa.Credentials.IsServiceAccount() {
 		return nil
 	}
 
@@ -1100,30 +1093,30 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 
 // CreateUser - create new user credentials and policy, if user already exists
 // they shall be rewritten with new inputs.
-func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
+func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
 	if !auth.IsAccessKeyValid(accessKey) {
-		return auth.ErrInvalidAccessKeyLength
+		return updatedAt, auth.ErrInvalidAccessKeyLength
 	}
 
 	if !auth.IsSecretKeyValid(ureq.SecretKey) {
-		return auth.ErrInvalidSecretKeyLength
+		return updatedAt, auth.ErrInvalidSecretKeyLength
 	}
 
-	err := sys.store.AddUser(ctx, accessKey, ureq)
+	updatedAt, err = sys.store.AddUser(ctx, accessKey, ureq)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForUser(ctx, accessKey, false)
-	return nil
+	return updatedAt, nil
 }
 
 // SetUserSecretKey - sets user secret key
@@ -1298,9 +1291,9 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 }
 
 // GetUser - get user credentials
-func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Credentials, ok bool) {
+func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (u UserIdentity, ok bool) {
 	if !sys.Initialized() {
-		return cred, false
+		return u, false
 	}
 
 	fallback := false
@@ -1311,7 +1304,7 @@ func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Cre
 		fallback = true
 	}
 
-	cred, ok = sys.store.GetUser(accessKey)
+	u, ok = sys.store.GetUser(accessKey)
 	if !ok && !fallback {
 		// accessKey not found, also
 		// IAM store is not in fallback mode
@@ -1320,10 +1313,10 @@ func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Cre
 		// exists now. If it doesn't proceed to
 		// fail.
 		sys.store.LoadUser(ctx, accessKey)
-		cred, ok = sys.store.GetUser(accessKey)
+		u, ok = sys.store.GetUser(accessKey)
 	}
 
-	return cred, ok && cred.IsValid()
+	return u, ok && u.Credentials.IsValid()
 }
 
 // Notify all other MinIO peers to load group.
@@ -1340,61 +1333,61 @@ func (sys *IAMSys) notifyForGroup(ctx context.Context, group string) {
 
 // AddUsersToGroup - adds users to a group, creating the group if
 // needed. No error if user(s) already are in the group.
-func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []string) error {
+func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.AddUsersToGroup(ctx, group, members)
+	updatedAt, err = sys.store.AddUsersToGroup(ctx, group, members)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForGroup(ctx, group)
-	return nil
+	return updatedAt, nil
 }
 
 // RemoveUsersFromGroup - remove users from group. If no users are
 // given, and the group is empty, deletes the group as well.
-func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) error {
+func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.RemoveUsersFromGroup(ctx, group, members)
+	updatedAt, err = sys.store.RemoveUsersFromGroup(ctx, group, members)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForGroup(ctx, group)
-	return nil
+	return updatedAt, nil
 }
 
 // SetGroupStatus - enable/disabled a group
-func (sys *IAMSys) SetGroupStatus(ctx context.Context, group string, enabled bool) error {
+func (sys *IAMSys) SetGroupStatus(ctx context.Context, group string, enabled bool) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.SetGroupStatus(ctx, group, enabled)
+	updatedAt, err = sys.store.SetGroupStatus(ctx, group, enabled)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForGroup(ctx, group)
-	return nil
+	return updatedAt, nil
 }
 
 // GetGroupDescription - builds up group description
@@ -1421,9 +1414,9 @@ func (sys *IAMSys) ListGroups(ctx context.Context) (r []string, err error) {
 }
 
 // PolicyDBSet - sets a policy for a user or group in the PolicyDB.
-func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup bool) error {
+func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup bool) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	// Determine user-type based on IDP mode.
@@ -1432,9 +1425,9 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup
 		userType = stsUser
 	}
 
-	err := sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
+	updatedAt, err = sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
 	if err != nil {
-		return err
+		return
 	}
 
 	// Notify all other MinIO peers to reload policy
@@ -1447,7 +1440,7 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup
 		}
 	}
 
-	return nil
+	return updatedAt, nil
 }
 
 // PolicyDBGet - gets policy set on a user or group. If a list of groups is

cmd/jwt.go

@@ -63,11 +63,11 @@ func authenticateJWTUsers(accessKey, secretKey string, expiry time.Duration) (st
 func authenticateJWTUsersWithCredentials(credentials auth.Credentials, expiresAt time.Time) (string, error) {
 	serverCred := globalActiveCred
 	if serverCred.AccessKey != credentials.AccessKey {
-		var ok bool
-		serverCred, ok = globalIAMSys.GetUser(context.TODO(), credentials.AccessKey)
+		u, ok := globalIAMSys.GetUser(context.TODO(), credentials.AccessKey)
 		if !ok {
 			return "", errInvalidAccessKeyID
 		}
+		serverCred = u.Credentials
 	}
 
 	if !serverCred.Equal(credentials) {
@@ -145,10 +145,11 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 		if claims.AccessKey == globalActiveCred.AccessKey {
 			return []byte(globalActiveCred.SecretKey), nil
 		}
-		cred, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
+		u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
 		if !ok {
 			return nil, errInvalidAccessKeyID
 		}
+		cred := u.Credentials
 		return []byte(cred.SecretKey), nil
 	}); err != nil {
 		return claims, nil, false, errAuthentication
@@ -157,11 +158,11 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 	var groups []string
 	if globalActiveCred.AccessKey != claims.AccessKey {
 		// Check if the access key is part of users credentials.
-		ucred, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
+		u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
 		if !ok {
 			return nil, nil, false, errInvalidAccessKeyID
 		}
-
+		ucred := u.Credentials
 		// get embedded claims
 		eclaims, s3Err := checkClaimsFromToken(req, ucred)
 		if s3Err != ErrNone {

cmd/last-minute.go

@@ -21,6 +21,8 @@ package cmd
 
 import (
 	"time"
+
+	"github.com/minio/madmin-go"
 )
 
 const (
@@ -75,29 +77,49 @@ func sizeTagToString(tag int) string {
 // AccElem holds information for calculating an average value
 type AccElem struct {
 	Total int64
+	Size  int64
 	N     int64
 }
 
 // Add a duration to a single element.
 func (a *AccElem) add(dur time.Duration) {
+	if dur < 0 {
+		dur = 0
+	}
 	a.Total += int64(dur)
 	a.N++
 }
 
+// Add a duration to a single element.
+func (a *AccElem) addSize(dur time.Duration, sz int64) {
+	if dur < 0 {
+		dur = 0
+	}
+	a.Total += int64(dur)
+	a.Size += sz
+	a.N++
+}
+
 // Merge b into a.
 func (a *AccElem) merge(b AccElem) {
 	a.N += b.N
 	a.Total += b.Total
+	a.Size += b.Size
 }
 
-// Avg converts total to average.
-func (a AccElem) avg() uint64 {
+// Avg returns average time spent.
+func (a AccElem) avg() time.Duration {
 	if a.N >= 1 && a.Total > 0 {
-		return uint64(a.Total / a.N)
+		return time.Duration(a.Total / a.N)
 	}
 	return 0
 }
 
+// asTimedAction returns the element as a madmin.TimedAction.
+func (a AccElem) asTimedAction() madmin.TimedAction {
+	return madmin.TimedAction{AccTime: uint64(a.Total), Count: uint64(a.N), Bytes: uint64(a.Size)}
+}
+
 // lastMinuteLatency keeps track of last minute latency.
 type lastMinuteLatency struct {
 	Totals  [60]AccElem
@@ -118,6 +140,7 @@ func (l lastMinuteLatency) merge(o lastMinuteLatency) (merged lastMinuteLatency)
 		merged.Totals[i] = AccElem{
 			Total: l.Totals[i].Total + o.Totals[i].Total,
 			N:     l.Totals[i].N + o.Totals[i].N,
+			Size:  l.Totals[i].Size + o.Totals[i].Size,
 		}
 	}
 	return merged
@@ -132,8 +155,17 @@ func (l *lastMinuteLatency) add(t time.Duration) {
 	l.LastSec = sec
 }
 
+// Add  a new duration data
+func (l *lastMinuteLatency) addSize(t time.Duration, sz int64) {
+	sec := time.Now().Unix()
+	l.forwardTo(sec)
+	winIdx := sec % 60
+	l.Totals[winIdx].addSize(t, sz)
+	l.LastSec = sec
+}
+
 // Merge all recorded latencies of last minute into one
-func (l *lastMinuteLatency) getAvgData() AccElem {
+func (l *lastMinuteLatency) getTotal() AccElem {
 	var res AccElem
 	sec := time.Now().Unix()
 	l.forwardTo(sec)
@@ -160,11 +192,11 @@ func (l *lastMinuteLatency) forwardTo(t int64) {
 	}
 }
 
-// LastMinuteLatencies keeps track of last minute latencies.
-type LastMinuteLatencies [sizeLastElemMarker]lastMinuteLatency
+// LastMinuteHistogram keeps track of last minute sizes added.
+type LastMinuteHistogram [sizeLastElemMarker]lastMinuteLatency
 
-// Merge safely merges two LastMinuteLatencies structures into one
-func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLatencies) {
+// Merge safely merges two LastMinuteHistogram structures into one
+func (l LastMinuteHistogram) Merge(o LastMinuteHistogram) (merged LastMinuteHistogram) {
 	for i := range l {
 		merged[i] = l[i].merge(o[i])
 	}
@@ -172,16 +204,16 @@ func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLate
 }
 
 // Add latency t from object with the specified size.
-func (l *LastMinuteLatencies) Add(size int64, t time.Duration) {
+func (l *LastMinuteHistogram) Add(size int64, t time.Duration) {
 	l[sizeToTag(size)].add(t)
 }
 
 // GetAvgData will return the average for each bucket from the last time minute.
 // The number of objects is also included.
-func (l *LastMinuteLatencies) GetAvgData() [sizeLastElemMarker]AccElem {
+func (l *LastMinuteHistogram) GetAvgData() [sizeLastElemMarker]AccElem {
 	var res [sizeLastElemMarker]AccElem
 	for i, elem := range l[:] {
-		res[i] = elem.getAvgData()
+		res[i] = elem.getTotal()
 	}
 	return res
 }

cmd/last-minute_gen.go

@@ -30,6 +30,12 @@ func (z *AccElem) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Total")
 				return
 			}
+		case "Size":
+			z.Size, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
 		case "N":
 			z.N, err = dc.ReadInt64()
 			if err != nil {
@@ -49,9 +55,9 @@ func (z *AccElem) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z AccElem) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
+	// map header, size 3
 	// write "Total"
-	err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+	err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 	if err != nil {
 		return
 	}
@@ -60,6 +66,16 @@ func (z AccElem) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Total")
 		return
 	}
+	// write "Size"
+	err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
 	// write "N"
 	err = en.Append(0xa1, 0x4e)
 	if err != nil {
@@ -76,10 +92,13 @@ func (z AccElem) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z AccElem) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
+	// map header, size 3
 	// string "Total"
-	o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+	o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 	o = msgp.AppendInt64(o, z.Total)
+	// string "Size"
+	o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.Size)
 	// string "N"
 	o = append(o, 0xa1, 0x4e)
 	o = msgp.AppendInt64(o, z.N)
@@ -110,6 +129,12 @@ func (z *AccElem) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Total")
 				return
 			}
+		case "Size":
+			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
 		case "N":
 			z.N, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
@@ -130,12 +155,12 @@ func (z *AccElem) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z AccElem) Msgsize() (s int) {
-	s = 1 + 6 + msgp.Int64Size + 2 + msgp.Int64Size
+	s = 1 + 6 + msgp.Int64Size + 5 + msgp.Int64Size + 2 + msgp.Int64Size
 	return
 }
 
 // DecodeMsg implements msgp.Decodable
-func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
+func (z *LastMinuteHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
 	var zb0001 uint32
 	zb0001, err = dc.ReadArrayHeader()
 	if err != nil {
@@ -195,6 +220,12 @@ func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
 								err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
 								return
 							}
+						case "Size":
+							z[za0001].Totals[za0002].Size, err = dc.ReadInt64()
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "Size")
+								return
+							}
 						case "N":
 							z[za0001].Totals[za0002].N, err = dc.ReadInt64()
 							if err != nil {
@@ -229,7 +260,7 @@ func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
+func (z *LastMinuteHistogram) EncodeMsg(en *msgp.Writer) (err error) {
 	err = en.WriteArrayHeader(uint32(sizeLastElemMarker))
 	if err != nil {
 		err = msgp.WrapError(err)
@@ -248,9 +279,9 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 		for za0002 := range z[za0001].Totals {
-			// map header, size 2
+			// map header, size 3
 			// write "Total"
-			err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+			err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 			if err != nil {
 				return
 			}
@@ -259,6 +290,16 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 				err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
 				return
 			}
+			// write "Size"
+			err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+			if err != nil {
+				return
+			}
+			err = en.WriteInt64(z[za0001].Totals[za0002].Size)
+			if err != nil {
+				err = msgp.WrapError(err, za0001, "Totals", za0002, "Size")
+				return
+			}
 			// write "N"
 			err = en.Append(0xa1, 0x4e)
 			if err != nil {
@@ -285,7 +326,7 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *LastMinuteHistogram) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	o = msgp.AppendArrayHeader(o, uint32(sizeLastElemMarker))
 	for za0001 := range z {
@@ -294,10 +335,13 @@ func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
 		o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
 		o = msgp.AppendArrayHeader(o, uint32(60))
 		for za0002 := range z[za0001].Totals {
-			// map header, size 2
+			// map header, size 3
 			// string "Total"
-			o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+			o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].Total)
+			// string "Size"
+			o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].Size)
 			// string "N"
 			o = append(o, 0xa1, 0x4e)
 			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].N)
@@ -310,7 +354,7 @@ func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
+func (z *LastMinuteHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var zb0001 uint32
 	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
@@ -370,6 +414,12 @@ func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
 								err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
 								return
 							}
+						case "Size":
+							z[za0001].Totals[za0002].Size, bts, err = msgp.ReadInt64Bytes(bts)
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "Size")
+								return
+							}
 						case "N":
 							z[za0001].Totals[za0002].N, bts, err = msgp.ReadInt64Bytes(bts)
 							if err != nil {
@@ -405,8 +455,8 @@ func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *LastMinuteLatencies) Msgsize() (s int) {
-	s = msgp.ArrayHeaderSize + (sizeLastElemMarker * (16 + (60 * (9 + msgp.Int64Size + msgp.Int64Size)) + msgp.Int64Size))
+func (z *LastMinuteHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (sizeLastElemMarker * (16 + (60 * (14 + msgp.Int64Size + msgp.Int64Size + msgp.Int64Size)) + msgp.Int64Size))
 	return
 }
 
@@ -460,6 +510,12 @@ func (z *lastMinuteLatency) DecodeMsg(dc *msgp.Reader) (err error) {
 							err = msgp.WrapError(err, "Totals", za0001, "Total")
 							return
 						}
+					case "Size":
+						z.Totals[za0001].Size, err = dc.ReadInt64()
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001, "Size")
+							return
+						}
 					case "N":
 						z.Totals[za0001].N, err = dc.ReadInt64()
 						if err != nil {
@@ -506,9 +562,9 @@ func (z *lastMinuteLatency) EncodeMsg(en *msgp.Writer) (err error) {
 		return
 	}
 	for za0001 := range z.Totals {
-		// map header, size 2
+		// map header, size 3
 		// write "Total"
-		err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+		err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 		if err != nil {
 			return
 		}
@@ -517,6 +573,16 @@ func (z *lastMinuteLatency) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "Totals", za0001, "Total")
 			return
 		}
+		// write "Size"
+		err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+		if err != nil {
+			return
+		}
+		err = en.WriteInt64(z.Totals[za0001].Size)
+		if err != nil {
+			err = msgp.WrapError(err, "Totals", za0001, "Size")
+			return
+		}
 		// write "N"
 		err = en.Append(0xa1, 0x4e)
 		if err != nil {
@@ -549,10 +615,13 @@ func (z *lastMinuteLatency) MarshalMsg(b []byte) (o []byte, err error) {
 	o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(60))
 	for za0001 := range z.Totals {
-		// map header, size 2
+		// map header, size 3
 		// string "Total"
-		o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+		o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 		o = msgp.AppendInt64(o, z.Totals[za0001].Total)
+		// string "Size"
+		o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+		o = msgp.AppendInt64(o, z.Totals[za0001].Size)
 		// string "N"
 		o = append(o, 0xa1, 0x4e)
 		o = msgp.AppendInt64(o, z.Totals[za0001].N)
@@ -613,6 +682,12 @@ func (z *lastMinuteLatency) UnmarshalMsg(bts []byte) (o []byte, err error) {
 							err = msgp.WrapError(err, "Totals", za0001, "Total")
 							return
 						}
+					case "Size":
+						z.Totals[za0001].Size, bts, err = msgp.ReadInt64Bytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001, "Size")
+							return
+						}
 					case "N":
 						z.Totals[za0001].N, bts, err = msgp.ReadInt64Bytes(bts)
 						if err != nil {
@@ -648,6 +723,6 @@ func (z *lastMinuteLatency) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *lastMinuteLatency) Msgsize() (s int) {
-	s = 1 + 7 + msgp.ArrayHeaderSize + (60 * (9 + msgp.Int64Size + msgp.Int64Size)) + 8 + msgp.Int64Size
+	s = 1 + 7 + msgp.ArrayHeaderSize + (60 * (14 + msgp.Int64Size + msgp.Int64Size + msgp.Int64Size)) + 8 + msgp.Int64Size
 	return
 }

cmd/last-minute_gen_test.go

@@ -122,8 +122,8 @@ func BenchmarkDecodeAccElem(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalLastMinuteLatencies(t *testing.T) {
-	v := LastMinuteLatencies{}
+func TestMarshalUnmarshalLastMinuteHistogram(t *testing.T) {
+	v := LastMinuteHistogram{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
@@ -145,8 +145,8 @@ func TestMarshalUnmarshalLastMinuteLatencies(t *testing.T) {
 	}
 }
 
-func BenchmarkMarshalMsgLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkMarshalMsgLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -154,8 +154,8 @@ func BenchmarkMarshalMsgLastMinuteLatencies(b *testing.B) {
 	}
 }
 
-func BenchmarkAppendMsgLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkAppendMsgLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
@@ -166,8 +166,8 @@ func BenchmarkAppendMsgLastMinuteLatencies(b *testing.B) {
 	}
 }
 
-func BenchmarkUnmarshalLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkUnmarshalLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
@@ -180,17 +180,17 @@ func BenchmarkUnmarshalLastMinuteLatencies(b *testing.B) {
 	}
 }
 
-func TestEncodeDecodeLastMinuteLatencies(t *testing.T) {
-	v := LastMinuteLatencies{}
+func TestEncodeDecodeLastMinuteHistogram(t *testing.T) {
+	v := LastMinuteHistogram{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 
 	m := v.Msgsize()
 	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeLastMinuteLatencies Msgsize() is inaccurate")
+		t.Log("WARNING: TestEncodeDecodeLastMinuteHistogram Msgsize() is inaccurate")
 	}
 
-	vn := LastMinuteLatencies{}
+	vn := LastMinuteHistogram{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
@@ -204,8 +204,8 @@ func TestEncodeDecodeLastMinuteLatencies(t *testing.T) {
 	}
 }
 
-func BenchmarkEncodeLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkEncodeLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
@@ -218,8 +218,8 @@ func BenchmarkEncodeLastMinuteLatencies(b *testing.B) {
 	en.Flush()
 }
 
-func BenchmarkDecodeLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkDecodeLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))

cmd/listen-notification-handlers.go

@@ -25,7 +25,8 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
-	policy "github.com/minio/pkg/bucket/policy"
+	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r *http.Request) {
@@ -100,13 +101,14 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 	pattern := event.NewPattern(prefix, suffix)
 
 	var eventNames []event.Name
+	var mask pubsub.Mask
 	for _, s := range values[peerRESTListenEvents] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-
+		mask.MergeMaskable(eventName)
 		eventNames = append(eventNames, eventName)
 	}
 
@@ -123,11 +125,11 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	listenCh := make(chan interface{}, 4000)
+	listenCh := make(chan pubsub.Maskable, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
-	listenFn := func(evI interface{}) bool {
+	err := globalHTTPListen.Subscribe(mask, listenCh, ctx.Done(), func(evI pubsub.Maskable) bool {
 		ev, ok := evI.(event.Event)
 		if !ok {
 			return false
@@ -138,14 +140,11 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 			}
 		}
 		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
-	}
-
-	err := globalHTTPListen.Subscribe(listenCh, ctx.Done(), listenFn)
+	})
 	if err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
 		return
 	}
-
 	if bucketName != "" {
 		values.Set(peerRESTListenBucket, bucketName)
 	}
@@ -173,7 +172,10 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 					return
 				}
 			}
-			w.(http.Flusher).Flush()
+			if len(listenCh) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
+			}
 		case <-keepAliveTicker.C:
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return

cmd/main.go

@@ -19,10 +19,12 @@ package cmd
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
 	"sort"
+	"strings"
 
 	"github.com/minio/cli"
 	"github.com/minio/minio/internal/color"
@@ -163,11 +165,23 @@ func newApp(name string) *cli.App {
 	return app
 }
 
+func startupBanner(banner io.Writer) {
+	fmt.Fprintln(banner, color.Blue("Copyright:")+color.Bold(" 2015-%s MinIO, Inc.", CopyrightYear))
+	fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
+	fmt.Fprintln(banner, color.Blue("Version:")+color.Bold(" %s (%s %s/%s)", ReleaseTag, runtime.Version(), runtime.GOOS, runtime.GOARCH))
+}
+
+func versionBanner(c *cli.Context) io.Reader {
+	banner := &strings.Builder{}
+	fmt.Fprintln(banner, color.Bold("%s version %s (commit-id=%s)", c.App.Name, c.App.Version, CommitID))
+	fmt.Fprintln(banner, color.Blue("Runtime:")+color.Bold(" %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH))
+	fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
+	fmt.Fprintln(banner, color.Blue("Copyright:")+color.Bold(" 2015-%s MinIO, Inc.", CopyrightYear))
+	return strings.NewReader(banner.String())
+}
+
 func printMinIOVersion(c *cli.Context) {
-	fmt.Fprintln(c.App.Writer, color.Greenf("%s version %s (commit-id=%s)", c.App.Name, c.App.Version, CommitID))
-	fmt.Fprintln(c.App.Writer, color.Greenf("Runtime: %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH))
-	fmt.Fprintln(c.App.Writer, color.Greenf("Copyright (c) 2015-%s MinIO, Inc.", CopyrightYear))
-	fmt.Fprintln(c.App.Writer, color.Red("Licence AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
+	io.Copy(c.App.Writer, versionBanner(c))
 }
 
 // Main main for minio server.

cmd/metacache-entries.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"os"
 	"sort"
 	"strings"
@@ -330,7 +331,9 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 		// shallow decode.
 		xl, err := entry.xlmeta()
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			if !errors.Is(err, errFileNotFound) {
+				logger.LogIf(GlobalContext, err)
+			}
 			continue
 		}
 		objsValid++

cmd/metacache-walk.go

@@ -248,11 +248,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				out <- metaCacheEntry{name: pop}
 				if opts.Recursive {
 					// Scan folder we found. Should be in correct sort order where we are.
-					forward = ""
-					if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, pop) {
-						forward = strings.TrimPrefix(opts.ForwardTo, pop)
-					}
-
 					err := scanDir(pop)
 					if err != nil && !IsErrIgnored(err, context.Canceled) {
 						logger.LogIf(ctx, err)

cmd/metrics-realtime.go

@@ -0,0 +1,109 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"github.com/minio/madmin-go"
+)
+
+func collectLocalMetrics(types madmin.MetricType, hosts map[string]struct{}) (m madmin.RealtimeMetrics) {
+	if types == madmin.MetricsNone {
+		return
+	}
+	if len(hosts) > 0 {
+		if _, ok := hosts[globalMinioAddr]; !ok {
+			return
+		}
+	}
+	if types.Contains(madmin.MetricsScanner) {
+		metrics := globalScannerMetrics.report()
+		m.Aggregated.Scanner = &metrics
+	}
+	if types.Contains(madmin.MetricsDisk) && !globalIsGateway {
+		m.Aggregated.Disk = collectDiskMetrics()
+	}
+	if types.Contains(madmin.MetricsOS) {
+		metrics := globalOSMetrics.report()
+		m.Aggregated.OS = &metrics
+	}
+	// Add types...
+
+	// ByHost is a shallow reference, so careful about sharing.
+	m.ByHost = map[string]madmin.Metrics{globalMinioAddr: m.Aggregated}
+	m.Hosts = append(m.Hosts, globalMinioAddr)
+
+	return m
+}
+
+func collectDiskMetrics() *madmin.DiskMetric {
+	objLayer := newObjectLayerFn()
+	disks := madmin.DiskMetric{
+		CollectedAt: time.Now(),
+	}
+
+	if objLayer == nil {
+		return nil
+	}
+	// only need Disks information in server mode.
+	storageInfo, errs := objLayer.LocalStorageInfo(GlobalContext)
+	for _, err := range errs {
+		if err != nil {
+			disks.Merge(&madmin.DiskMetric{NDisks: 1, Offline: 1})
+		}
+	}
+	for i, disk := range storageInfo.Disks {
+		if errs[i] != nil {
+			continue
+		}
+		var d madmin.DiskMetric
+		d.NDisks = 1
+		if disk.Healing {
+			d.Healing++
+		}
+		if disk.Metrics != nil {
+			d.LifeTimeOps = make(map[string]uint64, len(disk.Metrics.APICalls))
+			for k, v := range disk.Metrics.APICalls {
+				if v != 0 {
+					d.LifeTimeOps[k] = v
+				}
+			}
+			d.LastMinute.Operations = make(map[string]madmin.TimedAction, len(disk.Metrics.APICalls))
+			for k, v := range disk.Metrics.LastMinute {
+				if v.Count != 0 {
+					d.LastMinute.Operations[k] = v
+				}
+			}
+		}
+		disks.Merge(&d)
+	}
+	return &disks
+}
+
+func collectRemoteMetrics(ctx context.Context, types madmin.MetricType, hosts map[string]struct{}) (m madmin.RealtimeMetrics) {
+	if !globalIsDistErasure {
+		return
+	}
+	all := globalNotificationSys.GetMetrics(ctx, types, hosts)
+	for _, remote := range all {
+		m.Merge(&remote)
+	}
+	return m
+}

cmd/metrics-v2.go

@@ -31,8 +31,8 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/procfs"
 )
 
@@ -1222,7 +1222,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Help:      "Total number of unique objects scanned since server start",
 					Type:      counterMetric,
 				},
-				Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalObjects)),
+				Value: float64(globalScannerMetrics.lifetime(scannerMetricScanObject)),
 			},
 			{
 				Description: MetricDescription{
@@ -1232,7 +1232,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Help:      "Total number of object versions scanned since server start",
 					Type:      counterMetric,
 				},
-				Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalVersions)),
+				Value: float64(globalScannerMetrics.lifetime(scannerMetricApplyVersion)),
 			},
 			{
 				Description: MetricDescription{
@@ -1242,7 +1242,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Help:      "Total number of directories scanned since server start",
 					Type:      counterMetric,
 				},
-				Value: float64(atomic.LoadUint64(&globalScannerStats.accFolders)),
+				Value: float64(globalScannerMetrics.lifetime(scannerMetricScanFolder)),
 			},
 			{
 				Description: MetricDescription{
@@ -1252,7 +1252,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Help:      "Total number of bucket scans started since server start",
 					Type:      counterMetric,
 				},
-				Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsStarted)),
+				Value: float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDisk) + uint64(globalScannerMetrics.activeDisks())),
 			},
 			{
 				Description: MetricDescription{
@@ -1262,7 +1262,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Help:      "Total number of bucket scans finished since server start",
 					Type:      counterMetric,
 				},
-				Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsFinished)),
+				Value: float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDisk)),
 			},
 			{
 				Description: MetricDescription{
@@ -1272,12 +1272,12 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Help:      "Total number of object versions checked for ilm actions since server start",
 					Type:      counterMetric,
 				},
-				Value: float64(atomic.LoadUint64(&globalScannerStats.ilmChecks)),
+				Value: float64(globalScannerMetrics.lifetime(scannerMetricILM)),
 			},
 		}
-		for i := range globalScannerStats.actions {
+		for i := range globalScannerMetrics.actions {
 			action := lifecycle.Action(i)
-			v := atomic.LoadUint64(&globalScannerStats.actions[action])
+			v := globalScannerMetrics.lifetimeActions(action)
 			if v == 0 {
 				continue
 			}
@@ -1861,11 +1861,10 @@ func getLocalDiskStorageMetrics() *MetricsGroup {
 			if disk.Metrics == nil {
 				continue
 			}
-			for apiName, latency := range disk.Metrics.APILatencies {
-				val := latency.(uint64)
+			for apiName, latency := range disk.Metrics.LastMinute {
 				metrics = append(metrics, Metric{
 					Description:    getNodeDiskAPILatencyMD(),
-					Value:          float64(val / 1000),
+					Value:          float64(latency.Avg().Microseconds()),
 					VariableLabels: map[string]string{"disk": disk.DrivePath, "api": "storage." + apiName},
 				})
 			}
@@ -1904,12 +1903,12 @@ func getClusterStorageMetrics() *MetricsGroup {
 
 		metrics = append(metrics, Metric{
 			Description: getClusterCapacityUsageBytesMD(),
-			Value:       GetTotalUsableCapacity(storageInfo.Disks, storageInfo),
+			Value:       float64(GetTotalUsableCapacity(storageInfo.Disks, storageInfo)),
 		})
 
 		metrics = append(metrics, Metric{
 			Description: getClusterCapacityUsageFreeBytesMD(),
-			Value:       GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo),
+			Value:       float64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)),
 		})
 
 		metrics = append(metrics, Metric{
@@ -2130,55 +2129,89 @@ func metricsServerHandler() http.Handler {
 	registry := prometheus.NewRegistry()
 
 	// Report all other metrics
-	err := registry.Register(clusterCollector)
-	if err != nil {
-		logger.CriticalIf(GlobalContext, err)
-	}
+	logger.CriticalIf(GlobalContext, registry.Register(clusterCollector))
 
 	// DefaultGatherers include golang metrics and process metrics.
 	gatherers := prometheus.Gatherers{
 		registry,
 	}
 
-	// Delegate http serving to Prometheus client library, which will call collector.Collect.
-	return promhttp.InstrumentMetricHandler(
-		registry,
-		promhttp.HandlerFor(gatherers,
-			promhttp.HandlerOpts{
-				ErrorHandling: promhttp.ContinueOnError,
-			}),
-	)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+		if ok {
+			tc.funcName = "handler.MetricsCluster"
+			tc.responseRecorder.LogErrBody = true
+		}
+
+		mfs, err := gatherers.Gather()
+		if err != nil {
+			if len(mfs) == 0 {
+				writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
+				return
+			}
+		}
+
+		contentType := expfmt.Negotiate(r.Header)
+		w.Header().Set("Content-Type", string(contentType))
+
+		enc := expfmt.NewEncoder(w, contentType)
+		for _, mf := range mfs {
+			if err := enc.Encode(mf); err != nil {
+				logger.LogIf(r.Context(), err)
+				return
+			}
+		}
+		if closer, ok := enc.(expfmt.Closer); ok {
+			closer.Close()
+		}
+	})
 }
 
 func metricsNodeHandler() http.Handler {
 	registry := prometheus.NewRegistry()
 
-	err := registry.Register(nodeCollector)
-	if err != nil {
-		logger.CriticalIf(GlobalContext, err)
-	}
-	err = registry.Register(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{
+	logger.CriticalIf(GlobalContext, registry.Register(nodeCollector))
+	if err := registry.Register(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{
 		Namespace:    minioNamespace,
 		ReportErrors: true,
-	}))
-	if err != nil {
+	})); err != nil {
 		logger.CriticalIf(GlobalContext, err)
 	}
-	err = registry.Register(prometheus.NewGoCollector())
-	if err != nil {
+	if err := registry.Register(prometheus.NewGoCollector()); err != nil {
 		logger.CriticalIf(GlobalContext, err)
 	}
 	gatherers := prometheus.Gatherers{
 		registry,
 	}
-	// Delegate http serving to Prometheus client library, which will call collector.Collect.
-	return promhttp.InstrumentMetricHandler(
-		registry,
-		promhttp.HandlerFor(gatherers,
-			promhttp.HandlerOpts{
-				ErrorHandling: promhttp.ContinueOnError,
-			}),
-	)
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+		if ok {
+			tc.funcName = "handler.MetricsNode"
+			tc.responseRecorder.LogErrBody = true
+		}
+
+		mfs, err := gatherers.Gather()
+		if err != nil {
+			if len(mfs) == 0 {
+				writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
+				return
+			}
+		}
+
+		contentType := expfmt.Negotiate(r.Header)
+		w.Header().Set("Content-Type", string(contentType))
+
+		enc := expfmt.NewEncoder(w, contentType)
+		for _, mf := range mfs {
+			if err := enc.Encode(mf); err != nil {
+				logger.LogIf(r.Context(), err)
+				return
+			}
+		}
+		if closer, ok := enc.(expfmt.Closer); ok {
+			closer.Close()
+		}
+	})
 }
 
 func toSnake(camel string) (snake string) {

cmd/metrics.go

@@ -26,7 +26,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/expfmt"
 )
 
 var (
@@ -577,7 +577,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total usable capacity online in the cluster",
 			nil, nil),
 		prometheus.GaugeValue,
-		GetTotalUsableCapacity(server.Disks, s),
+		float64(GetTotalUsableCapacity(server.Disks, s)),
 	)
 	// Report total usable capacity free
 	ch <- prometheus.MustNewConstMetric(
@@ -586,7 +586,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total free usable capacity online in the cluster",
 			nil, nil),
 		prometheus.GaugeValue,
-		GetTotalUsableCapacityFree(server.Disks, s),
+		float64(GetTotalUsableCapacityFree(server.Disks, s)),
 	)
 
 	// MinIO Offline Disks per node
@@ -648,35 +648,59 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 func metricsHandler() http.Handler {
 	registry := prometheus.NewRegistry()
 
-	err := registry.Register(minioVersionInfo)
-	logger.LogIf(GlobalContext, err)
+	logger.CriticalIf(GlobalContext, registry.Register(minioVersionInfo))
 
-	err = registry.Register(httpRequestsDuration)
-	logger.LogIf(GlobalContext, err)
-
-	err = registry.Register(newMinioCollector())
-	logger.LogIf(GlobalContext, err)
+	logger.CriticalIf(GlobalContext, registry.Register(newMinioCollector()))
 
 	gatherers := prometheus.Gatherers{
 		prometheus.DefaultGatherer,
 		registry,
 	}
-	// Delegate http serving to Prometheus client library, which will call collector.Collect.
-	return promhttp.InstrumentMetricHandler(
-		registry,
-		promhttp.HandlerFor(gatherers,
-			promhttp.HandlerOpts{
-				ErrorHandling: promhttp.ContinueOnError,
-			}),
-	)
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+		if ok {
+			tc.funcName = "handler.MetricsLegacy"
+			tc.responseRecorder.LogErrBody = true
+		}
+
+		mfs, err := gatherers.Gather()
+		if err != nil {
+			if len(mfs) == 0 {
+				writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
+				return
+			}
+		}
+
+		contentType := expfmt.Negotiate(r.Header)
+		w.Header().Set("Content-Type", string(contentType))
+
+		enc := expfmt.NewEncoder(w, contentType)
+		for _, mf := range mfs {
+			if err := enc.Encode(mf); err != nil {
+				logger.LogIf(r.Context(), err)
+				return
+			}
+		}
+		if closer, ok := enc.(expfmt.Closer); ok {
+			closer.Close()
+		}
+	})
 }
 
 // AuthMiddleware checks if the bearer token is valid and authorized.
 func AuthMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		claims, groups, owner, authErr := metricsRequestAuthenticate(r)
 		if authErr != nil || !claims.VerifyIssuer("prometheus", true) {
-			w.WriteHeader(http.StatusForbidden)
+			if ok {
+				tc.funcName = "handler.MetricsAuth"
+				tc.responseRecorder.LogErrBody = true
+			}
+
+			writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), errAuthentication), r.URL)
 			return
 		}
 		// For authenticated users apply IAM policy.
@@ -688,7 +712,12 @@ func AuthMiddleware(h http.Handler) http.Handler {
 			IsOwner:         owner,
 			Claims:          claims.Map(),
 		}) {
-			w.WriteHeader(http.StatusForbidden)
+			if ok {
+				tc.funcName = "handler.MetricsAuth"
+				tc.responseRecorder.LogErrBody = true
+			}
+
+			writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), errAuthentication), r.URL)
 			return
 		}
 		h.ServeHTTP(w, r)

cmd/notification-summary.go

@@ -30,17 +30,17 @@ func GetTotalCapacity(diskInfo []madmin.Disk) (capacity uint64) {
 }
 
 // GetTotalUsableCapacity gets the total usable capacity in the cluster.
-// This value is not an accurate representation of total usable in a multi-tenant deployment.
-func GetTotalUsableCapacity(diskInfo []madmin.Disk, s StorageInfo) (capacity float64) {
-	raw := GetTotalCapacity(diskInfo)
-	var approxDataBlocks float64
-	var actualDisks float64
-	for _, scData := range s.Backend.StandardSCData {
-		approxDataBlocks += float64(scData)
-		actualDisks += float64(scData + s.Backend.StandardSCParity)
+func GetTotalUsableCapacity(diskInfo []madmin.Disk, s StorageInfo) (capacity uint64) {
+	if globalIsGateway {
+		return 0
 	}
-	ratio := approxDataBlocks / actualDisks
-	return float64(raw) * ratio
+	for _, disk := range diskInfo {
+		// Ignore parity disks
+		if disk.DiskIndex < s.Backend.StandardSCData[disk.PoolIndex] {
+			capacity += disk.TotalSpace
+		}
+	}
+	return
 }
 
 // GetTotalCapacityFree gets the total capacity free in the cluster.
@@ -52,15 +52,16 @@ func GetTotalCapacityFree(diskInfo []madmin.Disk) (capacity uint64) {
 }
 
 // GetTotalUsableCapacityFree gets the total usable capacity free in the cluster.
-// This value is not an accurate representation of total free in a multi-tenant deployment.
-func GetTotalUsableCapacityFree(diskInfo []madmin.Disk, s StorageInfo) (capacity float64) {
-	raw := GetTotalCapacityFree(diskInfo)
-	var approxDataBlocks float64
-	var actualDisks float64
-	for _, scData := range s.Backend.StandardSCData {
-		approxDataBlocks += float64(scData)
-		actualDisks += float64(scData + s.Backend.StandardSCParity)
+func GetTotalUsableCapacityFree(diskInfo []madmin.Disk, s StorageInfo) (capacity uint64) {
+	if globalIsGateway {
+		return 0
 	}
-	ratio := approxDataBlocks / actualDisks
-	return float64(raw) * ratio
+
+	for _, disk := range diskInfo {
+		// Ignore parity disks
+		if disk.DiskIndex < s.Backend.StandardSCData[disk.PoolIndex] {
+			capacity += disk.AvailableSpace
+		}
+	}
+	return
 }

cmd/notification.go

@@ -929,6 +929,38 @@ func (sys *NotificationSys) GetOSInfo(ctx context.Context) []madmin.OSInfo {
 	return reply
 }
 
+// GetMetrics - Get metrics from all peers.
+func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType, hosts map[string]struct{}) []madmin.RealtimeMetrics {
+	reply := make([]madmin.RealtimeMetrics, len(sys.peerClients))
+
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		host := client.host.String()
+		if len(hosts) > 0 {
+			if _, ok := hosts[host]; !ok {
+				continue
+			}
+		}
+
+		index := index
+		g.Go(func() error {
+			var err error
+			reply[index], err = sys.peerClients[index].GetMetrics(ctx, t)
+			return err
+		}, index)
+	}
+
+	for index, err := range g.Wait() {
+		if err != nil {
+			reply[index].Errors = []string{err.Error()}
+		}
+	}
+	return reply
+}
+
 // GetSysConfig - Get information about system config
 // (only the config that are of concern to minio)
 func (sys *NotificationSys) GetSysConfig(ctx context.Context) []madmin.SysConfig {
@@ -1288,8 +1320,7 @@ func sendEvent(args eventArgs) {
 	if globalNotificationSys == nil {
 		return
 	}
-
-	if globalHTTPListen.NumSubscribers() > 0 {
+	if globalHTTPListen.NumSubscribers(args.EventName) > 0 {
 		globalHTTPListen.Publish(args.ToEvent(false))
 	}
 

cmd/object-api-errors.go

@@ -541,7 +541,7 @@ type IncompleteBody GenericError
 
 // Error returns string an error formatted as the given text.
 func (e IncompleteBody) Error() string {
-	return e.Bucket + "/" + e.Object + "has incomplete body"
+	return e.Bucket + "/" + e.Object + " has incomplete body"
 }
 
 // InvalidRange - invalid range typed error.

cmd/object-handlers.go

@@ -420,11 +420,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 
 	gr, err := getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	if err != nil {
-		if isErrPreconditionFailed(err) {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
-		}
-
 		var (
 			reader *GetObjectReader
 			proxy  proxyResult
@@ -434,9 +429,12 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		if !proxytgts.Empty() {
 			// proxy to replication target if active-active replication is in place.
 			reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
-			if perr != nil && !isErrObjectNotFound(ErrorRespToObjectError(perr, bucket, object)) &&
-				!isErrVersionNotFound(ErrorRespToObjectError(perr, bucket, object)) {
-				logger.LogIf(ctx, fmt.Errorf("Replication proxy failed for %s/%s(%s) - %w", bucket, object, opts.VersionID, perr))
+			if perr != nil {
+				proxyGetErr := ErrorRespToObjectError(perr, bucket, object)
+				if !isErrObjectNotFound(proxyGetErr) && !isErrVersionNotFound(proxyGetErr) &&
+					!isErrPreconditionFailed(proxyGetErr) && !isErrInvalidRange(proxyGetErr) {
+					logger.LogIf(ctx, fmt.Errorf("Replication proxy failed for %s/%s(%s) - %w", bucket, object, opts.VersionID, perr))
+				}
 			}
 			if reader != nil && proxy.Proxy && perr == nil {
 				gr = reader
@@ -1446,9 +1444,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	var objInfo ObjectInfo
+	remoteCallRequired := isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
 
-	if isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) {
+	var objInfo ObjectInfo
+	var os *objSweeper
+	if remoteCallRequired {
 		var dstRecords []dns.SrvRecord
 		dstRecords, err = globalDNSConfig.Get(dstBucket)
 		if err != nil {
@@ -1484,8 +1484,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		objInfo.ETag = remoteObjInfo.ETag
 		objInfo.ModTime = remoteObjInfo.LastModified
 	} else {
-
-		os := newObjSweeper(dstBucket, dstObject).WithVersioning(dstOpts.Versioned, dstOpts.VersionSuspended)
+		os = newObjSweeper(dstBucket, dstObject).WithVersioning(dstOpts.Versioned, dstOpts.VersionSuspended)
 		// Get appropriate object info to identify the remote object to delete
 		if !srcInfo.metadataOnly {
 			goiOpts := os.GetOpts()
@@ -1508,11 +1507,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-
-		// Remove the transitioned object whose object version is being overwritten.
-		if !globalTierConfigMgr.Empty() {
-			logger.LogIf(ctx, os.Sweep())
-		}
 	}
 
 	objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
@@ -1544,9 +1538,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
-	if !globalTierConfigMgr.Empty() {
+
+	if !remoteCallRequired && !globalTierConfigMgr.Empty() {
 		// Schedule object for immediate transition if eligible.
 		enqueueTransitionImmediate(objInfo)
+		// Remove the transitioned object whose object version is being overwritten.
+		logger.LogIf(ctx, os.Sweep())
 	}
 }
 
@@ -3461,15 +3458,17 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			writeErrorResponse(ctx, w, toAPIError(ctx, errors.New("force-delete is forbidden in a locked-enabled bucket")), r.URL)
 			return
 		}
-		apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
-			ObjectV: ObjectV{
-				ObjectName: object,
-				VersionID:  vID,
-			},
-		}, goi, gerr)
-		if apiErr != ErrNone && apiErr != ErrNoSuchKey {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
-			return
+		if vID != "" {
+			apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
+				ObjectV: ObjectV{
+					ObjectName: object,
+					VersionID:  vID,
+				},
+			}, goi, gerr)
+			if apiErr != ErrNone && apiErr != ErrNoSuchKey {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
+				return
+			}
 		}
 	}
 

cmd/os-instrumented.go

@@ -20,10 +20,12 @@ package cmd
 import (
 	"os"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/disk"
+	ioutilx "github.com/minio/minio/internal/ioutil"
 )
 
 //go:generate stringer -type=osMetric -trimprefix=osMetric $GOFILE
@@ -41,33 +43,67 @@ const (
 	osMetricRemove
 	osMetricStat
 	osMetricAccess
+	osMetricCreate
+	osMetricReadDirent
+	osMetricFdatasync
+	osMetricSync
 	// .... add more
 
 	osMetricLast
 )
 
+var globalOSMetrics osMetrics
+
+func init() {
+	// Inject metrics.
+	ioutilx.OsOpenFile = OpenFile
+	ioutilx.OpenFileDirectIO = OpenFileDirectIO
+	ioutilx.OsOpen = Open
+}
+
+type osMetrics struct {
+	// All fields must be accessed atomically and aligned.
+	operations [osMetricLast]uint64
+	latency    [osMetricLast]lockedLastMinuteLatency
+}
+
+// time an os action.
+func (o *osMetrics) time(s osMetric) func() {
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&o.operations[s], 1)
+		o.latency[s].add(duration)
+	}
+}
+
+// incTime will increment time on metric s with a specific duration.
+func (o *osMetrics) incTime(s osMetric, d time.Duration) {
+	atomic.AddUint64(&o.operations[s], 1)
+	o.latency[s].add(d)
+}
+
 func osTrace(s osMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceOS,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  "os." + s.String(),
-		OSStats: madmin.TraceOSStats{
-			Duration: duration,
-			Path:     path,
-		},
+		Duration:  duration,
+		Path:      path,
 	}
 }
 
 func updateOSMetrics(s osMetric, paths ...string) func() {
-	if globalTrace.NumSubscribers() == 0 {
-		return func() {}
+	if globalTrace.NumSubscribers(madmin.TraceOS) == 0 {
+		return globalOSMetrics.time(s)
 	}
 
 	startTime := time.Now()
 	return func() {
 		duration := time.Since(startTime)
-
+		globalOSMetrics.incTime(s, duration)
 		globalTrace.Publish(osTrace(s, startTime, duration, strings.Join(paths, " -> ")))
 	}
 }
@@ -133,3 +169,47 @@ func Stat(name string) (os.FileInfo, error) {
 	defer updateOSMetrics(osMetricStat, name)()
 	return os.Stat(name)
 }
+
+// Create captures time taken to call os.Create
+func Create(name string) (*os.File, error) {
+	defer updateOSMetrics(osMetricCreate, name)()
+	return os.Create(name)
+}
+
+// Fdatasync captures time taken to call Fdatasync
+func Fdatasync(f *os.File) error {
+	fn := ""
+	if f != nil {
+		fn = f.Name()
+	}
+	defer updateOSMetrics(osMetricFdatasync, fn)()
+	return disk.Fdatasync(f)
+}
+
+// report returns all os metrics.
+func (o *osMetrics) report() madmin.OSMetrics {
+	var m madmin.OSMetrics
+	m.CollectedAt = time.Now()
+	m.LifeTimeOps = make(map[string]uint64, osMetricLast)
+	for i := osMetric(0); i < osMetricLast; i++ {
+		if n := atomic.LoadUint64(&o.operations[i]); n > 0 {
+			m.LifeTimeOps[i.String()] = n
+		}
+	}
+	if len(m.LifeTimeOps) == 0 {
+		m.LifeTimeOps = nil
+	}
+
+	m.LastMinute.Operations = make(map[string]madmin.TimedAction, osMetricLast)
+	for i := osMetric(0); i < osMetricLast; i++ {
+		lm := o.latency[i].total()
+		if lm.N > 0 {
+			m.LastMinute.Operations[i.String()] = lm.asTimedAction()
+		}
+	}
+	if len(m.LastMinute.Operations) == 0 {
+		m.LastMinute.Operations = nil
+	}
+
+	return m
+}

cmd/os-readdir_other.go

@@ -153,5 +153,6 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 
 func globalSync() {
 	// no-op not sure about plan9/solaris support for syscall support
+	defer globalOSMetrics.time(osMetricSync)
 	syscall.Sync()
 }

cmd/os-readdir_unix.go

@@ -122,7 +122,9 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 	for {
 		if boff >= nbuf {
 			boff = 0
+			stop := globalOSMetrics.time(osMetricReadDirent)
 			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
+			stop()
 			if err != nil {
 				if isSysErrNotDir(err) {
 					return nil
@@ -150,7 +152,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 		// support Dirent.Type and have DT_UNKNOWN (0) there
 		// instead.
 		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
-			fi, err := os.Stat(pathJoin(dirPath, string(name)))
+			fi, err := Stat(pathJoin(dirPath, string(name)))
 			if err != nil {
 				// It got deleted in the meantime, not found
 				// or returns too many symlinks ignore this
@@ -203,7 +205,9 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 	for count != 0 {
 		if boff >= nbuf {
 			boff = 0
+			stop := globalOSMetrics.time(osMetricReadDirent)
 			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
+			stop()
 			if err != nil {
 				if isSysErrNotDir(err) {
 					return nil, errFileNotFound
@@ -266,5 +270,6 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 }
 
 func globalSync() {
+	defer globalOSMetrics.time(osMetricSync)
 	syscall.Sync()
 }

cmd/os-readdir_windows.go

@@ -35,7 +35,7 @@ func access(name string) error {
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
 func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
-	f, err := os.Open(dirPath)
+	f, err := Open(dirPath)
 	if err != nil {
 		if osErrToFileErr(err) == errFileNotFound {
 			return nil
@@ -116,7 +116,7 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 
 // Return N entries at the directory dirPath.
 func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
-	f, err := os.Open(dirPath)
+	f, err := Open(dirPath)
 	if err != nil {
 		return nil, osErrToFileErr(err)
 	}

cmd/osmetric_string.go

@@ -18,12 +18,16 @@ func _() {
 	_ = x[osMetricRemove-7]
 	_ = x[osMetricStat-8]
 	_ = x[osMetricAccess-9]
-	_ = x[osMetricLast-10]
+	_ = x[osMetricCreate-10]
+	_ = x[osMetricReadDirent-11]
+	_ = x[osMetricFdatasync-12]
+	_ = x[osMetricSync-13]
+	_ = x[osMetricLast-14]
 }
 
-const _osMetric_name = "RemoveAllMkdirAllRenameOpenFileOpenOpenFileDirectIOLstatRemoveStatAccessLast"
+const _osMetric_name = "RemoveAllMkdirAllRenameOpenFileOpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncLast"
 
-var _osMetric_index = [...]uint8{0, 9, 17, 23, 31, 35, 51, 56, 62, 66, 72, 76}
+var _osMetric_index = [...]uint8{0, 9, 17, 23, 31, 35, 51, 56, 62, 66, 72, 78, 88, 97, 101, 105}
 
 func (i osMetric) String() string {
 	if i >= osMetric(len(_osMetric_index)-1) {

cmd/peer-rest-client.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/http"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/minio/internal/rest"
 	xnet "github.com/minio/pkg/net"
 	"github.com/tinylib/msgp/msgp"
@@ -193,6 +194,19 @@ func (client *peerRESTClient) GetMemInfo(ctx context.Context) (info madmin.MemIn
 	return info, err
 }
 
+// GetMetrics - fetch metrics from a remote node.
+func (client *peerRESTClient) GetMetrics(ctx context.Context, t madmin.MetricType) (info madmin.RealtimeMetrics, err error) {
+	values := make(url.Values)
+	values.Set(peerRESTTypes, strconv.FormatUint(uint64(t), 10))
+	respBody, err := client.callWithContext(ctx, peerRESTMethodMetrics, values, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
 // GetProcInfo - fetch MinIO process information for a remote node.
 func (client *peerRESTClient) GetProcInfo(ctx context.Context) (info madmin.ProcInfo, err error) {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodProcInfo, nil, nil, -1)
@@ -532,14 +546,9 @@ func (client *peerRESTClient) LoadTransitionTierConfig(ctx context.Context) erro
 	return nil
 }
 
-func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh <-chan struct{}, traceOpts madmin.ServiceTraceOpts) {
+func (client *peerRESTClient) doTrace(traceCh chan<- pubsub.Maskable, doneCh <-chan struct{}, traceOpts madmin.ServiceTraceOpts) {
 	values := make(url.Values)
-	values.Set(peerRESTTraceErr, strconv.FormatBool(traceOpts.OnlyErrors))
-	values.Set(peerRESTTraceS3, strconv.FormatBool(traceOpts.S3))
-	values.Set(peerRESTTraceStorage, strconv.FormatBool(traceOpts.Storage))
-	values.Set(peerRESTTraceOS, strconv.FormatBool(traceOpts.OS))
-	values.Set(peerRESTTraceInternal, strconv.FormatBool(traceOpts.Internal))
-	values.Set(peerRESTTraceThreshold, traceOpts.Threshold.String())
+	traceOpts.AddParams(values)
 
 	// To cancel the REST request in case doneCh gets closed.
 	ctx, cancel := context.WithCancel(GlobalContext)
@@ -578,7 +587,7 @@ func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh <-chan st
 	}
 }
 
-func (client *peerRESTClient) doListen(listenCh chan interface{}, doneCh <-chan struct{}, v url.Values) {
+func (client *peerRESTClient) doListen(listenCh chan<- pubsub.Maskable, doneCh <-chan struct{}, v url.Values) {
 	// To cancel the REST request in case doneCh gets closed.
 	ctx, cancel := context.WithCancel(GlobalContext)
 
@@ -617,7 +626,7 @@ func (client *peerRESTClient) doListen(listenCh chan interface{}, doneCh <-chan
 }
 
 // Listen - listen on peers.
-func (client *peerRESTClient) Listen(listenCh chan interface{}, doneCh <-chan struct{}, v url.Values) {
+func (client *peerRESTClient) Listen(listenCh chan<- pubsub.Maskable, doneCh <-chan struct{}, v url.Values) {
 	go func() {
 		for {
 			client.doListen(listenCh, doneCh, v)
@@ -633,7 +642,7 @@ func (client *peerRESTClient) Listen(listenCh chan interface{}, doneCh <-chan st
 }
 
 // Trace - send http trace request to peer nodes
-func (client *peerRESTClient) Trace(traceCh chan interface{}, doneCh <-chan struct{}, traceOpts madmin.ServiceTraceOpts) {
+func (client *peerRESTClient) Trace(traceCh chan<- pubsub.Maskable, doneCh <-chan struct{}, traceOpts madmin.ServiceTraceOpts) {
 	go func() {
 		for {
 			client.doTrace(traceCh, doneCh, traceOpts)
@@ -648,7 +657,7 @@ func (client *peerRESTClient) Trace(traceCh chan interface{}, doneCh <-chan stru
 	}()
 }
 
-func (client *peerRESTClient) doConsoleLog(logCh chan interface{}, doneCh <-chan struct{}) {
+func (client *peerRESTClient) doConsoleLog(logCh chan pubsub.Maskable, doneCh <-chan struct{}) {
 	// To cancel the REST request in case doneCh gets closed.
 	ctx, cancel := context.WithCancel(GlobalContext)
 
@@ -686,7 +695,7 @@ func (client *peerRESTClient) doConsoleLog(logCh chan interface{}, doneCh <-chan
 }
 
 // ConsoleLog - sends request to peer nodes to get console logs
-func (client *peerRESTClient) ConsoleLog(logCh chan interface{}, doneCh <-chan struct{}) {
+func (client *peerRESTClient) ConsoleLog(logCh chan pubsub.Maskable, doneCh <-chan struct{}) {
 	go func() {
 		for {
 			client.doConsoleLog(logCh, doneCh)

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v22" // Add bulk GetBucketStats
+	peerRESTVersion       = "v23" // Added /metrics
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -70,30 +70,26 @@ const (
 	peerRESTMethodGetLastDayTierStats         = "/getlastdaytierstats"
 	peerRESTMethodDevNull                     = "/devnull"
 	peerRESTMethodNetperf                     = "/netperf"
+	peerRESTMethodMetrics                     = "/metrics"
 )
 
 const (
-	peerRESTBucket         = "bucket"
-	peerRESTBuckets        = "buckets"
-	peerRESTUser           = "user"
-	peerRESTGroup          = "group"
-	peerRESTUserTemp       = "user-temp"
-	peerRESTPolicy         = "policy"
-	peerRESTUserOrGroup    = "user-or-group"
-	peerRESTIsGroup        = "is-group"
-	peerRESTSignal         = "signal"
-	peerRESTSubSys         = "sub-sys"
-	peerRESTProfiler       = "profiler"
-	peerRESTTraceErr       = "err"
-	peerRESTTraceInternal  = "internal"
-	peerRESTTraceStorage   = "storage"
-	peerRESTTraceS3        = "s3"
-	peerRESTTraceOS        = "os"
-	peerRESTTraceThreshold = "threshold"
-	peerRESTSize           = "size"
-	peerRESTConcurrent     = "concurrent"
-	peerRESTDuration       = "duration"
-	peerRESTStorageClass   = "storage-class"
+	peerRESTBucket       = "bucket"
+	peerRESTBuckets      = "buckets"
+	peerRESTUser         = "user"
+	peerRESTGroup        = "group"
+	peerRESTUserTemp     = "user-temp"
+	peerRESTPolicy       = "policy"
+	peerRESTUserOrGroup  = "user-or-group"
+	peerRESTIsGroup      = "is-group"
+	peerRESTSignal       = "signal"
+	peerRESTSubSys       = "sub-sys"
+	peerRESTProfiler     = "profiler"
+	peerRESTSize         = "size"
+	peerRESTConcurrent   = "concurrent"
+	peerRESTDuration     = "duration"
+	peerRESTStorageClass = "storage-class"
+	peerRESTTypes        = "types"
 
 	peerRESTListenBucket = "bucket"
 	peerRESTListenPrefix = "prefix"

cmd/peer-rest-server.go

@@ -35,6 +35,7 @@ import (
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/pubsub"
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -411,6 +412,28 @@ func (s *peerRESTServer) GetMemInfoHandler(w http.ResponseWriter, r *http.Reques
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
+// GetMetricsHandler - returns server metrics.
+func (s *peerRESTServer) GetMetricsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	var types madmin.MetricType
+	if t, _ := strconv.ParseUint(r.Form.Get(peerRESTTypes), 10, 64); t != 0 {
+		types = madmin.MetricType(t)
+	} else {
+		types = madmin.MetricsAll
+	}
+
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+
+	info := collectLocalMetrics(types, nil)
+
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
 // GetSysConfigHandler - returns system config information.
 // (only the config that are of concern to minio)
 func (s *peerRESTServer) GetSysConfigHandler(w http.ResponseWriter, r *http.Request) {
@@ -853,26 +876,26 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 	pattern := event.NewPattern(prefix, suffix)
 
 	var eventNames []event.Name
+	var mask pubsub.Mask
 	for _, ev := range values[peerRESTListenEvents] {
 		eventName, err := event.ParseName(ev)
 		if err != nil {
 			s.writeErrorResponse(w, err)
 			return
 		}
-
+		mask.MergeMaskable(eventName)
 		eventNames = append(eventNames, eventName)
 	}
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()})
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
+	doneCh := r.Context().Done()
 
 	// Listen Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan interface{}, 2000)
+	ch := make(chan pubsub.Maskable, 2000)
 
-	listenFn := func(evI interface{}) bool {
+	err := globalHTTPListen.Subscribe(mask, ch, doneCh, func(evI pubsub.Maskable) bool {
 		ev, ok := evI.(event.Event)
 		if !ok {
 			return false
@@ -883,14 +906,11 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
-	}
-
-	err := globalHTTPListen.Subscribe(ch, doneCh, listenFn)
+	})
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 
@@ -901,7 +921,10 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 			if err := enc.Encode(ev); err != nil {
 				return
 			}
-			w.(http.Flusher).Flush()
+			if len(ch) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
+			}
 		case <-r.Context().Done():
 			return
 		case <-keepAliveTicker.C:
@@ -913,23 +936,6 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func extractTraceOptsFromPeerRequest(r *http.Request) (opts madmin.ServiceTraceOpts, err error) {
-	opts.S3 = r.Form.Get(peerRESTTraceS3) == "true"
-	opts.OS = r.Form.Get(peerRESTTraceOS) == "true"
-	opts.Storage = r.Form.Get(peerRESTTraceStorage) == "true"
-	opts.Internal = r.Form.Get(peerRESTTraceInternal) == "true"
-	opts.OnlyErrors = r.Form.Get(peerRESTTraceErr) == "true"
-
-	if t := r.Form.Get(peerRESTTraceThreshold); t != "" {
-		d, err := time.ParseDuration(t)
-		if err != nil {
-			return opts, err
-		}
-		opts.Threshold = d
-	}
-	return
-}
-
 // TraceHandler sends http trace messages back to peer rest client
 func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -937,29 +943,27 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	traceOpts, err := extractTraceOptsFromPeerRequest(r)
+	var traceOpts madmin.ServiceTraceOpts
+	err := traceOpts.ParseParams(r)
 	if err != nil {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
 		return
 	}
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
 	// Trace Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan interface{}, 2000)
-
-	traceFn := func(entry interface{}) bool {
-		return mustTrace(entry, traceOpts)
-	}
-
-	err = globalTrace.Subscribe(ch, doneCh, traceFn)
+	ch := make(chan pubsub.Maskable, 2000)
+	mask := pubsub.MaskFromMaskable(traceOpts.TraceTypes())
+	err = globalTrace.Subscribe(mask, ch, r.Context().Done(), func(entry pubsub.Maskable) bool {
+		if e, ok := entry.(madmin.TraceInfo); ok {
+			return shouldTrace(e, traceOpts)
+		}
+		return false
+	})
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 
@@ -970,7 +974,6 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 			if err := enc.Encode(entry); err != nil {
 				return
 			}
-			w.(http.Flusher).Flush()
 		case <-r.Context().Done():
 			return
 		case <-keepAliveTicker.C:
@@ -1042,13 +1045,12 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	ch := make(chan interface{}, 2000)
-	err := globalConsoleSys.Subscribe(ch, doneCh, "", 0, string(logger.All), nil)
+	ch := make(chan pubsub.Maskable, 2000)
+	err := globalConsoleSys.Subscribe(ch, doneCh, "", 0, madmin.LogMaskAll, nil)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 
@@ -1275,6 +1277,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(httpTraceHdrs(server.GetProcInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(httpTraceHdrs(server.GetMemInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(httpTraceHdrs(server.GetMetricsHandler)).Queries(restQueries(peerRESTTypes)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(httpTraceHdrs(server.GetSysErrorsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(httpTraceHdrs(server.GetSysServicesHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(httpTraceHdrs(server.GetSysConfigHandler))

cmd/scannermetric_string.go

@@ -0,0 +1,39 @@
+// Code generated by "stringer -type=scannerMetric -trimprefix=scannerMetric data-scanner-metric.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[scannerMetricReadMetadata-0]
+	_ = x[scannerMetricCheckMissing-1]
+	_ = x[scannerMetricSaveUsage-2]
+	_ = x[scannerMetricApplyAll-3]
+	_ = x[scannerMetricApplyVersion-4]
+	_ = x[scannerMetricTierObjSweep-5]
+	_ = x[scannerMetricHealCheck-6]
+	_ = x[scannerMetricILM-7]
+	_ = x[scannerMetricCheckReplication-8]
+	_ = x[scannerMetricYield-9]
+	_ = x[scannerMetricStartTrace-10]
+	_ = x[scannerMetricScanObject-11]
+	_ = x[scannerMetricLastRealtime-12]
+	_ = x[scannerMetricScanFolder-13]
+	_ = x[scannerMetricScanCycle-14]
+	_ = x[scannerMetricScanBucketDisk-15]
+	_ = x[scannerMetricLast-16]
+}
+
+const _scannerMetric_name = "ReadMetadataCheckMissingSaveUsageApplyAllApplyVersionTierObjSweepHealCheckILMCheckReplicationYieldStartTraceScanObjectLastRealtimeScanFolderScanCycleScanBucketDiskLast"
+
+var _scannerMetric_index = [...]uint8{0, 12, 24, 33, 41, 53, 65, 74, 77, 93, 98, 108, 118, 130, 140, 149, 163, 167}
+
+func (i scannerMetric) String() string {
+	if i >= scannerMetric(len(_scannerMetric_index)-1) {
+		return "scannerMetric(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _scannerMetric_name[_scannerMetric_index[i]:_scannerMetric_index[i+1]]
+}

cmd/server-startup-msg.go

@@ -44,12 +44,19 @@ func mustGetStorageInfo(objAPI ObjectLayer) StorageInfo {
 
 // Prints the formatted startup message.
 func printStartupMessage(apiEndpoints []string, err error) {
+	logger.Info(color.Bold("MinIO Object Storage Server"))
 	if err != nil {
 		if globalConsoleSys != nil {
-			globalConsoleSys.Send(fmt.Sprintf("Server startup failed with '%v', some features may be missing", err), string(logger.All))
+			globalConsoleSys.Send(fmt.Sprintf("Server startup failed with '%v', some features may be missing", err))
 		}
 	}
 
+	if len(globalSubnetConfig.APIKey) == 0 && err == nil {
+		var builder strings.Builder
+		startupBanner(&builder)
+		logger.Info(builder.String())
+	}
+
 	strippedAPIEndpoints := stripStandardPorts(apiEndpoints, globalMinioHost)
 	// If cache layer is enabled, print cache capacity.
 	cachedObjAPI := newCachedObjectLayerFn()
@@ -72,12 +79,6 @@ func printStartupMessage(apiEndpoints []string, err error) {
 
 	// Prints documentation message.
 	printObjectAPIMsg()
-
-	if globalMinioConsolePortAuto && globalBrowserEnabled {
-		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
-			globalMinioConsolePort)
-		logger.Info(color.RedBold(msg))
-	}
 }
 
 // Returns true if input is IPv6
@@ -141,9 +142,13 @@ func printServerCommonMsg(apiEndpoints []string) {
 	}
 	printEventNotifiers()
 
+	if globalMinioConsolePortAuto && globalBrowserEnabled {
+		logger.Info(color.RedBold("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
+			globalMinioConsolePort))
+	}
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
-		logger.Info(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
+		logger.Info(color.Blue("Console: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
 		if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
 			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
 			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))

cmd/signature-v4-utils.go

@@ -152,16 +152,16 @@ func checkKeyValid(r *http.Request, accessKey string) (auth.Credentials, bool, A
 	cred := globalActiveCred
 	if cred.AccessKey != accessKey {
 		// Check if the access key is part of users credentials.
-		ucred, ok := globalIAMSys.GetUser(r.Context(), accessKey)
+		u, ok := globalIAMSys.GetUser(r.Context(), accessKey)
 		if !ok {
 			// Credentials will be invalid but and disabled
 			// return a different error in such a scenario.
-			if ucred.Status == auth.AccountOff {
+			if u.Credentials.Status == auth.AccountOff {
 				return cred, false, ErrAccessKeyDisabled
 			}
 			return cred, false, ErrInvalidAccessKeyID
 		}
-		cred = ucred
+		cred = u.Credentials
 	}
 
 	claims, s3Err := checkClaimsFromToken(r, cred)

cmd/site-replication.go

@@ -403,14 +403,15 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 
 	// Generate a secret key for the service account if not created already.
 	var secretKey string
-	svcCred, _, err := globalIAMSys.getServiceAccount(ctx, siteReplicatorSvcAcc)
+	var svcCred auth.Credentials
+	sa, _, err := globalIAMSys.getServiceAccount(ctx, siteReplicatorSvcAcc)
 	switch {
 	case err == errNoSuchServiceAccount:
 		_, secretKey, err = auth.GenerateCredentials()
 		if err != nil {
 			return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err))
 		}
-		svcCred, err = globalIAMSys.NewServiceAccount(ctx, sites[selfIdx].AccessKey, nil, newServiceAccountOpts{
+		svcCred, _, err = globalIAMSys.NewServiceAccount(ctx, sites[selfIdx].AccessKey, nil, newServiceAccountOpts{
 			accessKey: siteReplicatorSvcAcc,
 			secretKey: secretKey,
 		})
@@ -418,6 +419,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 			return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err))
 		}
 	case err == nil:
+		svcCred = sa.Credentials
 		secretKey = svcCred.SecretKey
 	default:
 		return madmin.ReplicateAddStatus{}, errSRBackendIssue(err)
@@ -525,7 +527,7 @@ func (c *SiteReplicationSys) PeerJoinReq(ctx context.Context, arg madmin.SRPeerJ
 
 	_, _, err := globalIAMSys.GetServiceAccount(ctx, arg.SvcAcctAccessKey)
 	if err == errNoSuchServiceAccount {
-		_, err = globalIAMSys.NewServiceAccount(ctx, arg.SvcAcctParent, nil, newServiceAccountOpts{
+		_, _, err = globalIAMSys.NewServiceAccount(ctx, arg.SvcAcctParent, nil, newServiceAccountOpts{
 			accessKey: arg.SvcAcctAccessKey,
 			secretKey: arg.SvcAcctSecretKey,
 		})
@@ -817,7 +819,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 			if err != nil {
 				return err
 			}
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 				return err
 			}
 			targetARN = bucketTarget.Arn
@@ -930,7 +932,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 			return err
 		}
 
-		err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, replCfgData)
+		_, err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, replCfgData)
 		return c.annotatePeerErr(peer.Name, "Error updating replication configuration", err)
 	}
 
@@ -1022,12 +1024,18 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 
 // PeerAddPolicyHandler - copies IAM policy to local. A nil policy argument,
 // causes the named policy to be deleted.
-func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyName string, p *iampolicy.Policy) error {
+func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyName string, p *iampolicy.Policy, updatedAt time.Time) error {
 	var err error
+	// skip overwrite of local update if peer sent stale info
+	if !updatedAt.IsZero() {
+		if p, err := globalIAMSys.store.GetPolicyDoc(policyName); err == nil && p.UpdateDate.After(updatedAt) {
+			return nil
+		}
+	}
 	if p == nil {
 		err = globalIAMSys.DeletePolicy(ctx, policyName, true)
 	} else {
-		err = globalIAMSys.SetPolicy(ctx, policyName, *p)
+		_, err = globalIAMSys.SetPolicy(ctx, policyName, *p)
 	}
 	if err != nil {
 		return wrapSRErr(err)
@@ -1036,10 +1044,17 @@ func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyNam
 }
 
 // PeerIAMUserChangeHandler - copies IAM user to local.
-func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, change *madmin.SRIAMUser) error {
+func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, change *madmin.SRIAMUser, updatedAt time.Time) error {
 	if change == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
+	// skip overwrite of local update if peer sent stale info
+	if !updatedAt.IsZero() {
+		if ui, err := globalIAMSys.GetUserInfo(ctx, change.AccessKey); err == nil && ui.UpdatedAt.After(updatedAt) {
+			return nil
+		}
+	}
+
 	var err error
 	if change.IsDeleteReq {
 		err = globalIAMSys.DeleteUser(ctx, change.AccessKey, true)
@@ -1051,9 +1066,9 @@ func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, chang
 		if userReq.Status != "" && userReq.SecretKey == "" {
 			// Status is set without secretKey updates means we are
 			// only changing the account status.
-			err = globalIAMSys.SetUserStatus(ctx, change.AccessKey, userReq.Status)
+			_, err = globalIAMSys.SetUserStatus(ctx, change.AccessKey, userReq.Status)
 		} else {
-			err = globalIAMSys.CreateUser(ctx, change.AccessKey, userReq)
+			_, err = globalIAMSys.CreateUser(ctx, change.AccessKey, userReq)
 		}
 	}
 	if err != nil {
@@ -1063,19 +1078,27 @@ func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, chang
 }
 
 // PeerGroupInfoChangeHandler - copies group changes to local.
-func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, change *madmin.SRGroupInfo) error {
+func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, change *madmin.SRGroupInfo, updatedAt time.Time) error {
 	if change == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
 	updReq := change.UpdateReq
 	var err error
+
+	// skip overwrite of local update if peer sent stale info
+	if !updatedAt.IsZero() {
+		if gd, err := globalIAMSys.GetGroupDescription(updReq.Group); err == nil && gd.UpdatedAt.After(updatedAt) {
+			return nil
+		}
+	}
+
 	if updReq.IsRemove {
-		err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
+		_, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
 	} else {
 		if updReq.Status != "" && len(updReq.Members) == 0 {
-			err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled)
+			_, err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled)
 		} else {
-			err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+			_, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
 		}
 	}
 	if err != nil {
@@ -1085,7 +1108,7 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
 }
 
 // PeerSvcAccChangeHandler - copies service-account change to local.
-func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change *madmin.SRSvcAccChange) error {
+func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change *madmin.SRSvcAccChange, updatedAt time.Time) error {
 	if change == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
@@ -1099,14 +1122,19 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 				return wrapSRErr(err)
 			}
 		}
-
+		// skip overwrite of local update if peer sent stale info
+		if !updatedAt.IsZero() && change.Create.AccessKey != "" {
+			if sa, _, err := globalIAMSys.getServiceAccount(ctx, change.Create.AccessKey); err == nil && sa.UpdatedAt.After(updatedAt) {
+				return nil
+			}
+		}
 		opts := newServiceAccountOpts{
 			accessKey:     change.Create.AccessKey,
 			secretKey:     change.Create.SecretKey,
 			sessionPolicy: sp,
 			claims:        change.Create.Claims,
 		}
-		_, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
+		_, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -1120,20 +1148,31 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 				return wrapSRErr(err)
 			}
 		}
+		// skip overwrite of local update if peer sent stale info
+		if !updatedAt.IsZero() {
+			if sa, _, err := globalIAMSys.getServiceAccount(ctx, change.Update.AccessKey); err == nil && sa.UpdatedAt.After(updatedAt) {
+				return nil
+			}
+		}
 		opts := updateServiceAccountOpts{
 			secretKey:     change.Update.SecretKey,
 			status:        change.Update.Status,
 			sessionPolicy: sp,
 		}
 
-		err = globalIAMSys.UpdateServiceAccount(ctx, change.Update.AccessKey, opts)
+		_, err = globalIAMSys.UpdateServiceAccount(ctx, change.Update.AccessKey, opts)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
 	case change.Delete != nil:
-		err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true)
-		if err != nil {
+		// skip overwrite of local update if peer sent stale info
+		if !updatedAt.IsZero() {
+			if sa, _, err := globalIAMSys.getServiceAccount(ctx, change.Delete.AccessKey); err == nil && sa.UpdatedAt.After(updatedAt) {
+				return nil
+			}
+		}
+		if err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true); err != nil {
 			return wrapSRErr(err)
 		}
 
@@ -1143,11 +1182,19 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 }
 
 // PeerPolicyMappingHandler - copies policy mapping to local.
-func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping *madmin.SRPolicyMapping) error {
+func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping *madmin.SRPolicyMapping, updatedAt time.Time) error {
 	if mapping == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
-	err := globalIAMSys.PolicyDBSet(ctx, mapping.UserOrGroup, mapping.Policy, mapping.IsGroup)
+	// skip overwrite of local update if peer sent stale info
+	if !updatedAt.IsZero() {
+		mp, ok := globalIAMSys.store.GetMappedPolicy(mapping.Policy, mapping.IsGroup)
+		if ok && mp.UpdatedAt.After(updatedAt) {
+			return nil
+		}
+	}
+
+	_, err := globalIAMSys.PolicyDBSet(ctx, mapping.UserOrGroup, mapping.Policy, mapping.IsGroup)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1155,10 +1202,19 @@ func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mappi
 }
 
 // PeerSTSAccHandler - replicates STS credential locally.
-func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *madmin.SRSTSCredential) error {
+func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *madmin.SRSTSCredential, updatedAt time.Time) error {
 	if stsCred == nil {
 		return errSRInvalidRequest(errInvalidArgument)
 	}
+	// skip overwrite of local update if peer sent stale info
+	if !updatedAt.IsZero() {
+		if u, err := globalIAMSys.GetUserInfo(ctx, stsCred.AccessKey); err == nil {
+			ok, _, _ := globalIAMSys.IsTempUser(stsCred.AccessKey)
+			if ok && u.UpdatedAt.After(updatedAt) {
+				return nil
+			}
+		}
+	}
 
 	// Verify the session token of the stsCred
 	claims, err := auth.ExtractClaims(stsCred.SessionToken, globalActiveCred.SecretKey)
@@ -1195,7 +1251,7 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
 	}
 
 	// Set these credentials to IAM.
-	if err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, stsCred.ParentPolicyMapping); err != nil {
+	if _, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, stsCred.ParentPolicyMapping); err != nil {
 		return fmt.Errorf("unable to save STS credential and/or parent policy mapping: %w", err)
 	}
 
@@ -1228,13 +1284,19 @@ func (c *SiteReplicationSys) BucketMetaHook(ctx context.Context, item madmin.SRB
 }
 
 // PeerBucketVersioningHandler - updates versioning config to local cluster.
-func (c *SiteReplicationSys) PeerBucketVersioningHandler(ctx context.Context, bucket string, versioning *string) error {
+func (c *SiteReplicationSys) PeerBucketVersioningHandler(ctx context.Context, bucket string, versioning *string, updatedAt time.Time) error {
 	if versioning != nil {
+		// skip overwrite if local update is newer than peer update.
+		if !updatedAt.IsZero() {
+			if _, updateTm, err := globalBucketMetadataSys.GetVersioningConfig(bucket); err == nil && updateTm.After(updatedAt) {
+				return nil
+			}
+		}
 		configData, err := base64.StdEncoding.DecodeString(*versioning)
 		if err != nil {
 			return wrapSRErr(err)
 		}
-		err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
+		_, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -1245,14 +1307,21 @@ func (c *SiteReplicationSys) PeerBucketVersioningHandler(ctx context.Context, bu
 }
 
 // PeerBucketPolicyHandler - copies/deletes policy to local cluster.
-func (c *SiteReplicationSys) PeerBucketPolicyHandler(ctx context.Context, bucket string, policy *policy.Policy) error {
+func (c *SiteReplicationSys) PeerBucketPolicyHandler(ctx context.Context, bucket string, policy *policy.Policy, updatedAt time.Time) error {
+	// skip overwrite if local update is newer than peer update.
+	if !updatedAt.IsZero() {
+		if _, updateTm, err := globalBucketMetadataSys.GetPolicyConfig(bucket); err == nil && updateTm.After(updatedAt) {
+			return nil
+		}
+	}
+
 	if policy != nil {
 		configData, err := json.Marshal(policy)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+		_, err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -1260,7 +1329,7 @@ func (c *SiteReplicationSys) PeerBucketPolicyHandler(ctx context.Context, bucket
 	}
 
 	// Delete the bucket policy
-	err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
+	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1269,13 +1338,20 @@ func (c *SiteReplicationSys) PeerBucketPolicyHandler(ctx context.Context, bucket
 }
 
 // PeerBucketTaggingHandler - copies/deletes tags to local cluster.
-func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucket string, tags *string) error {
+func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucket string, tags *string, updatedAt time.Time) error {
+	// skip overwrite if local update is newer than peer update.
+	if !updatedAt.IsZero() {
+		if _, updateTm, err := globalBucketMetadataSys.GetTaggingConfig(bucket); err == nil && updateTm.After(updatedAt) {
+			return nil
+		}
+	}
+
 	if tags != nil {
 		configData, err := base64.StdEncoding.DecodeString(*tags)
 		if err != nil {
 			return wrapSRErr(err)
 		}
-		err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
+		_, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -1283,7 +1359,7 @@ func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucke
 	}
 
 	// Delete the tags
-	err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
+	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1292,13 +1368,20 @@ func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucke
 }
 
 // PeerBucketObjectLockConfigHandler - sets object lock on local bucket.
-func (c *SiteReplicationSys) PeerBucketObjectLockConfigHandler(ctx context.Context, bucket string, objectLockData *string) error {
+func (c *SiteReplicationSys) PeerBucketObjectLockConfigHandler(ctx context.Context, bucket string, objectLockData *string, updatedAt time.Time) error {
 	if objectLockData != nil {
+		// skip overwrite if local update is newer than peer update.
+		if !updatedAt.IsZero() {
+			if _, updateTm, err := globalBucketMetadataSys.GetObjectLockConfig(bucket); err == nil && updateTm.After(updatedAt) {
+				return nil
+			}
+		}
+
 		configData, err := base64.StdEncoding.DecodeString(*objectLockData)
 		if err != nil {
 			return wrapSRErr(err)
 		}
-		err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
+		_, err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -1309,13 +1392,20 @@ func (c *SiteReplicationSys) PeerBucketObjectLockConfigHandler(ctx context.Conte
 }
 
 // PeerBucketSSEConfigHandler - copies/deletes SSE config to local cluster.
-func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, bucket string, sseConfig *string) error {
+func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, bucket string, sseConfig *string, updatedAt time.Time) error {
+	// skip overwrite if local update is newer than peer update.
+	if !updatedAt.IsZero() {
+		if _, updateTm, err := globalBucketMetadataSys.GetSSEConfig(bucket); err == nil && updateTm.After(updatedAt) {
+			return nil
+		}
+	}
+
 	if sseConfig != nil {
 		configData, err := base64.StdEncoding.DecodeString(*sseConfig)
 		if err != nil {
 			return wrapSRErr(err)
 		}
-		err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
+		_, err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -1323,7 +1413,7 @@ func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, buc
 	}
 
 	// Delete sse config
-	err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
+	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1331,14 +1421,21 @@ func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, buc
 }
 
 // PeerBucketQuotaConfigHandler - copies/deletes policy to local cluster.
-func (c *SiteReplicationSys) PeerBucketQuotaConfigHandler(ctx context.Context, bucket string, quota *madmin.BucketQuota) error {
+func (c *SiteReplicationSys) PeerBucketQuotaConfigHandler(ctx context.Context, bucket string, quota *madmin.BucketQuota, updatedAt time.Time) error {
+	// skip overwrite if local update is newer than peer update.
+	if !updatedAt.IsZero() {
+		if _, updateTm, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err == nil && updateTm.After(updatedAt) {
+			return nil
+		}
+	}
+
 	if quota != nil {
 		quotaData, err := json.Marshal(quota)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, quotaData); err != nil {
+		if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, quotaData); err != nil {
 			return wrapSRErr(err)
 		}
 
@@ -1346,7 +1443,7 @@ func (c *SiteReplicationSys) PeerBucketQuotaConfigHandler(ctx context.Context, b
 	}
 
 	// Delete the bucket policy
-	err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, nil)
+	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, nil)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1385,11 +1482,11 @@ func (c *SiteReplicationSys) getAdminClientWithEndpoint(ctx context.Context, dep
 }
 
 func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) {
-	creds, ok := globalIAMSys.store.GetUser(c.state.ServiceAccountAccessKey)
+	u, ok := globalIAMSys.store.GetUser(c.state.ServiceAccountAccessKey)
 	if !ok {
 		return nil, errors.New("site replication service account not found")
 	}
-	return &creds, nil
+	return &u.Credentials, nil
 }
 
 // listBuckets returns a consistent common view of latest unique buckets across
@@ -1437,7 +1534,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 		}
 
 		// Replicate bucket policy if present.
-		policy, err := globalPolicySys.Get(bucket)
+		policy, tm, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
 		found := true
 		if _, ok := err.(BucketPolicyNotFound); ok {
 			found = false
@@ -1450,9 +1547,10 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				return wrapSRErr(err)
 			}
 			err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypePolicy,
-				Bucket: bucket,
-				Policy: policyJSON,
+				Type:      madmin.SRBucketMetaTypePolicy,
+				Bucket:    bucket,
+				Policy:    policyJSON,
+				UpdatedAt: tm,
 			})
 			if err != nil {
 				return errSRBucketMetaError(err)
@@ -1460,7 +1558,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 		}
 
 		// Replicate bucket tags if present.
-		tags, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
+		tags, tm, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
 		found = true
 		if _, ok := err.(BucketTaggingNotFound); ok {
 			found = false
@@ -1474,9 +1572,10 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 			}
 			tagCfgStr := base64.StdEncoding.EncodeToString(tagCfg)
 			err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypeTags,
-				Bucket: bucket,
-				Tags:   &tagCfgStr,
+				Type:      madmin.SRBucketMetaTypeTags,
+				Bucket:    bucket,
+				Tags:      &tagCfgStr,
+				UpdatedAt: tm,
 			})
 			if err != nil {
 				return errSRBucketMetaError(err)
@@ -1484,7 +1583,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 		}
 
 		// Replicate object-lock config if present.
-		objLockCfg, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
+		objLockCfg, tm, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
 		found = true
 		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
 			found = false
@@ -1498,9 +1597,10 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 			}
 			objLockStr := base64.StdEncoding.EncodeToString(objLockCfgData)
 			err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypeObjectLockConfig,
-				Bucket: bucket,
-				Tags:   &objLockStr,
+				Type:      madmin.SRBucketMetaTypeObjectLockConfig,
+				Bucket:    bucket,
+				Tags:      &objLockStr,
+				UpdatedAt: tm,
 			})
 			if err != nil {
 				return errSRBucketMetaError(err)
@@ -1508,7 +1608,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 		}
 
 		// Replicate existing bucket bucket encryption settings
-		sseConfig, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
+		sseConfig, tm, err := globalBucketMetadataSys.GetSSEConfig(bucket)
 		found = true
 		if _, ok := err.(BucketSSEConfigNotFound); ok {
 			found = false
@@ -1525,13 +1625,14 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				Type:      madmin.SRBucketMetaTypeSSEConfig,
 				Bucket:    bucket,
 				SSEConfig: &sseConfigStr,
+				UpdatedAt: tm,
 			})
 			if err != nil {
 				return errSRBucketMetaError(err)
 			}
 		}
 
-		quotaConfig, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
+		quotaConfig, tm, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
 		found = true
 		if _, ok := err.(BucketQuotaConfigNotFound); ok {
 			found = false
@@ -1544,9 +1645,10 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				return wrapSRErr(err)
 			}
 			err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypeQuotaConfig,
-				Bucket: bucket,
-				Quota:  quotaConfigJSON,
+				Type:      madmin.SRBucketMetaTypeQuotaConfig,
+				Bucket:    bucket,
+				Quota:     quotaConfigJSON,
+				UpdatedAt: tm,
 			})
 			if err != nil {
 				return errSRBucketMetaError(err)
@@ -1560,20 +1662,21 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 	// Policies should be synced first.
 	{
 		// Replicate IAM policies on local to all peers.
-		allPolicies, err := globalIAMSys.ListPolicies(ctx, "")
+		allPolicyDocs, err := globalIAMSys.ListPolicyDocs(ctx, "")
 		if err != nil {
 			return errSRBackendIssue(err)
 		}
 
-		for pname, policy := range allPolicies {
-			policyJSON, err := json.Marshal(policy)
+		for pname, pdoc := range allPolicyDocs {
+			policyJSON, err := json.Marshal(pdoc.Policy)
 			if err != nil {
 				return wrapSRErr(err)
 			}
 			err = c.IAMChangeHook(ctx, madmin.SRIAMItem{
-				Type:   madmin.SRIAMItemPolicy,
-				Name:   pname,
-				Policy: policyJSON,
+				Type:      madmin.SRIAMItemPolicy,
+				Name:      pname,
+				Policy:    policyJSON,
+				UpdatedAt: pdoc.UpdateDate,
 			})
 			if err != nil {
 				return errSRIAMError(err)
@@ -1584,7 +1687,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 	// Next should be userAccounts those are local users, OIDC and LDAP will not
 	// may not have any local users.
 	{
-		userAccounts := make(map[string]auth.Credentials)
+		userAccounts := make(map[string]UserIdentity)
 		globalIAMSys.store.rlock()
 		err := globalIAMSys.store.loadUsers(ctx, regUser, userAccounts)
 		globalIAMSys.store.runlock()
@@ -1596,13 +1699,14 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 			if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
 				Type: madmin.SRIAMItemIAMUser,
 				IAMUser: &madmin.SRIAMUser{
-					AccessKey:   acc.AccessKey,
+					AccessKey:   acc.Credentials.AccessKey,
 					IsDeleteReq: false,
 					UserReq: &madmin.AddOrUpdateUserReq{
-						SecretKey: acc.SecretKey,
-						Status:    madmin.AccountStatus(acc.Status),
+						SecretKey: acc.Credentials.SecretKey,
+						Status:    madmin.AccountStatus(acc.Credentials.Status),
 					},
 				},
+				UpdatedAt: acc.UpdatedAt,
 			}); err != nil {
 				return errSRIAMError(err)
 			}
@@ -1632,6 +1736,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 						IsRemove: false,
 					},
 				},
+				UpdatedAt: group.UpdatedAt,
 			}); err != nil {
 				return errSRIAMError(err)
 			}
@@ -1641,7 +1746,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 	// Service accounts are the static accounts that should be synced with
 	// valid claims.
 	{
-		serviceAccounts := make(map[string]auth.Credentials)
+		serviceAccounts := make(map[string]UserIdentity)
 		globalIAMSys.store.rlock()
 		err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts)
 		globalIAMSys.store.runlock()
@@ -1656,12 +1761,12 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				continue
 			}
 
-			claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.AccessKey)
+			claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey)
 			if err != nil {
 				return errSRBackendIssue(err)
 			}
 
-			_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.AccessKey)
+			_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey)
 			if err != nil {
 				return errSRBackendIssue(err)
 			}
@@ -1678,15 +1783,16 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				Type: madmin.SRIAMItemSvcAcc,
 				SvcAccChange: &madmin.SRSvcAccChange{
 					Create: &madmin.SRSvcAccCreate{
-						Parent:        acc.ParentUser,
+						Parent:        acc.Credentials.ParentUser,
 						AccessKey:     user,
-						SecretKey:     acc.SecretKey,
-						Groups:        acc.Groups,
+						SecretKey:     acc.Credentials.SecretKey,
+						Groups:        acc.Credentials.Groups,
 						Claims:        claims,
 						SessionPolicy: json.RawMessage(policyJSON),
-						Status:        acc.Status,
+						Status:        acc.Credentials.Status,
 					},
 				},
+				UpdatedAt: acc.UpdatedAt,
 			})
 			if err != nil {
 				return errSRIAMError(err)
@@ -1715,6 +1821,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 					IsGroup:     false,
 					Policy:      mp.Policies,
 				},
+				UpdatedAt: mp.UpdatedAt,
 			})
 			if err != nil {
 				return errSRIAMError(err)
@@ -1733,6 +1840,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 					IsGroup:     true,
 					Policy:      mp.Policies,
 				},
+				UpdatedAt: mp.UpdatedAt,
 			})
 			if err != nil {
 				return errSRIAMError(err)
@@ -1761,6 +1869,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 					IsGroup:     false,
 					Policy:      mp.Policies,
 				},
+				UpdatedAt: mp.UpdatedAt,
 			})
 			if err != nil {
 				return errSRIAMError(err)
@@ -1779,6 +1888,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 					IsGroup:     true,
 					Policy:      mp.Policies,
 				},
+				UpdatedAt: mp.UpdatedAt,
 			})
 			if err != nil {
 				return errSRIAMError(err)
@@ -2069,11 +2179,11 @@ func (c *SiteReplicationSys) RemoveRemoteTargetsForEndpoint(ctx context.Context,
 			if err != nil {
 				return err
 			}
-			if err = globalBucketMetadataSys.Update(ctx, b.Name, bucketReplicationConfig, configData); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, b.Name, bucketReplicationConfig, configData); err != nil {
 				return err
 			}
 		} else {
-			if err := globalBucketMetadataSys.Update(ctx, b.Name, bucketReplicationConfig, nil); err != nil {
+			if _, err := globalBucketMetadataSys.Update(ctx, b.Name, bucketReplicationConfig, nil); err != nil {
 				return err
 			}
 		}
@@ -3198,8 +3308,8 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 					return info, errSRBackendIssue(err)
 				}
 				for _, tempAcct := range tempAccts {
-					info.UserInfoMap[tempAcct.AccessKey] = madmin.UserInfo{
-						Status: madmin.AccountStatus(tempAcct.Status),
+					info.UserInfoMap[tempAcct.Credentials.AccessKey] = madmin.UserInfo{
+						Status: madmin.AccountStatus(tempAcct.Credentials.Status),
 					}
 				}
 			}
@@ -3440,14 +3550,20 @@ type srStatusInfo struct {
 }
 
 func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer) error {
-	info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
-		Buckets: true,
-	})
+	buckets, err := c.listBuckets(ctx)
 	if err != nil {
 		return err
 	}
-
-	for bucket := range info.BucketStats {
+	for _, bi := range buckets {
+		bucket := bi.Name
+		info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
+			Entity:      madmin.SRBucketEntity,
+			EntityValue: bucket,
+		})
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
 		c.healCreateMissingBucket(ctx, objAPI, bucket, info)
 		c.healVersioningMetadata(ctx, objAPI, bucket, info)
 		c.healOLockConfigMetadata(ctx, objAPI, bucket, info)
@@ -3509,7 +3625,7 @@ func (c *SiteReplicationSys) healTagMetadata(ctx context.Context, objAPI ObjectL
 			continue
 		}
 		if dID == globalDeploymentID {
-			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, latestTaggingConfigBytes); err != nil {
+			if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, latestTaggingConfigBytes); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing tagging metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
@@ -3573,7 +3689,7 @@ func (c *SiteReplicationSys) healBucketPolicies(ctx context.Context, objAPI Obje
 			continue
 		}
 		if dID == globalDeploymentID {
-			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, latestIAMPolicy); err != nil {
+			if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, latestIAMPolicy); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing bucket policy metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
@@ -3585,9 +3701,10 @@ func (c *SiteReplicationSys) healBucketPolicies(ctx context.Context, objAPI Obje
 		}
 		peerName := info.Sites[dID].Name
 		if err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{
-			Type:   madmin.SRBucketMetaTypePolicy,
-			Bucket: bucket,
-			Policy: latestIAMPolicy,
+			Type:      madmin.SRBucketMetaTypePolicy,
+			Bucket:    bucket,
+			Policy:    latestIAMPolicy,
+			UpdatedAt: lastUpdate,
 		}); err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
 				fmt.Errorf("Error healing bucket policy metadata for peer %s from peer %s : %w",
@@ -3647,7 +3764,7 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 			continue
 		}
 		if dID == globalDeploymentID {
-			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, latestQuotaConfigBytes); err != nil {
+			if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, latestQuotaConfigBytes); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing quota metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
@@ -3660,9 +3777,10 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 		peerName := info.Sites[dID].Name
 
 		if err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{
-			Type:   madmin.SRBucketMetaTypeQuotaConfig,
-			Bucket: bucket,
-			Quota:  latestQuotaConfigBytes,
+			Type:      madmin.SRBucketMetaTypeQuotaConfig,
+			Bucket:    bucket,
+			Quota:     latestQuotaConfigBytes,
+			UpdatedAt: lastUpdate,
 		}); err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
 				fmt.Errorf("Error healing quota config metadata for peer %s from peer %s : %w",
@@ -3721,7 +3839,7 @@ func (c *SiteReplicationSys) healVersioningMetadata(ctx context.Context, objAPI
 			continue
 		}
 		if dID == globalDeploymentID {
-			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, latestVersioningConfigBytes); err != nil {
+			if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, latestVersioningConfigBytes); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing versioning metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
@@ -3736,6 +3854,7 @@ func (c *SiteReplicationSys) healVersioningMetadata(ctx context.Context, objAPI
 			Type:       madmin.SRBucketMetaTypeVersionConfig,
 			Bucket:     bucket,
 			Versioning: latestVersioningConfig,
+			UpdatedAt:  lastUpdate,
 		})
 		if err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
@@ -3795,7 +3914,7 @@ func (c *SiteReplicationSys) healSSEMetadata(ctx context.Context, objAPI ObjectL
 			continue
 		}
 		if dID == globalDeploymentID {
-			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, latestSSEConfigBytes); err != nil {
+			if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, latestSSEConfigBytes); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing sse metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
@@ -3810,6 +3929,7 @@ func (c *SiteReplicationSys) healSSEMetadata(ctx context.Context, objAPI ObjectL
 			Type:      madmin.SRBucketMetaTypeSSEConfig,
 			Bucket:    bucket,
 			SSEConfig: latestSSEConfig,
+			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
@@ -3869,7 +3989,7 @@ func (c *SiteReplicationSys) healOLockConfigMetadata(ctx context.Context, objAPI
 			continue
 		}
 		if dID == globalDeploymentID {
-			if err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, latestObjLockConfigBytes); err != nil {
+			if _, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, latestObjLockConfigBytes); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing objectlock config metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
@@ -3881,9 +4001,10 @@ func (c *SiteReplicationSys) healOLockConfigMetadata(ctx context.Context, objAPI
 		}
 		peerName := info.Sites[dID].Name
 		err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{
-			Type:   madmin.SRBucketMetaTypeObjectLockConfig,
-			Bucket: bucket,
-			Tags:   latestObjLockConfig,
+			Type:      madmin.SRBucketMetaTypeObjectLockConfig,
+			Bucket:    bucket,
+			Tags:      latestObjLockConfig,
+			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
@@ -4106,9 +4227,10 @@ func (c *SiteReplicationSys) healPolicies(ctx context.Context, objAPI ObjectLaye
 		}
 		peerName := info.Sites[dID].Name
 		err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
-			Type:   madmin.SRIAMItemPolicy,
-			Name:   policy,
-			Policy: latestPolicyStat.policy.Policy,
+			Type:      madmin.SRIAMItemPolicy,
+			Name:      policy,
+			Policy:    latestPolicyStat.policy.Policy,
+			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Error healing IAM policy %s from peer site %s -> site %s : %w", policy, latestPeerName, peerName, err))
@@ -4168,6 +4290,7 @@ func (c *SiteReplicationSys) healUserPolicies(ctx context.Context, objAPI Object
 				IsGroup:     false,
 				Policy:      latestUserStat.userPolicy.Policy,
 			},
+			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Error healing IAM user policy mapping for %s from peer site %s -> site %s : %w", user, latestPeerName, peerName, err))
@@ -4229,6 +4352,7 @@ func (c *SiteReplicationSys) healGroupPolicies(ctx context.Context, objAPI Objec
 				IsGroup:     true,
 				Policy:      latestGroupStat.groupPolicy.Policy,
 			},
+			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Error healing IAM group policy mapping for %s from peer site %s -> site %s : %w", group, latestPeerName, peerName, err))
@@ -4283,11 +4407,11 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 
 		peerName := info.Sites[dID].Name
 
-		creds, ok := globalIAMSys.GetUser(ctx, user)
+		u, ok := globalIAMSys.GetUser(ctx, user)
 		if !ok {
 			continue
 		}
-
+		creds := u.Credentials
 		// heal only the user accounts that are local users
 		if creds.IsServiceAccount() {
 			claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, creds.AccessKey)
@@ -4324,6 +4448,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 						Status:        creds.Status,
 					},
 				},
+				UpdatedAt: lastUpdate,
 			}); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing service account %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
 			}
@@ -4345,6 +4470,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 					ParentUser:          creds.ParentUser,
 					ParentPolicyMapping: u.PolicyName,
 				},
+				UpdatedAt: lastUpdate,
 			}); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Error healing temporary credentials %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
 			}
@@ -4360,6 +4486,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 					Status:    latestUserStat.userInfo.Status,
 				},
 			},
+			UpdatedAt: lastUpdate,
 		}); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Error healing user %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
 		}
@@ -4424,6 +4551,7 @@ func (c *SiteReplicationSys) healGroups(ctx context.Context, objAPI ObjectLayer,
 					IsRemove: false,
 				},
 			},
+			UpdatedAt: lastUpdate,
 		}); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Error healing group %s from peer site %s -> site %s : %w", group, latestPeerName, peerName, err))
 		}

cmd/storage-datatypes.go

@@ -47,8 +47,8 @@ type DiskInfo struct {
 // the number of calls of each API and the moving average of
 // the duration of each API.
 type DiskMetrics struct {
-	APILatencies map[string]uint64 `json:"apiLatencies,omitempty"`
-	APICalls     map[string]uint64 `json:"apiCalls,omitempty"`
+	LastMinute map[string]AccElem `json:"apiLatencies,omitempty"`
+	APICalls   map[string]uint64  `json:"apiCalls,omitempty"`
 }
 
 // VolsInfo is a collection of volume(bucket) information
@@ -176,6 +176,10 @@ type FileInfo struct {
 	// File mode bits.
 	Mode uint32 `msg:"m"`
 
+	// WrittenByVersion is the unix time stamp of the MinIO
+	// version that created this version of the object.
+	WrittenByVersion uint64 `msg:"wv"`
+
 	// File metadata
 	Metadata map[string]string `msg:"meta"`
 

cmd/storage-datatypes_gen.go

@@ -291,35 +291,35 @@ func (z *DiskMetrics) DecodeMsg(dc *msgp.Reader) (err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
-		case "APILatencies":
+		case "LastMinute":
 			var zb0002 uint32
 			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
-				err = msgp.WrapError(err, "APILatencies")
+				err = msgp.WrapError(err, "LastMinute")
 				return
 			}
-			if z.APILatencies == nil {
-				z.APILatencies = make(map[string]uint64, zb0002)
-			} else if len(z.APILatencies) > 0 {
-				for key := range z.APILatencies {
-					delete(z.APILatencies, key)
+			if z.LastMinute == nil {
+				z.LastMinute = make(map[string]AccElem, zb0002)
+			} else if len(z.LastMinute) > 0 {
+				for key := range z.LastMinute {
+					delete(z.LastMinute, key)
 				}
 			}
 			for zb0002 > 0 {
 				zb0002--
 				var za0001 string
-				var za0002 uint64
+				var za0002 AccElem
 				za0001, err = dc.ReadString()
 				if err != nil {
-					err = msgp.WrapError(err, "APILatencies")
+					err = msgp.WrapError(err, "LastMinute")
 					return
 				}
-				za0002, err = dc.ReadUint64()
+				err = za0002.DecodeMsg(dc)
 				if err != nil {
-					err = msgp.WrapError(err, "APILatencies", za0001)
+					err = msgp.WrapError(err, "LastMinute", za0001)
 					return
 				}
-				z.APILatencies[za0001] = za0002
+				z.LastMinute[za0001] = za0002
 			}
 		case "APICalls":
 			var zb0003 uint32
@@ -365,25 +365,25 @@ func (z *DiskMetrics) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *DiskMetrics) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 2
-	// write "APILatencies"
-	err = en.Append(0x82, 0xac, 0x41, 0x50, 0x49, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73)
+	// write "LastMinute"
+	err = en.Append(0x82, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65)
 	if err != nil {
 		return
 	}
-	err = en.WriteMapHeader(uint32(len(z.APILatencies)))
+	err = en.WriteMapHeader(uint32(len(z.LastMinute)))
 	if err != nil {
-		err = msgp.WrapError(err, "APILatencies")
+		err = msgp.WrapError(err, "LastMinute")
 		return
 	}
-	for za0001, za0002 := range z.APILatencies {
+	for za0001, za0002 := range z.LastMinute {
 		err = en.WriteString(za0001)
 		if err != nil {
-			err = msgp.WrapError(err, "APILatencies")
+			err = msgp.WrapError(err, "LastMinute")
 			return
 		}
-		err = en.WriteUint64(za0002)
+		err = za0002.EncodeMsg(en)
 		if err != nil {
-			err = msgp.WrapError(err, "APILatencies", za0001)
+			err = msgp.WrapError(err, "LastMinute", za0001)
 			return
 		}
 	}
@@ -416,12 +416,16 @@ func (z *DiskMetrics) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *DiskMetrics) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 2
-	// string "APILatencies"
-	o = append(o, 0x82, 0xac, 0x41, 0x50, 0x49, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73)
-	o = msgp.AppendMapHeader(o, uint32(len(z.APILatencies)))
-	for za0001, za0002 := range z.APILatencies {
+	// string "LastMinute"
+	o = append(o, 0x82, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.LastMinute)))
+	for za0001, za0002 := range z.LastMinute {
 		o = msgp.AppendString(o, za0001)
-		o = msgp.AppendUint64(o, za0002)
+		o, err = za0002.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "LastMinute", za0001)
+			return
+		}
 	}
 	// string "APICalls"
 	o = append(o, 0xa8, 0x41, 0x50, 0x49, 0x43, 0x61, 0x6c, 0x6c, 0x73)
@@ -451,35 +455,35 @@ func (z *DiskMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
-		case "APILatencies":
+		case "LastMinute":
 			var zb0002 uint32
 			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "APILatencies")
+				err = msgp.WrapError(err, "LastMinute")
 				return
 			}
-			if z.APILatencies == nil {
-				z.APILatencies = make(map[string]uint64, zb0002)
-			} else if len(z.APILatencies) > 0 {
-				for key := range z.APILatencies {
-					delete(z.APILatencies, key)
+			if z.LastMinute == nil {
+				z.LastMinute = make(map[string]AccElem, zb0002)
+			} else if len(z.LastMinute) > 0 {
+				for key := range z.LastMinute {
+					delete(z.LastMinute, key)
 				}
 			}
 			for zb0002 > 0 {
 				var za0001 string
-				var za0002 uint64
+				var za0002 AccElem
 				zb0002--
 				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "APILatencies")
+					err = msgp.WrapError(err, "LastMinute")
 					return
 				}
-				za0002, bts, err = msgp.ReadUint64Bytes(bts)
+				bts, err = za0002.UnmarshalMsg(bts)
 				if err != nil {
-					err = msgp.WrapError(err, "APILatencies", za0001)
+					err = msgp.WrapError(err, "LastMinute", za0001)
 					return
 				}
-				z.APILatencies[za0001] = za0002
+				z.LastMinute[za0001] = za0002
 			}
 		case "APICalls":
 			var zb0003 uint32
@@ -525,11 +529,11 @@ func (z *DiskMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *DiskMetrics) Msgsize() (s int) {
-	s = 1 + 13 + msgp.MapHeaderSize
-	if z.APILatencies != nil {
-		for za0001, za0002 := range z.APILatencies {
+	s = 1 + 11 + msgp.MapHeaderSize
+	if z.LastMinute != nil {
+		for za0001, za0002 := range z.LastMinute {
 			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + msgp.Uint64Size
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
 	s += 9 + msgp.MapHeaderSize
@@ -550,8 +554,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 26 {
-		err = msgp.ArrayError{Wanted: 26, Got: zb0001}
+	if zb0001 != 27 {
+		err = msgp.ArrayError{Wanted: 27, Got: zb0001}
 		return
 	}
 	z.Volume, err = dc.ReadString()
@@ -629,6 +633,11 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "Mode")
 		return
 	}
+	z.WrittenByVersion, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "WrittenByVersion")
+		return
+	}
 	var zb0002 uint32
 	zb0002, err = dc.ReadMapHeader()
 	if err != nil {
@@ -726,8 +735,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 26
-	err = en.Append(0xdc, 0x0, 0x1a)
+	// array header, size 27
+	err = en.Append(0xdc, 0x0, 0x1b)
 	if err != nil {
 		return
 	}
@@ -806,6 +815,11 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Mode")
 		return
 	}
+	err = en.WriteUint64(z.WrittenByVersion)
+	if err != nil {
+		err = msgp.WrapError(err, "WrittenByVersion")
+		return
+	}
 	err = en.WriteMapHeader(uint32(len(z.Metadata)))
 	if err != nil {
 		err = msgp.WrapError(err, "Metadata")
@@ -886,8 +900,8 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 26
-	o = append(o, 0xdc, 0x0, 0x1a)
+	// array header, size 27
+	o = append(o, 0xdc, 0x0, 0x1b)
 	o = msgp.AppendString(o, z.Volume)
 	o = msgp.AppendString(o, z.Name)
 	o = msgp.AppendString(o, z.VersionID)
@@ -903,6 +917,7 @@ func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendTime(o, z.ModTime)
 	o = msgp.AppendInt64(o, z.Size)
 	o = msgp.AppendUint32(o, z.Mode)
+	o = msgp.AppendUint64(o, z.WrittenByVersion)
 	o = msgp.AppendMapHeader(o, uint32(len(z.Metadata)))
 	for za0001, za0002 := range z.Metadata {
 		o = msgp.AppendString(o, za0001)
@@ -944,8 +959,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 26 {
-		err = msgp.ArrayError{Wanted: 26, Got: zb0001}
+	if zb0001 != 27 {
+		err = msgp.ArrayError{Wanted: 27, Got: zb0001}
 		return
 	}
 	z.Volume, bts, err = msgp.ReadStringBytes(bts)
@@ -1023,6 +1038,11 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "Mode")
 		return
 	}
+	z.WrittenByVersion, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "WrittenByVersion")
+		return
+	}
 	var zb0002 uint32
 	zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
@@ -1121,7 +1141,7 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *FileInfo) Msgsize() (s int) {
-	s = 3 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.TransitionStatus) + msgp.StringPrefixSize + len(z.TransitionedObjName) + msgp.StringPrefixSize + len(z.TransitionTier) + msgp.StringPrefixSize + len(z.TransitionVersionID) + msgp.BoolSize + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.MapHeaderSize
+	s = 3 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.TransitionStatus) + msgp.StringPrefixSize + len(z.TransitionedObjName) + msgp.StringPrefixSize + len(z.TransitionTier) + msgp.StringPrefixSize + len(z.TransitionVersionID) + msgp.BoolSize + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.Uint64Size + msgp.MapHeaderSize
 	if z.Metadata != nil {
 		for za0001, za0002 := range z.Metadata {
 			_ = za0002

cmd/storage-rest-common.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v45" // Added ReadXL API
+	storageRESTVersion       = "v46" // Added MinIO version to FileInfo
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )

cmd/sts-handlers.go

@@ -275,7 +275,8 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	cred.ParentUser = user.AccessKey
 
 	// Set the newly generated credentials.
-	if err = globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, ""); err != nil {
+	updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, "")
+	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
 		return
 	}
@@ -290,6 +291,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 				SessionToken: cred.SessionToken,
 				ParentUser:   cred.ParentUser,
 			},
+			UpdatedAt: updatedAt,
 		}); err != nil {
 			logger.LogIf(ctx, err)
 		}
@@ -469,7 +471,8 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Set the newly generated credentials.
-	if err = globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, policyName); err != nil {
+	updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, policyName)
+	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
 		return
 	}
@@ -484,6 +487,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 			ParentUser:          cred.ParentUser,
 			ParentPolicyMapping: policyName,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		logger.LogIf(ctx, err)
 	}
@@ -639,7 +643,8 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	// Set the newly generated credentials, policyName is empty on purpose
 	// LDAP policies are applied automatically using their ldapUser, ldapGroups
 	// mapping.
-	if err = globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, ""); err != nil {
+	updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, "")
+	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
 		return
 	}
@@ -653,6 +658,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 			SessionToken: cred.SessionToken,
 			ParentUser:   cred.ParentUser,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		logger.LogIf(ctx, err)
 	}
@@ -797,7 +803,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 
 	tmpCredentials.ParentUser = parentUser
 	policyName := certificate.Subject.CommonName
-	err = globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, policyName)
+	updatedAt, err := globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, policyName)
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
 		return
@@ -813,6 +819,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 			ParentUser:          tmpCredentials.ParentUser,
 			ParentPolicyMapping: policyName,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		logger.LogIf(ctx, err)
 	}
@@ -918,7 +925,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 	}
 
 	tmpCredentials.ParentUser = parentUser
-	err = globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, "")
+	updatedAt, err := globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, "")
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
 		return
@@ -933,6 +940,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 			SessionToken: tmpCredentials.SessionToken,
 			ParentUser:   tmpCredentials.ParentUser,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/update.go

@@ -95,7 +95,7 @@ func getModTime(path string) (t time.Time, err error) {
 
 	// Version is minio non-standard, we will use minio binary's
 	// ModTime as release time.
-	fi, err := os.Stat(absPath)
+	fi, err := Stat(absPath)
 	if err != nil {
 		return t, fmt.Errorf("Unable to get ModTime of %s. %w", absPath, err)
 	}

cmd/utils.go

@@ -54,6 +54,7 @@ import (
 	"github.com/minio/minio/internal/fips"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
+	ioutilx "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/audit"
 	"github.com/minio/minio/internal/rest"
@@ -121,15 +122,6 @@ func path2BucketObject(s string) (bucket, prefix string) {
 	return path2BucketObjectWithBasePath("", s)
 }
 
-func getWriteQuorum(drive int) int {
-	parity := getDefaultParityBlocks(drive)
-	quorum := drive - parity
-	if quorum == parity {
-		quorum++
-	}
-	return quorum
-}
-
 // CloneMSS is an exposed function of cloneMSS for gateway usage.
 var CloneMSS = cloneMSS
 
@@ -314,7 +306,7 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 			return nil, err
 		}
 		fn := filepath.Join(dirPath, "cpu.out")
-		f, err := os.Create(fn)
+		f, err := Create(fn)
 		if err != nil {
 			return nil, err
 		}
@@ -328,8 +320,8 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 			if err != nil {
 				return nil, err
 			}
-			defer os.RemoveAll(dirPath)
-			return ioutil.ReadFile(fn)
+			defer RemoveAll(dirPath)
+			return ioutilx.ReadFile(fn)
 		}
 	case madmin.ProfilerCPUIO:
 		// at 10k or more goroutines fgprof is likely to become
@@ -344,7 +336,7 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 			return nil, err
 		}
 		fn := filepath.Join(dirPath, "cpuio.out")
-		f, err := os.Create(fn)
+		f, err := Create(fn)
 		if err != nil {
 			return nil, err
 		}
@@ -358,8 +350,8 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 			if err != nil {
 				return nil, err
 			}
-			defer os.RemoveAll(dirPath)
-			return ioutil.ReadFile(fn)
+			defer RemoveAll(dirPath)
+			return ioutilx.ReadFile(fn)
 		}
 	case madmin.ProfilerMEM:
 		runtime.GC()
@@ -409,7 +401,7 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 			return nil, err
 		}
 		fn := filepath.Join(dirPath, "trace.out")
-		f, err := os.Create(fn)
+		f, err := Create(fn)
 		if err != nil {
 			return nil, err
 		}
@@ -424,8 +416,8 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 			if err != nil {
 				return nil, err
 			}
-			defer os.RemoveAll(dirPath)
-			return ioutil.ReadFile(fn)
+			defer RemoveAll(dirPath)
+			return ioutilx.ReadFile(fn)
 		}
 	default:
 		return nil, errors.New("profiler type unknown")
@@ -801,6 +793,8 @@ func likelyUnescapeGeneric(p string, escapeFn func(string) (string, error)) stri
 func updateReqContext(ctx context.Context, objects ...ObjectV) context.Context {
 	req := logger.GetReqInfo(ctx)
 	if req != nil {
+		req.Lock()
+		defer req.Unlock()
 		req.Objects = make([]logger.ObjectVersion, 0, len(objects))
 		for _, ov := range objects {
 			req.Objects = append(req.Objects, logger.ObjectVersion{

cmd/xl-storage-disk-id-check.go

@@ -78,20 +78,28 @@ type xlStorageDiskIDCheck struct {
 	diskID       string
 	storage      *xlStorage
 	health       *diskHealthTracker
+	metricsCache timedValue
 }
 
 func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics {
-	diskMetric := DiskMetrics{
-		APILatencies: make(map[string]uint64),
-		APICalls:     make(map[string]uint64),
-	}
-	for i, v := range p.apiLatencies {
-		diskMetric.APILatencies[storageMetric(i).String()] = v.value()
-	}
-	for i := range p.apiCalls {
-		diskMetric.APICalls[storageMetric(i).String()] = atomic.LoadUint64(&p.apiCalls[i])
-	}
-	return diskMetric
+	p.metricsCache.Once.Do(func() {
+		p.metricsCache.TTL = 100 * time.Millisecond
+		p.metricsCache.Update = func() (interface{}, error) {
+			diskMetric := DiskMetrics{
+				LastMinute: make(map[string]AccElem, len(p.apiLatencies)),
+				APICalls:   make(map[string]uint64, len(p.apiCalls)),
+			}
+			for i, v := range p.apiLatencies {
+				diskMetric.LastMinute[storageMetric(i).String()] = v.total()
+			}
+			for i := range p.apiCalls {
+				diskMetric.APICalls[storageMetric(i).String()] = atomic.LoadUint64(&p.apiCalls[i])
+			}
+			return diskMetric, nil
+		}
+	})
+	m, _ := p.metricsCache.Get()
+	return m.(DiskMetrics)
 }
 
 type lockedLastMinuteLatency struct {
@@ -105,10 +113,18 @@ func (e *lockedLastMinuteLatency) add(value time.Duration) {
 	e.lastMinuteLatency.add(value)
 }
 
-func (e *lockedLastMinuteLatency) value() uint64 {
+// addSize will add a duration and size.
+func (e *lockedLastMinuteLatency) addSize(value time.Duration, sz int64) {
 	e.Lock()
 	defer e.Unlock()
-	return e.lastMinuteLatency.getAvgData().avg()
+	e.lastMinuteLatency.addSize(value, sz)
+}
+
+// total returns the total call count and latency for the last minute.
+func (e *lockedLastMinuteLatency) total() AccElem {
+	e.Lock()
+	defer e.Unlock()
+	return e.lastMinuteLatency.getTotal()
 }
 
 func newXLStorageDiskIDCheck(storage *xlStorage) *xlStorageDiskIDCheck {
@@ -500,17 +516,26 @@ func storageTrace(s storageMetric, startTime time.Time, duration time.Duration,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  "storage." + s.String(),
-		StorageStats: madmin.TraceStorageStats{
-			Duration: duration,
-			Path:     path,
-		},
+		Duration:  duration,
+		Path:      path,
+	}
+}
+
+func scannerTrace(s scannerMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {
+	return madmin.TraceInfo{
+		TraceType: madmin.TraceScanner,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  "scanner." + s.String(),
+		Duration:  duration,
+		Path:      path,
 	}
 }
 
 // Update storage metrics
 func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(err *error) {
 	startTime := time.Now()
-	trace := globalTrace.NumSubscribers() > 0
+	trace := globalTrace.NumSubscribers(madmin.TraceStorage) > 0
 	return func(err *error) {
 		duration := time.Since(startTime)
 
@@ -647,18 +672,9 @@ func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMet
 	atomic.StoreInt64(&p.health.lastStarted, time.Now().UnixNano())
 	ctx = context.WithValue(ctx, healthDiskCtxKey{}, &healthDiskCtxValue{lastSuccess: &p.health.lastSuccess})
 	si := p.updateStorageMetrics(s, paths...)
-	t := time.Now()
 	var once sync.Once
 	return ctx, func(errp *error) {
 		once.Do(func() {
-			if false {
-				var ers string
-				if errp != nil {
-					err := *errp
-					ers = fmt.Sprint(err)
-				}
-				fmt.Println(time.Now().Format(time.RFC3339), "op", s, "took", time.Since(t), "result:", ers, "disk:", p.storage.String(), "path:", strings.Join(paths, "/"))
-			}
 			p.health.tokens <- struct{}{}
 			if errp != nil {
 				err := *errp

cmd/xl-storage-format-v2.go

@@ -178,10 +178,11 @@ type xlMetaV2Object struct {
 // on what Type field carries, it is imperative for the caller
 // to verify which journal type first before accessing rest of the fields.
 type xlMetaV2Version struct {
-	Type         VersionType           `json:"Type" msg:"Type"`
-	ObjectV1     *xlMetaV1Object       `json:"V1Obj,omitempty" msg:"V1Obj,omitempty"`
-	ObjectV2     *xlMetaV2Object       `json:"V2Obj,omitempty" msg:"V2Obj,omitempty"`
-	DeleteMarker *xlMetaV2DeleteMarker `json:"DelObj,omitempty" msg:"DelObj,omitempty"`
+	Type             VersionType           `json:"Type" msg:"Type"`
+	ObjectV1         *xlMetaV1Object       `json:"V1Obj,omitempty" msg:"V1Obj,omitempty"`
+	ObjectV2         *xlMetaV2Object       `json:"V2Obj,omitempty" msg:"V2Obj,omitempty"`
+	DeleteMarker     *xlMetaV2DeleteMarker `json:"DelObj,omitempty" msg:"DelObj,omitempty"`
+	WrittenByVersion uint64                `msg:"v"` // Tracks written by MinIO version
 }
 
 // xlFlags contains flags on the object.
@@ -409,16 +410,22 @@ func (j xlMetaV2Version) getVersionID() [16]byte {
 }
 
 // ToFileInfo returns FileInfo of the underlying type.
-func (j *xlMetaV2Version) ToFileInfo(volume, path string) (FileInfo, error) {
+func (j *xlMetaV2Version) ToFileInfo(volume, path string) (fi FileInfo, err error) {
+	if j == nil {
+		return fi, errFileNotFound
+	}
 	switch j.Type {
 	case ObjectType:
-		return j.ObjectV2.ToFileInfo(volume, path)
+		fi, err = j.ObjectV2.ToFileInfo(volume, path)
 	case DeleteType:
-		return j.DeleteMarker.ToFileInfo(volume, path)
+		fi, err = j.DeleteMarker.ToFileInfo(volume, path)
 	case LegacyType:
-		return j.ObjectV1.ToFileInfo(volume, path)
+		fi, err = j.ObjectV1.ToFileInfo(volume, path)
+	default:
+		return fi, errFileNotFound
 	}
-	return FileInfo{}, errFileNotFound
+	fi.WrittenByVersion = j.WrittenByVersion
+	return fi, err
 }
 
 const (
@@ -1182,6 +1189,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 				ModTime:   fi.ModTime.UnixNano(),
 				MetaSys:   make(map[string][]byte),
 			},
+			WrittenByVersion: globalVersionUnix,
 		}
 		if !ventry.Valid() {
 			return "", errors.New("internal error: invalid version entry generated")
@@ -1424,7 +1432,9 @@ func (x *xlMetaV2) AddVersion(fi FileInfo) error {
 		}
 	}
 
-	ventry := xlMetaV2Version{}
+	ventry := xlMetaV2Version{
+		WrittenByVersion: globalVersionUnix,
+	}
 
 	if fi.Deleted {
 		ventry.Type = DeleteType
@@ -1589,7 +1599,7 @@ func (x *xlMetaV2) AddLegacy(m *xlMetaV1Object) error {
 	}
 	m.VersionID = nullVersionID
 
-	return x.addVersion(xlMetaV2Version{ObjectV1: m, Type: LegacyType})
+	return x.addVersion(xlMetaV2Version{ObjectV1: m, Type: LegacyType, WrittenByVersion: globalVersionUnix})
 }
 
 // ToFileInfo converts xlMetaV2 into a common FileInfo datastructure

cmd/xl-storage-format-v2_gen.go

@@ -1774,6 +1774,12 @@ func (z *xlMetaV2Version) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "v":
+			z.WrittenByVersion, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "WrittenByVersion")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -1788,8 +1794,8 @@ func (z *xlMetaV2Version) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
-	zb0001Len := uint32(4)
-	var zb0001Mask uint8 /* 4 bits */
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
 	if z.ObjectV1 == nil {
 		zb0001Len--
 		zb0001Mask |= 0x2
@@ -1877,6 +1883,16 @@ func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
+	// write "v"
+	err = en.Append(0xa1, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.WrittenByVersion)
+	if err != nil {
+		err = msgp.WrapError(err, "WrittenByVersion")
+		return
+	}
 	return
 }
 
@@ -1884,8 +1900,8 @@ func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// omitempty: check for empty values
-	zb0001Len := uint32(4)
-	var zb0001Mask uint8 /* 4 bits */
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
 	if z.ObjectV1 == nil {
 		zb0001Len--
 		zb0001Mask |= 0x2
@@ -1945,6 +1961,9 @@ func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
+	// string "v"
+	o = append(o, 0xa1, 0x76)
+	o = msgp.AppendUint64(o, z.WrittenByVersion)
 	return
 }
 
@@ -2027,6 +2046,12 @@ func (z *xlMetaV2Version) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "v":
+			z.WrittenByVersion, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "WrittenByVersion")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -2059,6 +2084,7 @@ func (z *xlMetaV2Version) Msgsize() (s int) {
 	} else {
 		s += z.DeleteMarker.Msgsize()
 	}
+	s += 2 + msgp.Uint64Size
 	return
 }
 

cmd/xl-storage-free-version.go

@@ -35,7 +35,7 @@ func (j xlMetaV2Object) InitFreeVersion(fi FileInfo) (xlMetaV2Version, bool) {
 		if err != nil {
 			panic(fmt.Errorf("Invalid Tier Object delete marker versionId %s %v", fi.TierFreeVersionID(), err))
 		}
-		freeEntry := xlMetaV2Version{Type: DeleteType}
+		freeEntry := xlMetaV2Version{Type: DeleteType, WrittenByVersion: globalVersionUnix}
 		freeEntry.DeleteMarker = &xlMetaV2DeleteMarker{
 			VersionID: vID,
 			ModTime:   j.ModTime, // fi.ModTime may be empty

cmd/xl-storage.go

@@ -32,7 +32,6 @@ import (
 	"runtime"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -477,8 +476,12 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			// if no xl.meta/xl.json found, skip the file.
 			return sizeSummary{}, errSkipFile
 		}
+		stopFn := globalScannerMetrics.log(scannerMetricScanObject, s.diskPath, PathJoin(item.bucket, item.objectPath()))
+		defer stopFn()
 
+		doneSz := globalScannerMetrics.timeSize(scannerMetricReadMetadata)
 		buf, err := s.readMetadata(ctx, item.Path)
+		doneSz(len(buf))
 		if err != nil {
 			if intDataUpdateTracker.debug {
 				console.Debugf(color.Green("scannerBucket:")+" object path missing: %v: %w\n", item.Path, err)
@@ -502,8 +505,11 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		if noTiers = globalTierConfigMgr.Empty(); !noTiers {
 			sizeS.tiers = make(map[string]tierStats)
 		}
-		atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
+
+		done := globalScannerMetrics.time(scannerMetricApplyAll)
 		fivs.Versions, err = item.applyVersionActions(ctx, objAPI, fivs.Versions)
+		done()
+
 		if err != nil {
 			if intDataUpdateTracker.debug {
 				console.Debugf(color.Green("scannerBucket:")+" applying version actions failed: %v: %w\n", item.Path, err)
@@ -514,9 +520,10 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		versioned := vcfg != nil && vcfg.Versioned(item.objectPath())
 
 		for _, version := range fivs.Versions {
-			atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
 			oi := version.ToObjectInfo(item.bucket, item.objectPath(), versioned)
+			done = globalScannerMetrics.time(scannerMetricApplyVersion)
 			sz := item.applyActions(ctx, objAPI, oi, &sizeS)
+			done()
 			if oi.VersionID != "" && sz == oi.Size {
 				sizeS.versions++
 			}
@@ -528,7 +535,6 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			//    tracking deleted transitioned objects
 			switch {
 			case noTiers, oi.DeleteMarker, oi.TransitionedObject.FreeVersion:
-
 				continue
 			}
 			tier := minioHotTier
@@ -541,7 +547,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		// apply tier sweep action on free versions
 		for _, freeVersion := range fivs.FreeVersions {
 			oi := freeVersion.ToObjectInfo(item.bucket, item.objectPath(), versioned)
+			done = globalScannerMetrics.time(scannerMetricTierObjSweep)
 			item.applyTierObjSweep(ctx, objAPI, oi)
+			done()
 		}
 		return sizeS, nil
 	}, scanMode)
@@ -1847,7 +1855,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 	}
 
 	defer func() {
-		disk.Fdatasync(w) // Only interested in flushing the size_t not mtime/atime
+		Fdatasync(w) // Only interested in flushing the size_t not mtime/atime
 		w.Close()
 	}()
 

docs/distributed/DESIGN.md

@@ -41,7 +41,7 @@ Expansion of ellipses and choice of erasure sets based on this expansion is an a
 
 - Erasure coding used by MinIO is [Reed-Solomon](https://github.com/klauspost/reedsolomon) erasure coding scheme, which has a total shard maximum of 256 i.e 128 data and 128 parity. MinIO design goes beyond this limitation by doing some practical architecture choices.
 
-- Erasure set is a single erasure coding unit within a MinIO deployment. An object is sharded within an erasure set. Erasure set size is automatically calculated based on the number of disks. MinIO supports unlimited number of disks but each erasure set can be upto 16 disks and a minimum of 4 disks.
+- Erasure set is a single erasure coding unit within a MinIO deployment. An object is sharded within an erasure set. Erasure set size is automatically calculated based on the number of disks. MinIO supports unlimited number of disks but each erasure set can be upto 16 disks and a minimum of 2 disks.
 
 - We limited the number of drives to 16 for erasure set because, erasure code shards more than 16 can become chatty and do not have any performance advantages. Additionally since 16 drive erasure set gives you tolerance of 8 disks per object by default which is plenty in any practical scenario.
 

docs/distributed/README.md

@@ -8,7 +8,7 @@ MinIO in distributed mode can help you setup a highly-available storage system w
 
 ### Data protection
 
-Distributed MinIO provides protection against multiple node/drive failures and [bit rot](https://github.com/minio/minio/blob/master/docs/erasure/README.md#what-is-bit-rot-protection) using [erasure code](https://docs.min.io/docs/minio-erasure-code-quickstart-guide). As the minimum disks required for distributed MinIO is 4 (same as minimum disks required for erasure coding), erasure code automatically kicks in as you launch distributed MinIO.
+Distributed MinIO provides protection against multiple node/drive failures and [bit rot](https://github.com/minio/minio/blob/master/docs/erasure/README.md#what-is-bit-rot-protection) using [erasure code](https://docs.min.io/docs/minio-erasure-code-quickstart-guide). As the minimum disks required for distributed MinIO is 2 (same as minimum disks required for erasure coding), erasure code automatically kicks in as you launch distributed MinIO.
 
 If one or more disks are offline at the start of a PutObject or NewMultipartUpload operation the object will have additional data protection bits added automatically to provide additional safety for these objects.
 
@@ -22,9 +22,12 @@ Refer to sizing guide for more understanding on default values chosen depending
 
 ### Consistency Guarantees
 
-MinIO follows strict **read-after-write** and **list-after-write** consistency model for all i/o operations both in distributed and standalone modes. This consistency model is only guaranteed if you use disk filesystems such as xfs, ext4 or zfs etc.. for distributed setup.
+MinIO follows strict **read-after-write** and **list-after-write** consistency model for all i/o operations both in distributed and standalone modes. This consistency model is only guaranteed if you use disk filesystems such as xfs, zfs or btrfs etc.. for distributed setup.
+
+**In our tests we also found ext4 does not honor POSIX O_DIRECT/Fdatasync semantics, ext4 trades performance for consistency guarantees. Please avoid ext4 in your setup.**
+
+**If MinIO distributed setup is using NFS volumes underneath it is not guaranteed MinIO will provide these consistency guarantees since NFS is not strictly consistent (If you must use NFS we recommend that you atleast use NFSv4 instead of NFSv3 for relatively better outcomes).**
 
-**If MinIO distributed setup is using NFS volumes underneath it is not guaranteed MinIO will provide these consistency guarantees since NFS is not consistent filesystem by design (If you must use NFS we recommend that you atleast use NFSv4 instead of NFSv3).**
 
 ## Get started
 
@@ -41,7 +44,7 @@ To start a distributed MinIO instance, you just need to pass drive locations as
 **NOTE:**
 
 - All the nodes running distributed MinIO should share a common root credentials, for the nodes to connect and trust each other. To achieve this, it is **recommended** to export root user and root password as environment variables, `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD`, on all the nodes before executing MinIO server command. If not exported, default `minioadmin/minioadmin` credentials shall be used.
-- **MinIO creates erasure-coding sets of _4_ to _16_ drives per set.  The number of drives you provide in total must be a multiple of one of those numbers.**
+- **MinIO creates erasure-coding sets of _2_ to _16_ drives per set.  The number of drives you provide in total must be a multiple of one of those numbers.**
 - **MinIO chooses the largest EC set size which divides into the total number of drives or total number of nodes given - making sure to keep the uniform distribution i.e each node participates equal number of drives per set**.
 - **Each object is written to a single EC set, and therefore is spread over no more than 16 drives.**
 - **All the nodes running distributed MinIO setup are recommended to be homogeneous, i.e. same operating system, same number of disks and same network interconnects.**