update console to v0.19.3

For easier debugging. We still do not return compressed size for security reasons.

.github/workflows/go-healing.yml

@@ -48,3 +48,4 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify-healing
           make verify-healing-inconsistent-versions
+          make verify-healing-with-root-disks

.github/workflows/markdown-lint.yaml

@@ -27,4 +27,4 @@ jobs:
         npm install -g markdownlint-cli
         markdownlint --fix '**/*.md' \
           --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
-          --disable MD013 MD040
+          --disable MD013 MD040 MD051

.github/workflows/replication.yaml

@@ -1,4 +1,4 @@
-name: Multi-site replication tests
+name: MinIO advanced tests
 
 on:
   pull_request:
@@ -16,7 +16,7 @@ permissions:
 
 jobs:
   replication-test:
-    name: Replication Tests with Go ${{ matrix.go-version }}
+    name: Advanced Tests with Go ${{ matrix.go-version }}
     runs-on: ubuntu-latest
 
     strategy:
@@ -37,11 +37,18 @@ jobs:
           key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Test Decom
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-decom
+
       - name: Test Replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-replication
+
       - name: Test MinIO IDP for automatic site replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

Dockerfile.hotfix

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG RELEASE
 

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG TARGETARCH
 

Dockerfile.release.fips

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
 
 ARG TARGETARCH
 

Makefile

@@ -39,7 +39,14 @@ lint: ## runs golangci-lint suite of linters
 check: test
 test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@CGO_ENABLED=0 go test -tags kqueue ./...
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
+
+test-decom: install
+	@echo "Running minio decom tests"
+	@env bash $(PWD)/docs/distributed/decom.sh
+	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
+	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
+	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
 test-upgrade: build
 	@echo "Running minio upgrade tests"
@@ -51,9 +58,9 @@ test-race: verifiers build ## builds minio, runs linters, tests (race)
 
 test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
-	@CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
 test-replication: install ## verify multi site replication
 	@echo "Running tests for replicating three sites"
@@ -83,6 +90,11 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
+verify-healing-with-root-disks:
+	@echo "Verify healing with root disks"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
+
 verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@echo "Verify resolving inconsistent versions build with race"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@@ -128,6 +140,8 @@ clean: ## cleanup all generated assets
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@find . -name '*~' | xargs rm -fv
+	@find . -name '.#*#' | xargs rm -fv
+	@find . -name '#*#' | xargs rm -fv
 	@rm -rvf minio
 	@rm -rvf build
 	@rm -rvf release

README.fips.md

@@ -2,6 +2,6 @@
 
 MinIO creates FIPS builds using a patched version of the Go compiler (that uses BoringCrypto, from BoringSSL, which is [FIPS 140-2 validated](https://csrc.nist.gov/csrc/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp2964.pdf)) published by the Golang Team [here](https://github.com/golang/go/tree/dev.boringcrypto/misc/boring).
 
-MinIO FIPS executables are available at http://dl.min.io - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
+MinIO FIPS executables are available at <http://dl.min.io> - they are only published for `linux-amd64` architecture as binary files with the suffix `.fips`. We also publish corresponding container images to our official image repositories.
 
 We are not making any statements or representations about the suitability of this code or build in relation to the FIPS 140-2 standard. Interested users will have to evaluate for themselves whether this is useful for their own purposes.

buildscripts/race.sh

@@ -3,6 +3,8 @@
 set -e
 
 export GORACE="history_size=7"
+export MINIO_API_REQUESTS_MAX=10000
+
 ## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
 for d in $(go list ./... | grep -v dsync); do
     CGO_ENABLED=1 go test -v -race --timeout 100m "$d"

buildscripts/verify-healing-with-root-disks.sh

@@ -0,0 +1,97 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$(mktemp -d)"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+
+function start_minio() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    unset MINIO_CI_CD
+    unset CI
+
+    args=()
+    for i in $(seq 1 4); do
+            args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ")
+    done
+
+    for i in $(seq 1 4); do
+            "${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
+    done
+
+    # Wait until all nodes return 403
+    for i in $(seq 1 4); do
+	    while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do
+		    echo -n ".";
+                    sleep 1;
+            done
+    done
+
+ }
+
+# Prepare fake disks with losetup
+function prepare_block_devices() {
+        mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+        for i in 1 2 3 4; do
+                dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
+                mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
+                sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
+                sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
+                mkdir -p ${WORK_DIR}/mnt/disk$i/
+                sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+                sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+        done
+}
+
+# Start a distributed MinIO setup, unmount one disk and check if it is formatted
+function main() {
+    start_port=$(shuf -i 10000-65000 -n 1)
+    start_minio ${start_port}
+
+    # Unmount the disk, after the unmount the device id
+    # /tmp/xxx/mnt/disk4 will be the same as '/' and it
+    # will be detected as root disk
+    while [ "$u" != "0" ]; do
+	    sudo umount ${WORK_DIR}/mnt/disk4/
+	    u=$?
+	    sleep 1
+    done
+
+    # Wait until MinIO self heal kicks in
+    sleep 60
+
+    if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then
+	    echo "A root disk is formatted unexpectedely"
+            cat "${WORK_DIR}/server4.log"
+            exit -1
+    fi
+}
+
+function cleanup() {
+    pkill minio
+    sudo umount ${WORK_DIR}/mnt/disk{1..3}/
+    sudo rm /dev/minio-loopdisk*
+    rm -rf "$WORK_DIR"
+}
+
+( prepare_block_devices )
+( main "$@" )
+rv=$?
+
+cleanup
+exit "$rv"
+

cmd/admin-bucket-handlers.go

@@ -63,7 +63,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -92,15 +91,17 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	bucketMeta := madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeQuotaConfig,
-		Bucket: bucket,
-		Quota:  data,
+		Type:      madmin.SRBucketMetaTypeQuotaConfig,
+		Bucket:    bucket,
+		Quota:     data,
+		UpdatedAt: updatedAt,
 	}
 	if quotaConfig.Quota == 0 {
 		bucketMeta.Quota = nil
@@ -124,7 +125,6 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -169,7 +169,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -267,7 +266,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -298,7 +297,6 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 	if bucket != "" {
@@ -338,7 +336,6 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
@@ -362,7 +359,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -384,9 +381,9 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
+
 	var (
 		buckets []BucketInfo
 		err     error
@@ -640,7 +637,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
 	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 	data, err := ioutil.ReadAll(r.Body)
@@ -707,7 +703,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -721,6 +718,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 				Bucket:           bucket,
 				ObjectLockConfig: &cfgStr,
+				UpdatedAt:        updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -794,7 +792,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -804,7 +802,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 	for _, file := range zr.File {
 		reader, err := file.Open()
 		if err != nil {
-			writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+			writeErrorResponse(ctx, w, importError(ctx, err, file.Name, ""), r.URL)
 			return
 		}
 		sz := file.FileInfo().Size()
@@ -846,7 +844,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -883,15 +881,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
 			// Call site replication hook.
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypePolicy,
-				Bucket: bucket,
-				Policy: bucketPolicyBytes,
+				Type:      madmin.SRBucketMetaTypePolicy,
+				Bucket:    bucket,
+				Policy:    bucketPolicyBytes,
+				UpdatedAt: updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -922,7 +922,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
+			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -947,7 +947,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			kmsKey := encConfig.KeyID()
 			if kmsKey != "" {
 				kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
-				_, err := GlobalKMS.GenerateKey(kmsKey, kmsContext)
+				_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
 				if err != nil {
 					if errors.Is(err, kes.ErrKeyNotFound) {
 						writeErrorResponse(ctx, w, importError(ctx, errKMSKeyNotFound, file.Name, bucket), r.URL)
@@ -965,7 +965,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 
 			// Store the bucket encryption configuration in the object layer
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -979,6 +980,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Type:      madmin.SRBucketMetaTypeSSEConfig,
 				Bucket:    bucket,
 				SSEConfig: &cfgStr,
+				UpdatedAt: updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -998,7 +1000,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
@@ -1008,9 +1011,10 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			// errors.
 			cfgStr := base64.StdEncoding.EncodeToString(configData)
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypeTags,
-				Bucket: bucket,
-				Tags:   &cfgStr,
+				Type:      madmin.SRBucketMetaTypeTags,
+				Bucket:    bucket,
+				Tags:      &cfgStr,
+				UpdatedAt: updatedAt,
 			}); err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
@@ -1033,15 +1037,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 
-			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data); err != nil {
+			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
+			if err != nil {
 				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
 				return
 			}
 
 			bucketMeta := madmin.SRBucketMeta{
-				Type:   madmin.SRBucketMetaTypeQuotaConfig,
-				Bucket: bucket,
-				Quota:  data,
+				Type:      madmin.SRBucketMetaTypeQuotaConfig,
+				Bucket:    bucket,
+				Quota:     data,
+				UpdatedAt: updatedAt,
 			}
 			if quotaConfig.Quota == 0 {
 				bucketMeta.Quota = nil
@@ -1055,3 +1061,79 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 	}
 }
+
+// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
+// to the connected HTTP client. This is a MinIO only extension
+func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ReplicationDiff")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if globalIsGateway {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	opts := extractReplicateDiffOpts(r.Form)
+	if opts.ARN != "" {
+		tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, opts.ARN)
+		if tgt.Empty() {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, fmt.Errorf("invalid arn : '%s'", opts.ARN)), r.URL)
+			return
+		}
+	}
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	diffCh, err := getReplicationDiff(ctx, objectAPI, bucket, opts)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	enc := json.NewEncoder(w)
+	for {
+		select {
+		case entry, ok := <-diffCh:
+			if !ok {
+				return
+			}
+			if err := enc.Encode(entry); err != nil {
+				return
+			}
+			if len(diffCh) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
+			}
+		case <-keepAliveTicker.C:
+			if len(diffCh) > 0 {
+				continue
+			}
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-ctx.Done():
+			return
+		}
+	}
+}

cmd/admin-handler-utils.go

@@ -30,6 +30,10 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
+// validateAdminReq will validate request against and return whether it is allowed.
+// If any of the supplied actions are allowed it will be successful.
+// If nil ObjectLayer is returned, the operation is not permitted.
+// When nil ObjectLayer has been returned an error has always been sent to w.
 func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -41,11 +45,16 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	for _, action := range actions {
 		// Validate request signature.
 		cred, adminAPIErr := checkAdminRequestAuth(ctx, r, action, "")
-		if adminAPIErr != ErrNone {
+		switch adminAPIErr {
+		case ErrNone:
+			return objectAPI, cred
+		case ErrAccessDenied:
+			// Try another
+			continue
+		default:
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 			return nil, cred
 		}
-		return objectAPI, cred
 	}
 	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 	return nil, auth.Credentials{}
@@ -240,3 +249,11 @@ func importError(ctx context.Context, err error, fname, entity string) APIError
 	}
 	return toAPIError(ctx, fmt.Errorf("error importing %s from %s with: %w", entity, fname, err))
 }
+
+// wraps import error for more context
+func importErrorWithAPIErr(ctx context.Context, apiErr APIErrorCode, err error, fname, entity string) APIError {
+	if entity == "" {
+		return errorCodes.ToAPIErrWithErr(apiErr, fmt.Errorf("error importing %s with: %w", fname, err))
+	}
+	return errorCodes.ToAPIErrWithErr(apiErr, fmt.Errorf("error importing %s from %s with: %w", entity, fname, err))
+}

cmd/admin-handlers-idp-config.go

@@ -0,0 +1,321 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+// List of implemented ID config types.
+var idCfgTypes = set.CreateStringSet("openid")
+
+// SetIdentityProviderCfg:
+//
+// PUT <admin-prefix>/id-cfg?type=openid&name=dex1
+func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetIdentityCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err, logger.Application)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	var cfgDataBuilder strings.Builder
+	switch cfgType {
+	case "openid":
+		fmt.Fprintf(&cfgDataBuilder, "identity_openid")
+	}
+
+	// Ensure body content type is opaque.
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/octet-stream" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+		return
+	}
+
+	// Subsystem configuration name could be empty.
+	cfgName := mux.Vars(r)["name"]
+	if cfgName != "" {
+		fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
+	}
+
+	fmt.Fprintf(&cfgDataBuilder, "%s%s", config.KvSpaceSeparator, string(reqBytes))
+
+	cfgData := cfgDataBuilder.String()
+	subSys, _, _, err := config.GetSubSys(cfgData)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	cfg, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	dynamic, err := cfg.ReadConfig(strings.NewReader(cfgData))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// IDP config is not dynamic. Sanity check.
+	if dynamic {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		return
+	}
+
+	if err = validateConfig(cfg, subSys); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+
+	// Update the actual server config on disk.
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write to the config input KV to history.
+	if err = saveServerConfigHistory(ctx, objectAPI, []byte(cfgData)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetIdentityProviderCfg:
+//
+// GET <admin-prefix>/id-cfg?type=openid&name=dex_test
+//
+// GetIdentityProviderCfg returns a list of configured IDPs on the server if
+// name is empty. If name is non-empty, returns the configuration details for
+// the IDP of the given type and configuration name. The configuration name for
+// the default ("un-named") configuration target is `_`.
+func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetIdentityProviderCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	cfgName := r.Form.Get("name")
+	password := cred.SecretKey
+
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// If no cfgName is provided, we list.
+	if cfgName == "" {
+		a.listIdentityProviders(ctx, w, r, cfgType, password)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+
+	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	if err != nil {
+		if err == openid.ErrProviderConfigNotFound {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			return
+		}
+
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	res := madmin.IDPConfig{
+		Type: cfgType,
+		Name: cfgName,
+		Info: cfgInfos,
+	}
+	data, err := json.Marshal(res)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
+	// var subSys string
+	switch cfgType {
+	case "openid":
+		// subSys = config.IdentityOpenIDSubSys
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+	cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(cfgList)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// DeleteIdentityProviderCfg:
+//
+// DELETE <admin-prefix>/id-cfg?type=openid&name=dex_test
+func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteIdentityProviderCfg")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	cfgType := mux.Vars(r)["type"]
+	cfgName := mux.Vars(r)["name"]
+	if !idCfgTypes.Contains(cfgType) {
+		// TODO: change this to invalid type error when implementation
+		// is complete.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg := globalServerConfig.Clone()
+
+	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	if err != nil {
+		if err == openid.ErrProviderConfigNotFound {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			return
+		}
+
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	hasEnv := false
+	for _, ci := range cfgInfos {
+		if ci.IsCfg && ci.IsEnv {
+			hasEnv = true
+			break
+		}
+	}
+
+	if hasEnv {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
+		return
+	}
+
+	var subSys string
+	switch cfgType {
+	case "openid":
+		subSys = config.IdentityOpenIDSubSys
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	cfg, err = readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if err = validateConfig(cfg, subSys); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}
+	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	dynamic := config.SubSystemsDynamic.Contains(subSys)
+	if dynamic {
+		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
+	}
+}

cmd/admin-handlers-pools.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -149,8 +150,10 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 
 	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
+		apiErr := toAdminAPIErr(ctx, errInvalidArgument)
+		apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
 		// We didn't find any matching pools, invalid input
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
 		return
 	}
 

cmd/admin-handlers-site-replication.go

@@ -160,7 +160,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRIAMItemPolicy:
 		if item.Policy == nil {
-			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
 		} else {
 			policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
 			if perr != nil {
@@ -168,21 +168,21 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 				return
 			}
 			if policy.IsEmpty() {
-				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
 			} else {
-				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy, item.UpdatedAt)
 			}
 		}
 	case madmin.SRIAMItemSvcAcc:
-		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange)
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
 	case madmin.SRIAMItemPolicyMapping:
-		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping)
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
 	case madmin.SRIAMItemSTSAcc:
-		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential)
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
 	case madmin.SRIAMItemIAMUser:
-		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser)
+		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
 	case madmin.SRIAMItemGroupInfo:
-		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo)
+		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -214,7 +214,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRBucketMetaTypePolicy:
 		if item.Policy == nil {
-			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 		} else {
 			bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
 			if berr != nil {
@@ -222,33 +222,33 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				return
 			}
 			if bktPolicy.IsEmpty() {
-				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 			} else {
-				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy, item.UpdatedAt)
 			}
 		}
 	case madmin.SRBucketMetaTypeQuotaConfig:
 		if item.Quota == nil {
-			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil)
+			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 		} else {
 			quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
-			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig); err != nil {
+			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 		}
 	case madmin.SRBucketMetaTypeVersionConfig:
-		err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning)
+		err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeTags:
-		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
+		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeObjectLockConfig:
-		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
+		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeSSEConfig:
-		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
+		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)

cmd/admin-handlers-users.go

@@ -24,9 +24,12 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"sort"
+	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
@@ -69,6 +72,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 			AccessKey:   accessKey,
 			IsDeleteReq: true,
 		},
+		UpdatedAt: UTCNow(),
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -237,9 +241,9 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-
+	var updatedAt time.Time
 	if updReq.IsRemove {
-		err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
+		updatedAt, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
 	} else {
 		// Check if group already exists
 		if _, gerr := globalIAMSys.GetGroupDescription(updReq.Group); gerr != nil {
@@ -250,7 +254,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 				return
 			}
 		}
-		err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+		updatedAt, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
 	}
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -262,6 +266,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: updReq,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -338,12 +343,15 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 	group := vars["group"]
 	status := vars["status"]
 
-	var err error
+	var (
+		err       error
+		updatedAt time.Time
+	)
 	switch status {
 	case statusEnabled:
-		err = globalIAMSys.SetGroupStatus(ctx, group, true)
+		updatedAt, err = globalIAMSys.SetGroupStatus(ctx, group, true)
 	case statusDisabled:
-		err = globalIAMSys.SetGroupStatus(ctx, group, false)
+		updatedAt, err = globalIAMSys.SetGroupStatus(ctx, group, false)
 	default:
 		err = errInvalidArgument
 	}
@@ -361,6 +369,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 				IsRemove: false,
 			},
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -388,7 +397,8 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if err := globalIAMSys.SetUserStatus(ctx, accessKey, madmin.AccountStatus(status)); err != nil {
+	updatedAt, err := globalIAMSys.SetUserStatus(ctx, accessKey, madmin.AccountStatus(status))
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -402,6 +412,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 				Status: madmin.AccountStatus(status),
 			},
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -436,8 +447,8 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	userCred, exists := globalIAMSys.GetUser(ctx, accessKey)
-	if exists && (userCred.IsTemp() || userCred.IsServiceAccount()) {
+	user, exists := globalIAMSys.GetUser(ctx, accessKey)
+	if exists && (user.Credentials.IsTemp() || user.Credentials.IsServiceAccount()) {
 		// Updating STS credential is not allowed, and this API does not
 		// support updating service accounts.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
@@ -498,7 +509,8 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
+	updatedAt, err := globalIAMSys.CreateUser(ctx, accessKey, ureq)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -510,6 +522,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 			IsDeleteReq: false,
 			UserReq:     &ureq,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -629,7 +642,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			opts.claims[k] = v
 		}
 	} else {
-		// Need permission if we are creating a service acccount for a
+		// Need permission if we are creating a service account for a
 		// user <> to the request sender
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     requestorUser,
@@ -671,7 +684,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	opts.sessionPolicy = sp
-	newCred, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
+	newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -714,6 +727,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 					Status:        auth.AccountOn,
 				},
 			},
+			UpdatedAt: updatedAt,
 		})
 		if err != nil {
 			logger.LogIf(ctx, err)
@@ -797,7 +811,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		status:        updateReq.NewStatus,
 		sessionPolicy: sp,
 	}
-	err = globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
+	updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -815,6 +829,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 					SessionPolicy: updateReq.NewPolicy,
 				},
 			},
+			UpdatedAt: updatedAt,
 		})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1054,6 +1069,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 					AccessKey: serviceAccount,
 				},
 			},
+			UpdatedAt: UTCNow(),
 		}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1392,8 +1408,9 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	// Call cluster-replication policy creation hook to replicate policy deletion to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-		Type: madmin.SRIAMItemPolicy,
-		Name: policyName,
+		Type:      madmin.SRIAMItemPolicy,
+		Name:      policyName,
+		UpdatedAt: UTCNow(),
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1450,7 +1467,8 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	if err = globalIAMSys.SetPolicy(ctx, policyName, *iamPolicy); err != nil {
+	updatedAt, err := globalIAMSys.SetPolicy(ctx, policyName, *iamPolicy)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1458,9 +1476,10 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	// Call cluster-replication policy creation hook to replicate policy to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-		Type:   madmin.SRIAMItemPolicy,
-		Name:   policyName,
-		Policy: iamPolicyBytes,
+		Type:      madmin.SRIAMItemPolicy,
+		Name:      policyName,
+		Policy:    iamPolicyBytes,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1495,7 +1514,8 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		}
 	}
 
-	if err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup); err != nil {
+	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1507,8 +1527,680 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 			IsGroup:     isGroup,
 			Policy:      policyName,
 		},
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
+
+const (
+	allPoliciesFile            = "policies.json"
+	allUsersFile               = "users.json"
+	allGroupsFile              = "groups.json"
+	allSvcAcctsFile            = "svcaccts.json"
+	userPolicyMappingsFile     = "user_mappings.json"
+	groupPolicyMappingsFile    = "group_mappings.json"
+	stsUserPolicyMappingsFile  = "stsuser_mappings.json"
+	stsGroupPolicyMappingsFile = "stsgroup_mappings.json"
+	iamAssetsDir               = "iam-assets"
+)
+
+// ExportIAMHandler - exports all iam info as a zipped file
+func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ExportIAM")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportIAMAction)
+	if objectAPI == nil {
+		return
+	}
+	// Initialize a zip writer which will provide a zipped content
+	// of bucket metadata
+	zipWriter := zip.NewWriter(w)
+	defer zipWriter.Close()
+	rawDataFn := func(r io.Reader, filename string, sz int) error {
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    filename,
+			size:    int64(sz),
+			mode:    0o600,
+			modTime: time.Now(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		header.Method = zip.Deflate
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		if _, err := io.Copy(zwriter, r); err != nil {
+			logger.LogIf(ctx, err)
+		}
+		return nil
+	}
+
+	iamFiles := []string{
+		allPoliciesFile,
+		allUsersFile,
+		allGroupsFile,
+		allSvcAcctsFile,
+		userPolicyMappingsFile,
+		groupPolicyMappingsFile,
+		stsUserPolicyMappingsFile,
+		stsGroupPolicyMappingsFile,
+	}
+	for _, f := range iamFiles {
+		iamFile := pathJoin(iamAssetsDir, f)
+		switch f {
+		case allPoliciesFile:
+			allPolicies, err := globalIAMSys.ListPolicies(ctx, "")
+			if err != nil {
+				logger.LogIf(ctx, err)
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+
+			policiesData, err := json.Marshal(allPolicies)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			if err = rawDataFn(bytes.NewReader(policiesData), iamFile, len(policiesData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		case allUsersFile:
+			userIdentities := make(map[string]UserIdentity)
+			globalIAMSys.store.rlock()
+			err := globalIAMSys.store.loadUsers(ctx, regUser, userIdentities)
+			globalIAMSys.store.runlock()
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			userAccounts := make(map[string]madmin.AddOrUpdateUserReq)
+			for u, uid := range userIdentities {
+				status := madmin.AccountDisabled
+				if uid.Credentials.IsValid() {
+					status = madmin.AccountEnabled
+				}
+				userAccounts[u] = madmin.AddOrUpdateUserReq{
+					SecretKey: uid.Credentials.SecretKey,
+					Status:    status,
+				}
+			}
+			userData, err := json.Marshal(userAccounts)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+
+			if err = rawDataFn(bytes.NewReader(userData), iamFile, len(userData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		case allGroupsFile:
+			groups := make(map[string]GroupInfo)
+			globalIAMSys.store.rlock()
+			err := globalIAMSys.store.loadGroups(ctx, groups)
+			globalIAMSys.store.runlock()
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			groupData, err := json.Marshal(groups)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+
+			if err = rawDataFn(bytes.NewReader(groupData), iamFile, len(groupData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		case allSvcAcctsFile:
+			serviceAccounts := make(map[string]UserIdentity)
+			globalIAMSys.store.rlock()
+			err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts)
+			globalIAMSys.store.runlock()
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			svcAccts := make(map[string]madmin.SRSvcAccCreate)
+			for user, acc := range serviceAccounts {
+				claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+					return
+				}
+				_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+					return
+				}
+
+				var policyJSON []byte
+				if policy != nil {
+					policyJSON, err = json.Marshal(policy)
+					if err != nil {
+						writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+						return
+					}
+				}
+				svcAccts[user] = madmin.SRSvcAccCreate{
+					Parent:        acc.Credentials.ParentUser,
+					AccessKey:     user,
+					SecretKey:     acc.Credentials.SecretKey,
+					Groups:        acc.Credentials.Groups,
+					Claims:        claims,
+					SessionPolicy: json.RawMessage(policyJSON),
+					Status:        acc.Credentials.Status,
+				}
+			}
+
+			svcAccData, err := json.Marshal(svcAccts)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+
+			if err = rawDataFn(bytes.NewReader(svcAccData), iamFile, len(svcAccData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		case userPolicyMappingsFile:
+			userPolicyMap := make(map[string]MappedPolicy)
+			globalIAMSys.store.rlock()
+			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
+			globalIAMSys.store.runlock()
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			userPolData, err := json.Marshal(userPolicyMap)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+
+			if err = rawDataFn(bytes.NewReader(userPolData), iamFile, len(userPolData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		case groupPolicyMappingsFile:
+			groupPolicyMap := make(map[string]MappedPolicy)
+			globalIAMSys.store.rlock()
+			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
+			globalIAMSys.store.runlock()
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			grpPolData, err := json.Marshal(groupPolicyMap)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+
+			if err = rawDataFn(bytes.NewReader(grpPolData), iamFile, len(grpPolData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		case stsUserPolicyMappingsFile:
+			userPolicyMap := make(map[string]MappedPolicy)
+			globalIAMSys.store.rlock()
+			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
+			globalIAMSys.store.runlock()
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			userPolData, err := json.Marshal(userPolicyMap)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			if err = rawDataFn(bytes.NewReader(userPolData), iamFile, len(userPolData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		case stsGroupPolicyMappingsFile:
+			groupPolicyMap := make(map[string]MappedPolicy)
+			globalIAMSys.store.rlock()
+			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
+			globalIAMSys.store.runlock()
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			grpPolData, err := json.Marshal(groupPolicyMap)
+			if err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+			if err = rawDataFn(bytes.NewReader(grpPolData), iamFile, len(grpPolData)); err != nil {
+				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
+				return
+			}
+		}
+	}
+}
+
+// ImportIAM - imports all IAM info into MinIO
+func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ImportIAM")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+	reader := bytes.NewReader(data)
+	zr, err := zip.NewReader(reader, int64(len(data)))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+	// import policies first
+	{
+
+		f, err := zr.Open(pathJoin(iamAssetsDir, allPoliciesFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var allPolicies map[string]iampolicy.Policy
+			data, err = ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
+				return
+			}
+			err = json.Unmarshal(data, &allPolicies)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allPoliciesFile, ""), r.URL)
+				return
+			}
+			for policyName, policy := range allPolicies {
+				if policy.IsEmpty() {
+					err = globalIAMSys.DeletePolicy(ctx, policyName, true)
+				} else {
+					_, err = globalIAMSys.SetPolicy(ctx, policyName, policy)
+				}
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, allPoliciesFile, policyName), r.URL)
+					return
+				}
+			}
+		}
+	}
+
+	// import users
+	{
+		f, err := zr.Open(pathJoin(iamAssetsDir, allUsersFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allUsersFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var userAccts map[string]madmin.AddOrUpdateUserReq
+			data, err := ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allUsersFile, ""), r.URL)
+				return
+			}
+			err = json.Unmarshal(data, &userAccts)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allUsersFile, ""), r.URL)
+				return
+			}
+			for accessKey, ureq := range userAccts {
+				// Not allowed to add a user with same access key as root credential
+				if owner && accessKey == cred.AccessKey {
+					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
+					return
+				}
+
+				user, exists := globalIAMSys.GetUser(ctx, accessKey)
+				if exists && (user.Credentials.IsTemp() || user.Credentials.IsServiceAccount()) {
+					// Updating STS credential is not allowed, and this API does not
+					// support updating service accounts.
+					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
+					return
+				}
+
+				if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
+					// Incoming access key matches parent user then we should
+					// reject password change requests.
+					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
+					return
+				}
+
+				// Check if accessKey has beginning and end space characters, this only applies to new users.
+				if !exists && hasSpaceBE(accessKey) {
+					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, err, allUsersFile, accessKey), r.URL)
+					return
+				}
+
+				checkDenyOnly := false
+				if accessKey == cred.AccessKey {
+					// Check that there is no explicit deny - otherwise it's allowed
+					// to change one's own password.
+					checkDenyOnly = true
+				}
+
+				if !globalIAMSys.IsAllowed(iampolicy.Args{
+					AccountName:     cred.AccessKey,
+					Groups:          cred.Groups,
+					Action:          iampolicy.CreateUserAdminAction,
+					ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+					IsOwner:         owner,
+					Claims:          claims,
+					DenyOnly:        checkDenyOnly,
+				}) {
+					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allUsersFile, accessKey), r.URL)
+					return
+				}
+				if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
+					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, toAdminAPIErrCode(ctx, err), err, allUsersFile, accessKey), r.URL)
+					return
+				}
+
+			}
+		}
+	}
+
+	// import groups
+	{
+		f, err := zr.Open(pathJoin(iamAssetsDir, allGroupsFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allGroupsFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var grpInfos map[string]GroupInfo
+			data, err := ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allGroupsFile, ""), r.URL)
+				return
+			}
+			if err = json.Unmarshal(data, &grpInfos); err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allGroupsFile, ""), r.URL)
+				return
+			}
+			for group, grpInfo := range grpInfos {
+				// Check if group already exists
+				if _, gerr := globalIAMSys.GetGroupDescription(group); gerr != nil {
+					// If group does not exist, then check if the group has beginning and end space characters
+					// we will reject such group names.
+					if errors.Is(gerr, errNoSuchGroup) && hasSpaceBE(group) {
+						writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, err, allGroupsFile, group), r.URL)
+						return
+					}
+				}
+				if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, allGroupsFile, group), r.URL)
+					return
+				}
+			}
+		}
+	}
+
+	// import service accounts
+	{
+		f, err := zr.Open(pathJoin(iamAssetsDir, allSvcAcctsFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allSvcAcctsFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var serviceAcctReqs map[string]madmin.SRSvcAccCreate
+			data, err := ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allSvcAcctsFile, ""), r.URL)
+				return
+			}
+			if err = json.Unmarshal(data, &serviceAcctReqs); err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allSvcAcctsFile, ""), r.URL)
+				return
+			}
+			for user, svcAcctReq := range serviceAcctReqs {
+				var sp *iampolicy.Policy
+				var err error
+				if len(svcAcctReq.SessionPolicy) > 0 {
+					sp, err = iampolicy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
+					if err != nil {
+						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
+						return
+					}
+				}
+				// service account access key cannot have space characters beginning and end of the string.
+				if hasSpaceBE(svcAcctReq.AccessKey) {
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
+					return
+				}
+				if !globalIAMSys.IsAllowed(iampolicy.Args{
+					AccountName:     svcAcctReq.AccessKey,
+					Groups:          svcAcctReq.Groups,
+					Action:          iampolicy.CreateServiceAccountAdminAction,
+					ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+					IsOwner:         owner,
+					Claims:          claims,
+				}) {
+					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allSvcAcctsFile, user), r.URL)
+					return
+				}
+				updateReq := true
+				_, _, err = globalIAMSys.GetServiceAccount(ctx, svcAcctReq.AccessKey)
+				if err != nil {
+					if !errors.Is(err, errNoSuchServiceAccount) {
+						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
+						return
+					}
+					updateReq = false
+				}
+				if updateReq {
+					opts := updateServiceAccountOpts{
+						secretKey:     svcAcctReq.SecretKey,
+						status:        svcAcctReq.Status,
+						sessionPolicy: sp,
+					}
+					_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
+					if err != nil {
+						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
+						return
+					}
+					continue
+				}
+				opts := newServiceAccountOpts{
+					accessKey:     user,
+					secretKey:     svcAcctReq.SecretKey,
+					sessionPolicy: sp,
+					claims:        svcAcctReq.Claims,
+				}
+
+				// In case of LDAP we need to resolve the targetUser to a DN and
+				// query their groups:
+				if globalLDAPConfig.Enabled {
+					opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
+					targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
+					if err != nil {
+						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
+						return
+					}
+					opts.claims[ldapUser] = targetUser // username DN
+				}
+
+				if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
+					return
+				}
+
+			}
+		}
+	}
+
+	// import user policy mappings
+	{
+		f, err := zr.Open(pathJoin(iamAssetsDir, userPolicyMappingsFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, userPolicyMappingsFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var userPolicyMap map[string]MappedPolicy
+			data, err := ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, userPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			if err = json.Unmarshal(data, &userPolicyMap); err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, userPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			for u, pm := range userPolicyMap {
+				// disallow setting policy mapping if user is a temporary user
+				ok, _, err := globalIAMSys.IsTempUser(u)
+				if err != nil && err != errNoSuchUser {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
+					return
+				}
+				if ok {
+					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, userPolicyMappingsFile, u), r.URL)
+					return
+				}
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
+					return
+				}
+			}
+		}
+	}
+
+	// import group policy mappings
+	{
+		f, err := zr.Open(pathJoin(iamAssetsDir, groupPolicyMappingsFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, groupPolicyMappingsFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var grpPolicyMap map[string]MappedPolicy
+			data, err := ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, groupPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			if err = json.Unmarshal(data, &grpPolicyMap); err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, groupPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			for g, pm := range grpPolicyMap {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
+					return
+				}
+			}
+		}
+	}
+
+	// import sts user policy mappings
+	{
+		f, err := zr.Open(pathJoin(iamAssetsDir, stsUserPolicyMappingsFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsUserPolicyMappingsFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var userPolicyMap map[string]MappedPolicy
+			data, err := ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsUserPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			if err = json.Unmarshal(data, &userPolicyMap); err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, stsUserPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			for u, pm := range userPolicyMap {
+				// disallow setting policy mapping if user is a temporary user
+				ok, _, err := globalIAMSys.IsTempUser(u)
+				if err != nil && err != errNoSuchUser {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
+					return
+				}
+				if ok {
+					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL)
+					return
+				}
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
+					return
+				}
+			}
+		}
+	}
+
+	// import sts group policy mappings
+	{
+		f, err := zr.Open(pathJoin(iamAssetsDir, stsGroupPolicyMappingsFile))
+		switch {
+		case errors.Is(err, os.ErrNotExist):
+		case err != nil:
+			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
+			return
+		default:
+			defer f.Close()
+			var grpPolicyMap map[string]MappedPolicy
+			data, err := ioutil.ReadAll(f)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			if err = json.Unmarshal(data, &grpPolicyMap); err != nil {
+				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, stsGroupPolicyMappingsFile, ""), r.URL)
+				return
+			}
+			for g, pm := range grpPolicyMap {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
+					return
+				}
+			}
+		}
+	}
+}

cmd/admin-handlers.go

@@ -28,6 +28,7 @@ import (
 	"hash/crc32"
 	"io"
 	"io/ioutil"
+	"math"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -50,6 +51,7 @@ import (
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
+	"github.com/minio/minio/internal/pubsub"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
 	"github.com/secure-io/sio-go"
@@ -343,6 +345,92 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// MetricsHandler - GET /minio/admin/v3/metrics
+// ----------
+// Get realtime server metrics
+func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "Metrics")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+	const defaultMetricsInterval = time.Second
+
+	interval, err := time.ParseDuration(r.Form.Get("interval"))
+	if err != nil || interval < time.Second {
+		interval = defaultMetricsInterval
+	}
+	n, err := strconv.Atoi(r.Form.Get("n"))
+	if err != nil || n <= 0 {
+		n = math.MaxInt32
+	}
+	var types madmin.MetricType
+	if t, _ := strconv.ParseUint(r.Form.Get("types"), 10, 64); t != 0 {
+		types = madmin.MetricType(t)
+	} else {
+		types = madmin.MetricsAll
+	}
+	hosts := strings.Split(r.Form.Get("hosts"), ",")
+	byhost := strings.EqualFold(r.Form.Get("by-host"), "true")
+	var hostMap map[string]struct{}
+	if len(hosts) > 0 && hosts[0] != "" {
+		hostMap = make(map[string]struct{}, len(hosts))
+		for _, k := range hosts {
+			if k != "" {
+				hostMap[k] = struct{}{}
+			}
+		}
+	}
+	done := ctx.Done()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	w.Header().Set(xhttp.ContentType, string(mimeJSON))
+
+	for n > 0 {
+		var m madmin.RealtimeMetrics
+		mLocal := collectLocalMetrics(types, hostMap)
+		m.Merge(&mLocal)
+
+		// Allow half the interval for collecting remote...
+		cctx, cancel := context.WithTimeout(ctx, interval/2)
+		mRemote := collectRemoteMetrics(cctx, types, hostMap)
+		cancel()
+		m.Merge(&mRemote)
+		if !byhost {
+			m.ByHost = nil
+		}
+
+		m.Final = n <= 1
+		// Marshal API response
+		jsonBytes, err := json.Marshal(m)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		_, err = w.Write(jsonBytes)
+		if err != nil {
+			n = 0
+		}
+
+		n--
+		if n <= 0 {
+			break
+		}
+
+		// Flush before waiting for next...
+		w.(http.Flusher).Flush()
+
+		select {
+		case <-ticker.C:
+		case <-done:
+			return
+		}
+	}
+}
+
 // DataUsageInfoHandler - GET /minio/admin/v3/datausage
 // ----------
 // Get server/cluster data usage info
@@ -1061,17 +1149,17 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SpeedtestHandler - Deprecated. See ObjectSpeedtestHandler
-func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	a.ObjectSpeedtestHandler(w, r)
+// SpeedtestHandler - Deprecated. See ObjectSpeedTestHandler
+func (a adminAPIHandlers) SpeedTestHandler(w http.ResponseWriter, r *http.Request) {
+	a.ObjectSpeedTestHandler(w, r)
 }
 
-// ObjectSpeedtestHandler - reports maximum speed of a cluster by performing PUT and
+// ObjectSpeedTestHandler - reports maximum speed of a cluster by performing PUT and
 // GET operations on the server, supports auto tuning by default by automatically
 // increasing concurrency and stopping when we have reached the limits on the
 // system.
-func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ObjectSpeedtestHandler")
+func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ObjectSpeedTestHandler")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -1088,8 +1176,9 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
+	storageClass := strings.TrimSpace(r.Form.Get(peerRESTStorageClass))
+	customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
 	autotune := r.Form.Get("autotune") == "true"
-	storageClass := r.Form.Get("storage-class")
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1124,15 +1213,23 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 		autotune = false
 	}
 
-	bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
-		return
+	if customBucket == "" {
+		customBucket = globalObjectPerfBucket
+
+		bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, customBucket)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+
+		if !bucketExists {
+			defer deleteObjectPerfBucket(objectAPI)
+		}
 	}
 
-	if !bucketExists {
-		defer deleteObjectPerfBucket(objectAPI)
-	}
+	defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
+		DeletePrefix: true,
+	})
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1144,7 +1241,14 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)
-	ch := objectSpeedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
+	ch := objectSpeedTest(ctx, speedTestOpts{
+		objectSize:       size,
+		concurrencyStart: concurrent,
+		duration:         duration,
+		autotune:         autotune,
+		storageClass:     storageClass,
+		bucketName:       customBucket,
+	})
 	for {
 		select {
 		case <-ctx.Done():
@@ -1167,9 +1271,8 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	}
 }
 
-func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer) (bucketExists bool, err error) {
-	err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
-	if err != nil {
+func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
+	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
@@ -1189,7 +1292,7 @@ func deleteObjectPerfBucket(objectAPI ObjectLayer) {
 func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
 	storageInfo, _ := objectAPI.StorageInfo(ctx)
 	capacityNeeded := uint64(concurrent * size)
-	capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))
+	capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)
 
 	if capacity < capacityNeeded {
 		return false, false, fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s",
@@ -1300,72 +1403,45 @@ const (
 // - input entry is not of the type *madmin.TraceInfo*
 // - errOnly entries are to be traced, not status code 2xx, 3xx.
 // - madmin.TraceInfo type is asked by opts
-func mustTrace(entry interface{}, opts madmin.ServiceTraceOpts) (shouldTrace bool) {
-	trcInfo, ok := entry.(madmin.TraceInfo)
-	if !ok {
+func shouldTrace(trcInfo madmin.TraceInfo, opts madmin.ServiceTraceOpts) (shouldTrace bool) {
+	// Reject all unwanted types.
+	want := opts.TraceTypes()
+	if !want.Contains(trcInfo.TraceType) {
 		return false
 	}
 
-	// Override shouldTrace decision with errOnly filtering
-	defer func() {
-		if shouldTrace && opts.OnlyErrors {
-			shouldTrace = trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
-		}
-	}()
+	isHTTP := trcInfo.TraceType.Overlaps(madmin.TraceInternal|madmin.TraceS3) && trcInfo.HTTP != nil
 
-	if opts.Threshold > 0 {
-		var latency time.Duration
-		switch trcInfo.TraceType {
-		case madmin.TraceOS:
-			latency = trcInfo.OSStats.Duration
-		case madmin.TraceStorage:
-			latency = trcInfo.StorageStats.Duration
-		case madmin.TraceHTTP:
-			latency = trcInfo.CallStats.Latency
-		}
-		if latency < opts.Threshold {
-			return false
-		}
+	// Check latency...
+	if opts.Threshold > 0 && trcInfo.Duration < opts.Threshold {
+		return false
 	}
 
-	if opts.Internal && trcInfo.TraceType == madmin.TraceHTTP && HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) {
-		return true
+	// Check internal path
+	isInternal := isHTTP && HasPrefix(trcInfo.HTTP.ReqInfo.Path, minioReservedBucketPath+SlashSeparator)
+	if isInternal && !opts.Internal {
+		return false
 	}
 
-	if opts.S3 && trcInfo.TraceType == madmin.TraceHTTP && !HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) {
-		return true
+	// Filter non-errors.
+	if isHTTP && opts.OnlyErrors && trcInfo.HTTP.RespInfo.StatusCode < http.StatusBadRequest {
+		return false
 	}
 
-	if opts.Storage && trcInfo.TraceType == madmin.TraceStorage {
-		return true
-	}
-
-	return opts.OS && trcInfo.TraceType == madmin.TraceOS
+	return true
 }
 
 func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err error) {
-	q := r.Form
-
-	opts.OnlyErrors = q.Get("err") == "true"
-	opts.S3 = q.Get("s3") == "true"
-	opts.Internal = q.Get("internal") == "true"
-	opts.Storage = q.Get("storage") == "true"
-	opts.OS = q.Get("os") == "true"
-
+	if err := opts.ParseParams(r); err != nil {
+		return opts, err
+	}
 	// Support deprecated 'all' query
-	if q.Get("all") == "true" {
+	if r.Form.Get("all") == "true" {
 		opts.S3 = true
 		opts.Internal = true
 		opts.Storage = true
 		opts.OS = true
-	}
-
-	if t := q.Get("threshold"); t != "" {
-		d, err := time.ParseDuration(t)
-		if err != nil {
-			return opts, err
-		}
-		opts.Threshold = d
+		// Older mc - cannot deal with more types...
 	}
 	return
 }
@@ -1388,20 +1464,21 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-
 	setEventStreamHeaders(w)
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan interface{}, 4000)
+	traceCh := make(chan pubsub.Maskable, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
+	mask := pubsub.MaskFromMaskable(traceOpts.TraceTypes())
 
-	traceFn := func(entry interface{}) bool {
-		return mustTrace(entry, traceOpts)
-	}
-
-	err = globalTrace.Subscribe(traceCh, ctx.Done(), traceFn)
+	err = globalTrace.Subscribe(mask, traceCh, ctx.Done(), func(entry pubsub.Maskable) bool {
+		if e, ok := entry.(madmin.TraceInfo); ok {
+			return shouldTrace(e, traceOpts)
+		}
+		return false
+	})
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
 		return
@@ -1442,7 +1519,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// The handler sends console logs to the connected HTTP client.
+// The ConsoleLogHandler handler sends console logs to the connected HTTP client.
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
@@ -1460,11 +1537,10 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 		limitLines = 10
 	}
 
-	logKind := r.Form.Get("logType")
-	if logKind == "" {
-		logKind = string(logger.All)
+	logKind := madmin.LogKind(strings.ToUpper(r.Form.Get("logType"))).LogMask()
+	if logKind == 0 {
+		logKind = madmin.LogMaskAll
 	}
-	logKind = strings.ToUpper(logKind)
 
 	// Avoid reusing tcp connection if read timeout is hit
 	// This is needed to make r.Context().Done() work as
@@ -1473,7 +1549,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	setEventStreamHeaders(w)
 
-	logCh := make(chan interface{}, 4000)
+	logCh := make(chan pubsub.Maskable, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
@@ -1539,7 +1615,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if err := GlobalKMS.CreateKey(r.Form.Get("key-id")); err != nil {
+	if err := GlobalKMS.CreateKey(ctx, r.Form.Get("key-id")); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1561,7 +1637,7 @@ func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	stat, err := GlobalKMS.Stat()
+	stat, err := GlobalKMS.Stat(ctx)
 	if err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
 		return
@@ -1600,7 +1676,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	stat, err := GlobalKMS.Stat()
+	stat, err := GlobalKMS.Stat(ctx)
 	if err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
 		return
@@ -1616,7 +1692,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 
 	kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
 	// 1. Generate a new key using the KMS.
-	key, err := GlobalKMS.GenerateKey(keyID, kmsContext)
+	key, err := GlobalKMS.GenerateKey(ctx, keyID, kmsContext)
 	if err != nil {
 		response.EncryptionErr = err.Error()
 		resp, err := json.Marshal(response)
@@ -2142,9 +2218,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				autotune = false
 			}
 
-			bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
+			bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, globalObjectPerfBucket)
 			if err != nil {
-				healthInfo.Perf.Error = "Could not make object perf bucket: " + err.Error()
+				healthInfo.Perf.Error = "Unable to create bucket: " + err.Error()
 				partialWrite(healthInfo)
 				return
 			}
@@ -2154,7 +2230,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 
 			opts := speedTestOpts{
-				throughputSize:   size,
+				objectSize:       size,
 				concurrencyStart: concurrent,
 				duration:         10 * time.Second,
 				autotune:         autotune,
@@ -2466,7 +2542,7 @@ func fetchKMSStatus() madmin.KMS {
 		return kmsStat
 	}
 
-	stat, err := GlobalKMS.Stat()
+	stat, err := GlobalKMS.Stat(context.Background())
 	if err != nil {
 		kmsStat.Status = string(madmin.ItemOffline)
 		return kmsStat
@@ -2479,7 +2555,7 @@ func fetchKMSStatus() madmin.KMS {
 
 	kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
 	// 1. Generate a new key using the KMS.
-	key, err := GlobalKMS.GenerateKey("", kmsContext)
+	key, err := GlobalKMS.GenerateKey(context.Background(), "", kmsContext)
 	if err != nil {
 		kmsStat.Encrypt = fmt.Sprintf("Encryption failed: %v", err)
 	} else {

cmd/admin-heal-ops.go

@@ -91,8 +91,11 @@ type allHealState struct {
 	sync.RWMutex
 
 	// map of heal path to heal sequence
-	healSeqMap     map[string]*healSequence // Indexed by endpoint
-	healLocalDisks map[Endpoint]struct{}
+	healSeqMap map[string]*healSequence // Indexed by endpoint
+	// keep track of the healing status of disks in the memory
+	//   false: the disk needs to be healed but no healing routine is started
+	//    true: the disk is currently healing
+	healLocalDisks map[Endpoint]bool
 	healStatus     map[string]healingTracker // Indexed by disk ID
 }
 
@@ -100,7 +103,7 @@ type allHealState struct {
 func newHealState(cleanup bool) *allHealState {
 	hstate := &allHealState{
 		healSeqMap:     make(map[string]*healSequence),
-		healLocalDisks: map[Endpoint]struct{}{},
+		healLocalDisks: make(map[Endpoint]bool),
 		healStatus:     make(map[string]healingTracker),
 	}
 	if cleanup {
@@ -109,13 +112,6 @@ func newHealState(cleanup bool) *allHealState {
 	return hstate
 }
 
-func (ahs *allHealState) healDriveCount() int {
-	ahs.RLock()
-	defer ahs.RUnlock()
-
-	return len(ahs.healLocalDisks)
-}
-
 func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
 	ahs.Lock()
 	defer ahs.Unlock()
@@ -165,23 +161,34 @@ func (ahs *allHealState) getLocalHealingDisks() map[string]madmin.HealingDisk {
 	return dst
 }
 
+// getHealLocalDiskEndpoints() returns the list of disks that need
+// to be healed but there is no healing routine in progress on them.
 func (ahs *allHealState) getHealLocalDiskEndpoints() Endpoints {
 	ahs.RLock()
 	defer ahs.RUnlock()
 
 	var endpoints Endpoints
-	for ep := range ahs.healLocalDisks {
-		endpoints = append(endpoints, ep)
+	for ep, healing := range ahs.healLocalDisks {
+		if !healing {
+			endpoints = append(endpoints, ep)
+		}
 	}
 	return endpoints
 }
 
+func (ahs *allHealState) markDiskForHealing(ep Endpoint) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	ahs.healLocalDisks[ep] = true
+}
+
 func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
 	ahs.Lock()
 	defer ahs.Unlock()
 
 	for _, ep := range healLocalDisks {
-		ahs.healLocalDisks[ep] = struct{}{}
+		ahs.healLocalDisks[ep] = false
 	}
 }
 
@@ -804,16 +811,6 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 	}
 }
 
-// healDiskFormat - heals format.json, return value indicates if a
-// failure error occurred.
-func (h *healSequence) healDiskFormat() error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
-	return h.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
-}
-
 // healBuckets - check for all buckets heal or just particular bucket.
 func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	if h.isQuitting() {

cmd/admin-router.go

@@ -66,6 +66,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
+		// Metrics operation
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(gz(httpTraceAll(adminAPI.MetricsHandler)))
 
 		if globalIsDistErasure || globalIsErasure {
 			// Heal operations
@@ -170,6 +172,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// Set Group Status
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetGroupStatus))).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
+		// Export IAM info to zipped file
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-iam").HandlerFunc(httpTraceHdrs(adminAPI.ExportIAM))
+
+		// Import IAM info
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
+
+		// IDentity Provider configuration APIs
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg))).Queries("type", "{type:.*}")
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/idp-config").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg))).Queries("type", "{type:.*}").Queries("name", "{name:.*}")
+
+		// -- END IAM APIs --
+
 		// GetBucketQuotaConfig
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
@@ -187,6 +202,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// RemoveRemoteTargetHandler
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+		// ReplicationDiff - MinIO extension API
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
 
 		// Bucket migration operations
 		// ExportBucketMetaHandler
@@ -229,8 +247,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 				Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
 		}
 
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedTestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
 

cmd/api-errors.go

@@ -267,6 +267,8 @@ const (
 	ErrAdminConfigNoQuorum
 	ErrAdminConfigTooLarge
 	ErrAdminConfigBadJSON
+	ErrAdminNoSuchConfigTarget
+	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
@@ -1240,11 +1242,21 @@ var errorCodes = errorCodeMap{
 			maxEConfigJSONSize),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoSuchConfigTarget: {
+		Code:           "XMinioAdminNoSuchConfigTarget",
+		Description:    "No such named configuration target exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigBadJSON: {
 		Code:           "XMinioAdminConfigBadJSON",
 		Description:    "JSON configuration provided is of incorrect format",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigEnvOverridden: {
+		Code:           "XMinioAdminConfigEnvOverridden",
+		Description:    "Unable to update config via Admin API due to environment variable override",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigDuplicateKeys: {
 		Code:           "XMinioAdminConfigDuplicateKeys",
 		Description:    "JSON configuration provided has objects with duplicate keys",

cmd/api-headers.go

@@ -199,5 +199,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 	}
 
+	if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
+		if i := strings.LastIndexByte(v, '/'); i >= 0 {
+			v = v[i+1:]
+		}
+		w.Header()[xhttp.MinIOCompressed] = []string{v}
+	}
+
 	return nil
 }

cmd/auth-handler.go

@@ -500,11 +500,18 @@ func isSupportedS3AuthType(aType authType) bool {
 func setAuthHandler(h http.Handler) http.Handler {
 	// handler for validating incoming authorization headers.
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		aType := getRequestAuthType(r)
 		if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
 			// Verify if date headers are set, if not reject the request
 			amzDate, errCode := parseAmzDateHeader(r)
 			if errCode != ErrNone {
+				if ok {
+					tc.funcName = "handler.Auth"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				// All our internal APIs are sensitive towards Date
 				// header, for all requests where Date header is not
 				// present we will reject such clients.
@@ -516,6 +523,11 @@ func setAuthHandler(h http.Handler) http.Handler {
 			// or in the future, reject request otherwise.
 			curTime := UTCNow()
 			if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
+				if ok {
+					tc.funcName = "handler.Auth"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -525,6 +537,12 @@ func setAuthHandler(h http.Handler) http.Handler {
 			h.ServeHTTP(w, r)
 			return
 		}
+
+		if ok {
+			tc.funcName = "handler.Auth"
+			tc.responseRecorder.LogErrBody = true
+		}
+
 		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
 		atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
 	})

cmd/background-heal-ops.go

@@ -22,6 +22,7 @@ import (
 	"runtime"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/pubsub"
 )
 
 // healTask represents what to heal along with options
@@ -49,10 +50,10 @@ type healRoutine struct {
 	workers int
 }
 
-func systemIO() int {
+func activeListeners() int {
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	return int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
+	return int(globalHTTPListen.NumSubscribers(pubsub.MaskAll)) + int(globalTrace.NumSubscribers(pubsub.MaskAll))
 }
 
 func waitForLowHTTPReq() {
@@ -61,7 +62,7 @@ func waitForLowHTTPReq() {
 		currentIO = httpServer.GetRequestCount
 	}
 
-	globalHealConfig.Wait(currentIO, systemIO)
+	globalHealConfig.Wait(currentIO, activeListeners)
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {

cmd/background-newdisks-heal-ops.go

@@ -26,15 +26,12 @@ import (
 	"os"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/console"
 )
 
 const (
@@ -97,7 +94,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 		return nil, err
 	}
 	b, err := disk.ReadAll(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename))
+		pathJoin(bucketMetaPrefix, healingTrackerFilename))
 	if err != nil {
 		return nil, err
 	}
@@ -158,15 +155,19 @@ func (h *healingTracker) save(ctx context.Context) error {
 	}
 	globalBackgroundHealState.updateHealStatus(h)
 	return h.disk.WriteAll(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
+		pathJoin(bucketMetaPrefix, healingTrackerFilename),
 		htrackerBytes)
 }
 
 // delete the tracker on disk.
 func (h *healingTracker) delete(ctx context.Context) error {
 	return h.disk.Delete(ctx, minioMetaBucket,
-		pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
-		false)
+		pathJoin(bucketMetaPrefix, healingTrackerFilename),
+		DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		},
+	)
 }
 
 func (h *healingTracker) isHealed(bucket string) bool {
@@ -258,26 +259,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
-	bgSeq := mustGetHealSequence(ctx)
-
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
-	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
-		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content - use 'mc admin heal alias/ --verbose' to check the status",
-			drivesToHeal, defaultMonitorNewDiskInterval))
-
-		// Heal any disk format and metadata early, if possible.
-		// Start with format healing
-		if err := bgSeq.healDiskFormat(); err != nil {
-			if newObjectLayerFn() != nil {
-				// log only in situations, when object layer
-				// has fully initialized.
-				logger.LogIf(bgSeq.ctx, err)
-			}
-		}
-	}
-
-	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
+	go monitorLocalDisksAndHeal(ctx, z)
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
@@ -299,10 +283,112 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	return disksToHeal
 }
 
+var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+
+func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
+	logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
+
+	disk, format, err := connectEndpoint(endpoint)
+	if err != nil {
+		return fmt.Errorf("Error: %w, %s", err, endpoint)
+	}
+
+	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
+	if poolIdx < 0 {
+		return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
+	}
+
+	// Calculate the set index where the current endpoint belongs
+	z.serverPools[poolIdx].erasureDisksMu.RLock()
+	setIdx, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
+	z.serverPools[poolIdx].erasureDisksMu.RUnlock()
+	if err != nil {
+		return err
+	}
+	if setIdx < 0 {
+		return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
+	}
+
+	// Prevent parallel erasure set healing
+	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-disk-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
+	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer locker.Unlock(lkctx.Cancel)
+
+	buckets, _ := z.ListBuckets(ctx)
+
+	// Buckets data are dispersed in multiple zones/sets, make
+	// sure to heal all bucket metadata configuration.
+	buckets = append(buckets, BucketInfo{
+		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+	}, BucketInfo{
+		Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+	})
+
+	// Heal latest buckets first.
+	sort.Slice(buckets, func(i, j int) bool {
+		a, b := strings.HasPrefix(buckets[i].Name, minioMetaBucket), strings.HasPrefix(buckets[j].Name, minioMetaBucket)
+		if a != b {
+			return a
+		}
+		return buckets[i].Created.After(buckets[j].Created)
+	})
+
+	if serverDebugLog {
+		logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
+	}
+
+	// Load healing tracker in this disk
+	tracker, err := loadHealingTracker(ctx, disk)
+	if err != nil {
+		// So someone changed the drives underneath, healing tracker missing.
+		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
+			disk, humanize.Ordinal(poolIdx+1), err))
+		tracker = newHealingTracker(disk)
+	}
+
+	// Load bucket totals
+	cache := dataUsageCache{}
+	if err := cache.load(ctx, z.serverPools[poolIdx].sets[setIdx], dataUsageCacheName); err == nil {
+		dataUsageInfo := cache.dui(dataUsageRoot, nil)
+		tracker.ObjectsTotalCount = dataUsageInfo.ObjectsTotalCount
+		tracker.ObjectsTotalSize = dataUsageInfo.ObjectsTotalSize
+	}
+
+	tracker.PoolIndex, tracker.SetIndex, tracker.DiskIndex = disk.GetDiskLoc()
+	tracker.setQueuedBuckets(buckets)
+	if err := tracker.save(ctx); err != nil {
+		return err
+	}
+
+	// Start or resume healing of this erasure set
+	if err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker); err != nil {
+		return err
+	}
+
+	if tracker.ItemsFailed > 0 {
+		logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	} else {
+		logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	}
+
+	if serverDebugLog {
+		tracker.printTo(os.Stdout)
+		logger.Info("\n")
+	}
+
+	logger.LogIf(ctx, tracker.delete(ctx))
+
+	return nil
+}
+
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq *healSequence) {
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	diskCheckTimer := time.NewTimer(defaultMonitorNewDiskInterval)
 	defer diskCheckTimer.Stop()
@@ -312,139 +398,34 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 		case <-ctx.Done():
 			return
 		case <-diskCheckTimer.C:
-			var erasureSetInPoolDisksToHeal []map[int][]StorageAPI
-
 			healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints()
-			if len(healDisks) > 0 {
-				// Reformat disks
-				bgSeq.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
-
-				// Ensure that reformatting disks is finished
-				bgSeq.queueHealTask(healSource{bucket: nopHeal}, madmin.HealItemMetadata)
-
-				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal - 'mc admin heal alias/ --verbose' to check the status.",
-					len(healDisks)))
-
-				erasureSetInPoolDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
-				for i := range z.serverPools {
-					erasureSetInPoolDisksToHeal[i] = map[int][]StorageAPI{}
-				}
+			if len(healDisks) == 0 {
+				// Reset for next interval.
+				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
+				continue
 			}
 
-			if serverDebugLog && len(healDisks) > 0 {
-				console.Debugf(color.Green("healDisk:")+" disk check timer fired, attempting to heal %d drives\n", len(healDisks))
+			// Reformat disks immediately
+			_, err := z.HealFormat(context.Background(), false)
+			if err != nil && !errors.Is(err, errNoHealRequired) {
+				logger.LogIf(ctx, err)
+				// Reset for next interval.
+				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
+				continue
 			}
 
-			// heal only if new disks found.
-			for _, endpoint := range healDisks {
-				disk, format, err := connectEndpoint(endpoint)
-				if err != nil {
-					printEndpointError(endpoint, err, true)
-					continue
-				}
-
-				poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
-				if poolIdx < 0 {
-					continue
-				}
-
-				// Calculate the set index where the current endpoint belongs
-				z.serverPools[poolIdx].erasureDisksMu.RLock()
-				// Protect reading reference format.
-				setIndex, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
-				z.serverPools[poolIdx].erasureDisksMu.RUnlock()
-				if err != nil {
-					printEndpointError(endpoint, err, false)
-					continue
-				}
-
-				erasureSetInPoolDisksToHeal[poolIdx][setIndex] = append(erasureSetInPoolDisksToHeal[poolIdx][setIndex], disk)
-			}
-
-			buckets, _ := z.ListBuckets(ctx)
-
-			// Buckets data are dispersed in multiple zones/sets, make
-			// sure to heal all bucket metadata configuration.
-			buckets = append(buckets, BucketInfo{
-				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
-			}, BucketInfo{
-				Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
-			})
-
-			// Heal latest buckets first.
-			sort.Slice(buckets, func(i, j int) bool {
-				a, b := strings.HasPrefix(buckets[i].Name, minioMetaBucket), strings.HasPrefix(buckets[j].Name, minioMetaBucket)
-				if a != b {
-					return a
-				}
-				return buckets[i].Created.After(buckets[j].Created)
-			})
-
-			// TODO(klauspost): This will block until all heals are done,
-			// in the future this should be able to start healing other sets at once.
-			var wg sync.WaitGroup
-			for i, setMap := range erasureSetInPoolDisksToHeal {
-				i := i
-				for setIndex, disks := range setMap {
-					if len(disks) == 0 {
-						continue
+			for _, disk := range healDisks {
+				go func(disk Endpoint) {
+					globalBackgroundHealState.markDiskForHealing(disk)
+					err := healFreshDisk(ctx, z, disk)
+					if err != nil {
+						printEndpointError(disk, err, false)
+						return
 					}
-					wg.Add(1)
-					go func(setIndex int, disks []StorageAPI) {
-						defer wg.Done()
-						for _, disk := range disks {
-							if serverDebugLog {
-								logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
-							}
-
-							// So someone changed the drives underneath, healing tracker missing.
-							tracker, err := loadHealingTracker(ctx, disk)
-							if err != nil {
-								logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
-									disk, humanize.Ordinal(i+1), err))
-								tracker = newHealingTracker(disk)
-							}
-
-							// Load bucket totals
-							cache := dataUsageCache{}
-							if err := cache.load(ctx, z.serverPools[i].sets[setIndex], dataUsageCacheName); err == nil {
-								dataUsageInfo := cache.dui(dataUsageRoot, nil)
-								tracker.ObjectsTotalCount = dataUsageInfo.ObjectsTotalCount
-								tracker.ObjectsTotalSize = dataUsageInfo.ObjectsTotalSize
-							}
-
-							tracker.PoolIndex, tracker.SetIndex, tracker.DiskIndex = disk.GetDiskLoc()
-							tracker.setQueuedBuckets(buckets)
-							if err := tracker.save(ctx); err != nil {
-								logger.LogIf(ctx, err)
-								// Unable to write healing tracker, permission denied or some
-								// other unexpected error occurred. Proceed to look for new
-								// disks to be healed again, we cannot proceed further.
-								return
-							}
-
-							err = z.serverPools[i].sets[setIndex].healErasureSet(ctx, tracker.QueuedBuckets, tracker)
-							if err != nil {
-								logger.LogIf(ctx, err)
-								continue
-							}
-
-							if serverDebugLog {
-								logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
-									humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
-								logger.Info("Summary:\n")
-								tracker.printTo(os.Stdout)
-								logger.Info("\n")
-							}
-							logger.LogIf(ctx, tracker.delete(ctx))
-
-							// Only upon success pop the healed disk.
-							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())
-						}
-					}(setIndex, disks)
-				}
+					// Only upon success pop the healed disk.
+					globalBackgroundHealState.popHealLocalDisks(disk)
+				}(disk)
 			}
-			wg.Wait()
 
 			// Reset for next interval.
 			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)

cmd/bitrot-streaming.go

@@ -152,10 +152,10 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		if len(b.data) == 0 && b.tillOffset != streamOffset {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
-			if err != nil {
+			if err != nil && err != errDiskNotFound {
 				logger.LogIf(GlobalContext,
-					fmt.Errorf("Error(%w) reading erasure shards at (%s: %s/%s), will attempt to reconstruct if we have quorum",
-						err, b.disk, b.volume, b.filePath))
+					fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
+						b.disk, b.volume, b.filePath, err))
 			}
 		} else {
 			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)

cmd/bucket-encryption-handlers.go

@@ -90,7 +90,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	kmsKey := encConfig.KeyID()
 	if kmsKey != "" {
 		kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
-		_, err := GlobalKMS.GenerateKey(kmsKey, kmsContext)
+		_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
 		if err != nil {
 			if errors.Is(err, kes.ErrKeyNotFound) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errKMSKeyNotFound), r.URL)
@@ -108,7 +108,8 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}
 
 	// Store the bucket encryption configuration in the object layer
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -122,6 +123,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: &cfgStr,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -202,7 +204,8 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Delete bucket encryption config from object layer
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -212,6 +215,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: nil,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-handlers.go

@@ -525,7 +525,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			VersionSuspended: vc.Suspended(),
 		}
 
-		if replicateDeletes || hasLockEnabled || !globalTierConfigMgr.Empty() {
+		if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() {
 			if !globalTierConfigMgr.Empty() && object.VersionID == "" && opts.VersionSuspended {
 				opts.VersionID = nullVersionID
 			}
@@ -554,7 +554,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				object.ReplicateDecisionStr = dsc.String()
 			}
 		}
-		if hasLockEnabled {
+		if object.VersionID != "" && hasLockEnabled {
 			if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
 				apiErr := errorCodes.ToAPIErr(apiErrCode)
 				deleteResults[index].errInfo = DeleteError{
@@ -649,6 +649,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			dv := DeletedObjectReplicationInfo{
 				DeletedObject: dobj,
 				Bucket:        bucket,
+				EventType:     ReplicateIncomingDelete,
 			}
 			scheduleReplicationDelete(ctx, dv, objectAPI)
 		}
@@ -864,7 +865,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
+	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -1040,7 +1041,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
+		if crypto.Requested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 			if crypto.SSECopy.IsRequested(r.Header) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
 				return
@@ -1066,7 +1067,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					return
 				}
 			}
-			reader, objectEncryptionKey, err = newEncryptReader(hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
+			reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1115,9 +1116,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		Host:         handlers.GetSourceIP(r),
 	})
 
-	if successRedirect != "" {
-		// Replace raw query params..
-		redirectURL.RawQuery = getRedirectPostRawQuery(objInfo)
+	if redirectURL != nil { // success_action_redirect is valid and set.
+		v := redirectURL.Query()
+		v.Add("bucket", objInfo.Bucket)
+		v.Add("key", objInfo.Name)
+		v.Add("etag", "\""+objInfo.ETag+"\"")
+		redirectURL.RawQuery = v.Encode()
 		writeRedirectSeeOther(w, redirectURL.String())
 		return
 	}
@@ -1385,7 +1389,8 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1399,6 +1404,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 		Bucket:           bucket,
 		ObjectLockConfig: &cfgStr,
+		UpdatedAt:        updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1490,7 +1496,8 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1501,9 +1508,10 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
 	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeTags,
-		Bucket: bucket,
-		Tags:   &cfgStr,
+		Type:      madmin.SRBucketMetaTypeTags,
+		Bucket:    bucket,
+		Tags:      &cfgStr,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1572,14 +1580,16 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypeTags,
-		Bucket: bucket,
+		Type:      madmin.SRBucketMetaTypeTags,
+		Bucket:    bucket,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-lifecycle-handlers.go

@@ -91,7 +91,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -168,7 +168,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-listobjects-handlers.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -101,7 +100,6 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectVersionsInfo.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -167,7 +165,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -246,7 +243,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -347,7 +343,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	if err = DecryptETags(ctx, GlobalKMS, listObjectsInfo.Objects); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -75,28 +75,28 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 
 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
-func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) error {
+func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if globalIsGateway && globalGatewayName != NASBackendGateway {
 		if configFile == bucketPolicyConfig {
 			if configData == nil {
-				return objAPI.DeleteBucketPolicy(ctx, bucket)
+				return updatedAt, objAPI.DeleteBucketPolicy(ctx, bucket)
 			}
 			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
 			if err != nil {
-				return err
+				return updatedAt, err
 			}
-			return objAPI.SetBucketPolicy(ctx, bucket, config)
+			return updatedAt, objAPI.SetBucketPolicy(ctx, bucket, config)
 		}
-		return NotImplemented{}
+		return updatedAt, NotImplemented{}
 	}
 
 	if bucket == minioMetaBucket {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
@@ -105,56 +105,56 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 			// Only single drive mode needs this fallback.
 			meta = newBucketMetadata(bucket)
 		} else {
-			return err
+			return updatedAt, err
 		}
 	}
-
+	updatedAt = UTCNow()
 	switch configFile {
 	case bucketPolicyConfig:
 		meta.PolicyConfigJSON = configData
-		meta.PolicyConfigUpdatedAt = UTCNow()
+		meta.PolicyConfigUpdatedAt = updatedAt
 	case bucketNotificationConfig:
 		meta.NotificationConfigXML = configData
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
-		meta.EncryptionConfigUpdatedAt = UTCNow()
+		meta.EncryptionConfigUpdatedAt = updatedAt
 	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
-		meta.TaggingConfigUpdatedAt = UTCNow()
+		meta.TaggingConfigUpdatedAt = updatedAt
 	case bucketQuotaConfigFile:
 		meta.QuotaConfigJSON = configData
-		meta.QuotaConfigUpdatedAt = UTCNow()
+		meta.QuotaConfigUpdatedAt = updatedAt
 	case objectLockConfig:
 		meta.ObjectLockConfigXML = configData
-		meta.ObjectLockConfigUpdatedAt = UTCNow()
+		meta.ObjectLockConfigUpdatedAt = updatedAt
 	case bucketVersioningConfig:
 		meta.VersioningConfigXML = configData
-		meta.VersioningConfigUpdatedAt = UTCNow()
+		meta.VersioningConfigUpdatedAt = updatedAt
 	case bucketReplicationConfig:
 		meta.ReplicationConfigXML = configData
-		meta.ReplicationConfigUpdatedAt = UTCNow()
+		meta.ReplicationConfigUpdatedAt = updatedAt
 	case bucketTargetsFile:
-		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, kms.Context{
+		meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
 			bucket:            meta.Name,
 			bucketTargetsFile: bucketTargetsFile,
 		})
 		if err != nil {
-			return fmt.Errorf("Error encrypting bucket target metadata %w", err)
+			return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
 		}
 	default:
-		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
+		return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
 
 	if err := meta.Save(ctx, objAPI); err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.Set(bucket, meta)
 	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket) // Do not use caller context here
 
-	return nil
+	return updatedAt, nil
 }
 
 // Get metadata for a bucket.

cmd/bucket-metadata.go

@@ -438,7 +438,7 @@ func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI Obje
 		return nil
 	}
 
-	encBytes, metaBytes, err := encryptBucketMetadata(b.Name, b.BucketTargetsConfigJSON, kms.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
+	encBytes, metaBytes, err := encryptBucketMetadata(ctx, b.Name, b.BucketTargetsConfigJSON, kms.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
 	if err != nil {
 		return err
 	}
@@ -449,14 +449,14 @@ func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI Obje
 }
 
 // encrypt bucket metadata if kms is configured.
-func encryptBucketMetadata(bucket string, input []byte, kmsContext kms.Context) (output, metabytes []byte, err error) {
+func encryptBucketMetadata(ctx context.Context, bucket string, input []byte, kmsContext kms.Context) (output, metabytes []byte, err error) {
 	if GlobalKMS == nil {
 		output = input
 		return
 	}
 
 	metadata := make(map[string]string)
-	key, err := GlobalKMS.GenerateKey("", kmsContext)
+	key, err := GlobalKMS.GenerateKey(ctx, "", kmsContext)
 	if err != nil {
 		return
 	}
@@ -465,7 +465,7 @@ func encryptBucketMetadata(bucket string, input []byte, kmsContext kms.Context)
 	objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader)
 	sealedKey := objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "")
 	crypto.S3.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey)
-	_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.CipherSuitesDARE()})
+	_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
 	if err != nil {
 		return output, metabytes, err
 	}
@@ -495,6 +495,6 @@ func decryptBucketMetadata(input []byte, bucket string, meta map[string]string,
 	}
 
 	outbuf := bytes.NewBuffer(nil)
-	_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.CipherSuitesDARE()})
+	_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
 	return outbuf.Bytes(), err
 }

cmd/bucket-notification-handlers.go

@@ -160,7 +160,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers.go

@@ -103,16 +103,18 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Call site replication hook.
 	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypePolicy,
-		Bucket: bucket,
-		Policy: bucketPolicyBytes,
+		Type:      madmin.SRBucketMetaTypePolicy,
+		Bucket:    bucket,
+		Policy:    bucketPolicyBytes,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -148,15 +150,17 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
 	// Call site replication hook.
 	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-		Type:   madmin.SRBucketMetaTypePolicy,
-		Bucket: bucket,
+		Type:      madmin.SRBucketMetaTypePolicy,
+		Bucket:    bucket,
+		UpdatedAt: updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-replication-handlers.go

@@ -86,7 +86,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
+	if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -165,7 +165,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
 		return
 	}
-	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -202,6 +202,12 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+
+	if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	var usageInfo BucketUsageInfo
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err == nil && !dataUsageInfo.LastUpdate.IsZero() {

cmd/bucket-replication-utils.go

@@ -21,12 +21,14 @@ import (
 	"bytes"
 	"fmt"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
@@ -503,6 +505,8 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 	}
 	var dsc ReplicateDecision
 	var tgtStatuses map[string]replication.StatusType
+	var purgeStatuses map[string]VersionPurgeStatusType
+
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
 			ObjectV: ObjectV{
@@ -516,15 +520,17 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 		}, replication.HealReplicationType, ObjectOptions{}))
 	}
 	tgtStatuses = replicationStatusesMap(oi.ReplicationStatusInternal)
-
+	purgeStatuses = versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
 	existingObjResync := rcfg.Resync(GlobalContext, oi, &dsc, tgtStatuses)
-
+	tm, _ := time.Parse(time.RFC3339Nano, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
 	return ReplicateObjectInfo{
-		ObjectInfo:        oi,
-		OpType:            replication.HealReplicationType,
-		Dsc:               dsc,
-		ExistingObjResync: existingObjResync,
-		TargetStatuses:    tgtStatuses,
+		ObjectInfo:           oi,
+		OpType:               replication.HealReplicationType,
+		Dsc:                  dsc,
+		ExistingObjResync:    existingObjResync,
+		TargetStatuses:       tgtStatuses,
+		TargetPurgeStatuses:  purgeStatuses,
+		ReplicationTimestamp: tm,
 	}
 }
 
@@ -724,3 +730,10 @@ func parseSizeFromContentRange(h http.Header) (sz int64, err error) {
 	}
 	return int64(usz), nil
 }
+
+func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) {
+	opts.Verbose = q.Get("verbose") == "true"
+	opts.ARN = q.Get("arn")
+	opts.Prefix = q.Get("prefix")
+	return
+}

cmd/bucket-replication.go

@@ -347,7 +347,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
 // deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
 // on target.
-func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer, trigger string) {
+func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
 	var replicationStatus replication.StatusType
 	bucket := dobj.Bucket
 	versionID := dobj.DeleteMarkerVersionID
@@ -358,7 +358,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	defer func() {
 		replStatus := string(replicationStatus)
 		auditLogInternal(context.Background(), bucket, dobj.ObjectName, AuditLogOptions{
-			Trigger:   trigger,
+			Event:     dobj.EventType,
 			APIName:   ReplicateDeleteAPI,
 			VersionID: versionID,
 			Status:    replStatus,
@@ -479,6 +479,10 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		eventName = event.ObjectReplicationFailed
 	}
 	drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
+	if replicationStatus != prevStatus {
+		drs.ReplicationTimeStamp = UTCNow()
+	}
+
 	dobjInfo, err := objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
 		VersionID:         versionID,
 		MTime:             dobj.DeleteMarkerMTime.Time,
@@ -857,7 +861,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
-func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer, trigger string) {
+func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) {
 	var replicationStatus replication.StatusType
 	defer func() {
 		if replicationStatus.Empty() {
@@ -868,7 +872,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			replicationStatus = ri.ReplicationStatus
 		}
 		auditLogInternal(ctx, ri.Bucket, ri.Name, AuditLogOptions{
-			Trigger:   trigger,
+			Event:     ri.EventType,
 			APIName:   ReplicateObjectAPI,
 			VersionID: ri.VersionID,
 			Status:    replicationStatus.String(),
@@ -990,6 +994,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	// the target site is down. Leave it to scanner to catch up instead.
 	if rinfos.ReplicationStatus() != replication.Completed && ri.RetryCount < 1 {
 		ri.OpType = replication.HealReplicationType
+		ri.EventType = ReplicateMRF
 		ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal()
 		ri.RetryCount++
 		globalReplicationPool.queueReplicaFailedTask(ri)
@@ -1291,6 +1296,7 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 type DeletedObjectReplicationInfo struct {
 	DeletedObject
 	Bucket    string
+	EventType string
 	OpType    replication.Type
 	ResetID   string
 	TargetArn string
@@ -1313,12 +1319,15 @@ const (
 
 	// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
 	ReplicateMRF = "replicate:mrf"
-	// ReplicateIncoming - audit trail indicating replication started [could be from incoming/existing/heal activity]
+	// ReplicateIncoming - audit trail of inline replication
 	ReplicateIncoming = "replicate:incoming"
+	// ReplicateIncomingDelete - audit trail of inline replication of deletes.
+	ReplicateIncomingDelete = "replicate:incoming:delete"
+
 	// ReplicateHeal - audit trail for healing of failed/pending replications
 	ReplicateHeal = "replicate:heal"
-	// ReplicateDelete - audit trail for delete replication
-	ReplicateDelete = "replicate:delete"
+	// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
+	ReplicateHealDelete = "replicate:heal:delete"
 )
 
 var (
@@ -1364,13 +1373,14 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	pool.ResizeWorkers(opts.Workers)
 	pool.ResizeFailedWorkers(opts.FailedWorkers)
 	go pool.AddExistingObjectReplicateWorker()
-	go pool.periodicResyncMetaSave(ctx, o)
+	go pool.updateResyncStatus(ctx, o)
 	return pool
 }
 
 // AddMRFWorker adds a pending/failed replication worker to handle requests that could not be queued
 // to the other workers
 func (p *ReplicationPool) AddMRFWorker() {
+	defer p.mrfWorkerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1379,7 +1389,7 @@ func (p *ReplicationPool) AddMRFWorker() {
 			if !ok {
 				return
 			}
-			replicateObject(p.ctx, oi, p.objLayer, ReplicateMRF)
+			replicateObject(p.ctx, oi, p.objLayer)
 		case <-p.mrfWorkerKillCh:
 			return
 		}
@@ -1397,12 +1407,12 @@ func (p *ReplicationPool) AddWorker() {
 			if !ok {
 				return
 			}
-			replicateObject(p.ctx, oi, p.objLayer, ReplicateIncoming)
+			replicateObject(p.ctx, oi, p.objLayer)
 		case doi, ok := <-p.replicaDeleteCh:
 			if !ok {
 				return
 			}
-			replicateDelete(p.ctx, doi, p.objLayer, ReplicateDelete)
+			replicateDelete(p.ctx, doi, p.objLayer)
 		case <-p.workerKillCh:
 			return
 		}
@@ -1419,12 +1429,12 @@ func (p *ReplicationPool) AddExistingObjectReplicateWorker() {
 			if !ok {
 				return
 			}
-			replicateObject(p.ctx, oi, p.objLayer, ReplicateExisting)
+			replicateObject(p.ctx, oi, p.objLayer)
 		case doi, ok := <-p.existingReplicaDeleteCh:
 			if !ok {
 				return
 			}
-			replicateDelete(p.ctx, doi, p.objLayer, ReplicateExistingDelete)
+			replicateDelete(p.ctx, doi, p.objLayer)
 		}
 	}
 }
@@ -1482,7 +1492,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication failed workers could not keep up with healing failures - consider increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
 	}
 }
 
@@ -1508,7 +1518,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		})
 	case ch <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1545,7 +1555,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1722,10 +1732,11 @@ func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs
 }
 
 func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, dsc ReplicateDecision, opType replication.Type) {
+	ri := ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType, Dsc: dsc, EventType: ReplicateIncoming}
 	if dsc.Synchronous() {
-		replicateObject(ctx, ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType, Dsc: dsc}, o, ReplicateIncoming)
+		replicateObject(ctx, ri, o)
 	} else {
-		globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType, Dsc: dsc})
+		globalReplicationPool.queueReplicaTask(ri)
 	}
 	if sz, err := objInfo.GetActualSize(); err == nil {
 		for arn := range dsc.targetsMap {
@@ -1956,10 +1967,10 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 	return calculateBucketReplicationStats(bucket, u, bucketStats)
 }
 
-const resyncTimeInterval = time.Minute * 10
+const resyncTimeInterval = time.Minute * 1
 
-// periodicResyncMetaSave saves in-memory resync meta stats to disk in periodic intervals
-func (p *ReplicationPool) periodicResyncMetaSave(ctx context.Context, objectAPI ObjectLayer) {
+// updateResyncStatus persists in-memory resync metadata stats to disk at periodic intervals
+func (p *ReplicationPool) updateResyncStatus(ctx context.Context, objectAPI ObjectLayer) {
 	resyncTimer := time.NewTimer(resyncTimeInterval)
 	defer resyncTimer.Stop()
 
@@ -2085,13 +2096,15 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI
 					DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 					DeleteMarker:          roi.DeleteMarker,
 				},
-				Bucket: roi.Bucket,
-				OpType: replication.ExistingObjectReplicationType,
+				Bucket:    roi.Bucket,
+				OpType:    replication.ExistingObjectReplicationType,
+				EventType: ReplicateExistingDelete,
 			}
-			replicateDelete(ctx, doi, objectAPI, ReplicateDelete)
+			replicateDelete(ctx, doi, objectAPI)
 		} else {
 			roi.OpType = replication.ExistingObjectReplicationType
-			replicateObject(ctx, roi, objectAPI, ReplicateExisting)
+			roi.EventType = ReplicateExisting
+			replicateObject(ctx, roi, objectAPI)
 		}
 		_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, miniogo.StatObjectOptions{
 			VersionID: roi.VersionID,
@@ -2205,18 +2218,21 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo,
 	for index := range buckets {
 		meta, err := loadBucketResyncMetadata(ctx, buckets[index].Name, objAPI)
 		if err != nil {
-			if errors.Is(err, errVolumeNotFound) {
-				meta = newBucketResyncStatus(buckets[index].Name)
-			} else {
+			if !errors.Is(err, errVolumeNotFound) {
 				logger.LogIf(ctx, err)
-				continue
 			}
+			continue
 		}
+		p.resyncState.Lock()
 		p.resyncState.statusMap[buckets[index].Name] = meta
+		p.resyncState.Unlock()
 	}
 	for index := range buckets {
 		bucket := buckets[index].Name
+		p.resyncState.RLock()
 		m, ok := p.resyncState.statusMap[bucket]
+		p.resyncState.RUnlock()
+
 		if ok {
 			for arn, st := range m.TargetsMap {
 				if st.ResyncStatus == ResyncFailed || st.ResyncStatus == ResyncStarted {
@@ -2283,3 +2299,89 @@ func saveResyncStatus(ctx context.Context, bucket string, brs BucketReplicationR
 	configFile := path.Join(bucketMetaPrefix, bucket, replicationDir, resyncFileName)
 	return saveConfig(ctx, objectAPI, configFile, buf)
 }
+
+// getReplicationDiff returns unreplicated objects in a channel
+func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string, opts madmin.ReplDiffOpts) (diffCh chan madmin.DiffInfo, err error) {
+	objInfoCh := make(chan ObjectInfo)
+	if err := objAPI.Walk(ctx, bucket, opts.Prefix, objInfoCh, ObjectOptions{}); err != nil {
+		logger.LogIf(ctx, err)
+		return diffCh, err
+	}
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return diffCh, err
+	}
+	tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return diffCh, err
+	}
+	rcfg := replicationConfig{
+		Config:  cfg,
+		remotes: tgts,
+	}
+	diffCh = make(chan madmin.DiffInfo, 4000)
+	go func() {
+		defer close(diffCh)
+		for obj := range objInfoCh {
+			// Ignore object prefixes which are excluded
+			// from versioning via the MinIO bucket versioning extension.
+			if globalBucketVersioningSys.PrefixSuspended(bucket, obj.Name) {
+				continue
+			}
+			roi := getHealReplicateObjectInfo(obj, rcfg)
+			switch roi.ReplicationStatus {
+			case replication.Completed, replication.Replica:
+				if !opts.Verbose {
+					continue
+				}
+				fallthrough
+			default:
+				// ignore pre-existing objects that don't satisfy replication rule(s)
+				if roi.ReplicationStatus.Empty() && !roi.ExistingObjResync.mustResync() {
+					continue
+				}
+				tgtsMap := make(map[string]madmin.TgtDiffInfo)
+				for arn, st := range roi.TargetStatuses {
+					if opts.ARN == "" || opts.ARN == arn {
+						if !opts.Verbose && (st == replication.Completed || st == replication.Replica) {
+							continue
+						}
+						tgtsMap[arn] = madmin.TgtDiffInfo{
+							ReplicationStatus: st.String(),
+						}
+					}
+				}
+				for arn, st := range roi.TargetPurgeStatuses {
+					if opts.ARN == "" || opts.ARN == arn {
+						if !opts.Verbose && st == Complete {
+							continue
+						}
+						t, ok := tgtsMap[arn]
+						if !ok {
+							t = madmin.TgtDiffInfo{}
+						}
+						t.DeleteReplicationStatus = string(st)
+						tgtsMap[arn] = t
+					}
+				}
+				select {
+				case diffCh <- madmin.DiffInfo{
+					Object:                  obj.Name,
+					VersionID:               obj.VersionID,
+					LastModified:            obj.ModTime,
+					IsDeleteMarker:          obj.DeleteMarker,
+					ReplicationStatus:       string(roi.ReplicationStatus),
+					DeleteReplicationStatus: string(roi.VersionPurgeStatus),
+					ReplicationTimestamp:    roi.ReplicationTimestamp,
+					Targets:                 tgtsMap,
+				}:
+				case <-ctx.Done():
+					return
+				}
+			}
+		}
+	}()
+	return diffCh, nil
+}

cmd/bucket-stats.go

@@ -26,7 +26,7 @@ import (
 // ReplicationLatency holds information of bucket operations latency, such us uploads
 type ReplicationLatency struct {
 	// Single & Multipart PUTs latency
-	UploadHistogram LastMinuteLatencies
+	UploadHistogram LastMinuteHistogram
 }
 
 // Merge two replication latency into a new one
@@ -41,7 +41,7 @@ func (rl ReplicationLatency) getUploadLatency() (ret map[string]uint64) {
 	avg := rl.UploadHistogram.GetAvgData()
 	for k, v := range avg {
 		// Convert nanoseconds to milliseconds
-		ret[sizeTagToString(k)] = v.avg() / uint64(time.Millisecond)
+		ret[sizeTagToString(k)] = uint64(v.avg() / time.Millisecond)
 	}
 	return
 }

cmd/bucket-targets.go

@@ -19,13 +19,12 @@ package cmd
 
 import (
 	"context"
-	"net/http"
 	"sync"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
-	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -273,17 +272,20 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 	}
 	sys.Lock()
 	defer sys.Unlock()
-	if tgts == nil || tgts.Empty() {
-		// remove target and arn association
-		if tgts, ok := sys.targetsMap[bucket]; ok {
-			for _, t := range tgts {
-				if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
-					tgt.healthCancelFn()
-				}
-				delete(sys.arnRemotesMap, t.Arn)
+
+	// Remove existingtarget and arn association
+	if tgts, ok := sys.targetsMap[bucket]; ok {
+		for _, t := range tgts {
+			if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
+				tgt.healthCancelFn()
 			}
+			delete(sys.arnRemotesMap, t.Arn)
 		}
 		delete(sys.targetsMap, bucket)
+	}
+
+	// No need for more if not adding anything
+	if tgts == nil || tgts.Empty() {
 		sys.updateBandwidthLimit(bucket, 0)
 		return
 	}
@@ -325,25 +327,16 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 	sys.targetsMap[bucket.Name] = cfg.Targets
 }
 
-// getRemoteTargetInstanceTransport contains a singleton roundtripper.
-var (
-	getRemoteTargetInstanceTransport     http.RoundTripper
-	getRemoteTargetInstanceTransportOnce sync.Once
-)
-
 // Returns a minio-go Client configured to access remote host described in replication target config.
 func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {
 	config := tcfg.Credentials
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
-	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = NewRemoteTargetHTTPTransport()
-	})
 
 	api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,
 		Region:    tcfg.Region,
-		Transport: getRemoteTargetInstanceTransport,
+		Transport: globalRemoteTargetTransport,
 	})
 	if err != nil {
 		return nil, err

cmd/bucket-versioning-handler.go

@@ -97,7 +97,8 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
+	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -111,6 +112,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,
+		UpdatedAt:  updatedAt,
 	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/callhome.go

@@ -134,3 +134,17 @@ func performCallhome(ctx context.Context) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
 	}
 }
+
+const (
+	callhomeURL    = "https://subnet.min.io/api/callhome"
+	callhomeURLDev = "http://localhost:9000/api/callhome"
+)
+
+func sendCallhomeInfo(ch CallhomeInfo) error {
+	url := callhomeURL
+	if globalIsCICD {
+		url = callhomeURLDev
+	}
+	_, err := globalSubnetConfig.Post(url, ch)
+	return err
+}

cmd/common-main.go

@@ -48,6 +48,7 @@ import (
 	"github.com/inconshreveable/mousetrap"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2"
 	consoleCerts "github.com/minio/console/pkg/certs"
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
@@ -102,6 +103,11 @@ func init() {
 		PersistOnFailure: false,
 	}
 
+	t, _ := minioVersionToReleaseTime(Version)
+	if !t.IsZero() {
+		globalVersionUnix = uint64(t.Unix())
+	}
+
 	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
 
 	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile()
@@ -200,28 +206,6 @@ func minioConfigToConsoleFeatures() {
 	if globalLDAPConfig.Enabled {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
-	// if IDP is enabled, set IDP environment variables
-	if globalOpenIDConfig.ProviderCfgs[config.Default] != nil {
-		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.ProviderCfgs[config.Default].URL.String())
-		os.Setenv("CONSOLE_IDP_CLIENT_ID", globalOpenIDConfig.ProviderCfgs[config.Default].ClientID)
-		os.Setenv("CONSOLE_IDP_SECRET", globalOpenIDConfig.ProviderCfgs[config.Default].ClientSecret)
-		os.Setenv("CONSOLE_IDP_HMAC_SALT", globalDeploymentID)
-		os.Setenv("CONSOLE_IDP_HMAC_PASSPHRASE", globalOpenIDConfig.ProviderCfgs[config.Default].ClientID)
-		os.Setenv("CONSOLE_IDP_SCOPES", strings.Join(globalOpenIDConfig.ProviderCfgs[config.Default].DiscoveryDoc.ScopesSupported, ","))
-		if globalOpenIDConfig.ProviderCfgs[config.Default].ClaimUserinfo {
-			os.Setenv("CONSOLE_IDP_USERINFO", config.EnableOn)
-		}
-		if globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURIDynamic {
-			// Enable dynamic redirect-uri's based on incoming 'host' header,
-			// Overrides any other callback URL.
-			os.Setenv("CONSOLE_IDP_CALLBACK_DYNAMIC", config.EnableOn)
-		}
-		if globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURI != "" {
-			os.Setenv("CONSOLE_IDP_CALLBACK", globalOpenIDConfig.ProviderCfgs[config.Default].RedirectURI)
-		} else {
-			os.Setenv("CONSOLE_IDP_CALLBACK", getConsoleEndpoints()[0]+"/oauth_callback")
-		}
-	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
 	if globalSubnetConfig.License != "" {
@@ -235,6 +219,29 @@ func minioConfigToConsoleFeatures() {
 	}
 }
 
+func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
+	m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))
+	for name, cfg := range globalOpenIDConfig.ProviderCfgs {
+		callback := getConsoleEndpoints()[0] + "/oauth_callback"
+		if cfg.RedirectURI != "" {
+			callback = cfg.RedirectURI
+		}
+		m[name] = consoleoauth2.ProviderConfig{
+			URL:                     cfg.URL.String(),
+			DisplayName:             cfg.DisplayName,
+			ClientID:                cfg.ClientID,
+			ClientSecret:            cfg.ClientSecret,
+			HMACSalt:                globalDeploymentID,
+			HMACPassphrase:          cfg.ClientID,
+			Scopes:                  strings.Join(cfg.DiscoveryDoc.ScopesSupported, ","),
+			Userinfo:                cfg.ClaimUserinfo,
+			RedirectCallbackDynamic: cfg.RedirectURIDynamic,
+			RedirectCallback:        callback,
+		}
+	}
+	return m
+}
+
 func initConsoleServer() (*restapi.Server, error) {
 	// unset all console_ environment variables.
 	for _, cenv := range env.List(consolePrefix) {
@@ -257,7 +264,7 @@ func initConsoleServer() (*restapi.Server, error) {
 		return nil, err
 	}
 
-	api := operations.NewConsoleAPI(swaggerSpec)
+	api := operations.NewConsoleAPI(swaggerSpec, buildOpenIDConsoleConfig())
 
 	if !serverDebugLog {
 		// Disable console logging if server debug log is not enabled
@@ -425,7 +432,6 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 		if err != nil {
 			logger.FatalIf(err, "Unable to get free port for console on the host")
 		}
-		globalMinioConsolePortAuto = true
 		consoleAddr = net.JoinHostPort("", p.String())
 	}
 
@@ -490,6 +496,12 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 			Skip: true,
 		}, nil
 	}
+	if strings.HasPrefix(envEntry, "#") {
+		// Skip commented lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
 	const envSeparator = "="
 	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
 	if len(envTokens) != 2 {
@@ -499,13 +511,6 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 	key := envTokens[0]
 	val := envTokens[1]
 
-	if strings.HasPrefix(key, "#") {
-		// Skip commented lines
-		return envKV{
-			Skip: true,
-		}, nil
-	}
-
 	// Remove quotes from the value if found
 	if len(val) >= 2 {
 		quote := val[0]
@@ -865,7 +870,7 @@ func handleKMSConfig() {
 		// This implicitly checks that we can communicate to KES. We don't treat
 		// a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+		if err = KMS.CreateKey(context.Background(), defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
 		GlobalKMS = KMS

cmd/common-main_test.go

@@ -136,6 +136,7 @@ export MINIO_ROOT_PASSWORD=minio123`,
 		},
 		{
 			`
+# simple comment
 # MINIO_ROOT_USER=minioadmin
 # MINIO_ROOT_PASSWORD=minioadmin
 MINIO_ROOT_USER=minio

cmd/config-current.go

@@ -364,7 +364,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	case config.SubnetSubSys:
-		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default]); err != nil {
+		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], nil); err != nil {
 			return err
 		}
 	case config.CallhomeSubSys:
@@ -599,7 +599,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
 
-	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default])
+	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 	}

cmd/config-encrypted.go

@@ -70,7 +70,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 	}
 
 	if encrypted && GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat()
+		stat, err := GlobalKMS.Stat(ctx)
 		if err != nil {
 			return err
 		}
@@ -139,7 +139,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		return nil
 	}
 	if encrypted && GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat()
+		stat, err := GlobalKMS.Stat(context.Background())
 		if err != nil {
 			return err
 		}

cmd/consolelogger.go

@@ -18,10 +18,11 @@
 package cmd
 
 import (
-	ring "container/ring"
+	"container/ring"
 	"context"
 	"sync"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/logger/target/console"
@@ -71,11 +72,11 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) {
 // HasLogListeners returns true if console log listeners are registered
 // for this node or peers
 func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
-	return sys != nil && sys.pubsub.NumSubscribers() > 0
+	return sys != nil && sys.pubsub.NumSubscribers(madmin.LogMaskAll) > 0
 }
 
 // Subscribe starts console logging for this node.
-func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) error {
+func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan pubsub.Maskable, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry pubsub.Maskable) bool) error {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
 		logger.AddSystemTarget(sys)
@@ -115,8 +116,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
 			}
 		}
 	}
-
-	return sys.pubsub.Subscribe(subCh, doneCh, filter)
+	return sys.pubsub.Subscribe(pubsub.MaskFromMaskable(madmin.LogMaskAll), subCh, doneCh, filter)
 }
 
 // Init if HTTPConsoleLoggerSys is valid, always returns nil right now
@@ -163,9 +163,9 @@ func (sys *HTTPConsoleLoggerSys) Type() types.TargetType {
 
 // Send log message 'e' to console and publish to console
 // log pubsub system
-func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
+func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
 	var lg log.Info
-	switch e := e.(type) {
+	switch e := entry.(type) {
 	case log.Entry:
 		lg = log.Info{Entry: e, NodeName: sys.nodeName}
 	case string:
@@ -179,5 +179,5 @@ func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
 	sys.logBuf = sys.logBuf.Next()
 	sys.Unlock()
 
-	return sys.console.Send(e, string(logger.All))
+	return sys.console.Send(entry, string(logger.All))
 }

cmd/data-scanner-metric.go

@@ -0,0 +1,315 @@
+package cmd
+
+import (
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+)
+
+//go:generate stringer -type=scannerMetric -trimprefix=scannerMetric $GOFILE
+
+type scannerMetric uint8
+
+type scannerMetrics struct {
+	// All fields must be accessed atomically and aligned.
+	operations [scannerMetricLast]uint64
+	latency    [scannerMetricLastRealtime]lockedLastMinuteLatency
+
+	// actions records actions performed.
+	actions        [lifecycle.ActionCount]uint64
+	actionsLatency [lifecycle.ActionCount]lockedLastMinuteLatency
+
+	// currentPaths contains (string,*currentPathTracker) for each disk processing.
+	// Alignment not required.
+	currentPaths sync.Map
+
+	cycleInfoMu sync.Mutex
+	cycleInfo   *currentScannerCycle
+}
+
+var globalScannerMetrics scannerMetrics
+
+const (
+	// START Realtime metrics, that only to records
+	// last minute latencies and total operation count.
+	scannerMetricReadMetadata scannerMetric = iota
+	scannerMetricCheckMissing
+	scannerMetricSaveUsage
+	scannerMetricApplyAll
+	scannerMetricApplyVersion
+	scannerMetricTierObjSweep
+	scannerMetricHealCheck
+	scannerMetricILM
+	scannerMetricCheckReplication
+	scannerMetricYield
+
+	// START Trace metrics:
+	scannerMetricStartTrace
+	scannerMetricScanObject // Scan object. All operations included.
+
+	// END realtime metrics:
+	scannerMetricLastRealtime
+
+	// Trace only metrics:
+	scannerMetricScanFolder     // Scan a folder on disk, recursively.
+	scannerMetricScanCycle      // Full cycle, cluster global
+	scannerMetricScanBucketDisk // Single bucket on one disk
+
+	// Must be last:
+	scannerMetricLast
+)
+
+// log scanner action.
+// Use for s > scannerMetricStartTrace
+func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].add(duration)
+		}
+
+		if s > scannerMetricStartTrace && globalTrace.NumSubscribers(madmin.TraceScanner) > 0 {
+			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " ")))
+		}
+	}
+}
+
+// time a scanner action.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) time(s scannerMetric) func() {
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].add(duration)
+		}
+	}
+}
+
+// timeSize add time and size of a scanner action.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) timeSize(s scannerMetric) func(sz int) {
+	startTime := time.Now()
+	return func(sz int) {
+		duration := time.Since(startTime)
+
+		atomic.AddUint64(&p.operations[s], 1)
+		if s < scannerMetricLastRealtime {
+			p.latency[s].addSize(duration, int64(sz))
+		}
+	}
+}
+
+// incTime will increment time on metric s with a specific duration.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
+	atomic.AddUint64(&p.operations[s], 1)
+	if s < scannerMetricLastRealtime {
+		p.latency[s].add(d)
+	}
+}
+
+func (p *scannerMetrics) incNoTime(s scannerMetric) {
+	atomic.AddUint64(&p.operations[s], 1)
+	if s < scannerMetricLastRealtime {
+		p.latency[s].add(0)
+	}
+}
+
+// timeILM times an ILM action.
+// lifecycle.NoneAction is ignored.
+// Use for s < scannerMetricLastRealtime
+func (p *scannerMetrics) timeILM(a lifecycle.Action) func() {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return func() {}
+	}
+	startTime := time.Now()
+	return func() {
+		duration := time.Since(startTime)
+		atomic.AddUint64(&p.actions[a], 1)
+		p.actionsLatency[a].add(duration)
+	}
+}
+
+type currentPathTracker struct {
+	name *unsafe.Pointer // contains atomically accessed *string
+}
+
+// currentPathUpdater provides a lightweight update function for keeping track of
+// current objects for each disk.
+// Returns a function that can be used to update the current object
+// and a function to call to when processing finished.
+func (p *scannerMetrics) currentPathUpdater(disk, initial string) (update func(path string), done func()) {
+	initialPtr := unsafe.Pointer(&initial)
+	tracker := &currentPathTracker{
+		name: &initialPtr,
+	}
+
+	p.currentPaths.Store(disk, tracker)
+	return func(path string) {
+			atomic.StorePointer(tracker.name, unsafe.Pointer(&path))
+		}, func() {
+			p.currentPaths.Delete(disk)
+		}
+}
+
+// getCurrentPaths returns the paths currently being processed.
+func (p *scannerMetrics) getCurrentPaths() []string {
+	var res []string
+	prefix := globalLocalNodeName + "/"
+	p.currentPaths.Range(func(key, value interface{}) bool {
+		// We are a bit paranoid, but better miss an entry than crash.
+		name, ok := key.(string)
+		if !ok {
+			return true
+		}
+		obj, ok := value.(*currentPathTracker)
+		if !ok {
+			return true
+		}
+		strptr := (*string)(atomic.LoadPointer(obj.name))
+		if strptr != nil {
+			res = append(res, pathJoin(prefix, name, *strptr))
+		}
+		return true
+	})
+	return res
+}
+
+// activeDisks returns the number of currently active disks.
+// (since this is concurrent it may not be 100% reliable)
+func (p *scannerMetrics) activeDisks() int {
+	var i int
+	p.currentPaths.Range(func(k, v interface{}) bool {
+		i++
+		return true
+	})
+	return i
+}
+
+// lifetime returns the lifetime count of the specified metric.
+func (p *scannerMetrics) lifetime(m scannerMetric) uint64 {
+	if m < 0 || m >= scannerMetricLast {
+		return 0
+	}
+	val := atomic.LoadUint64(&p.operations[m])
+	return val
+}
+
+// lastMinute returns the last minute statistics of a metric.
+// m should be < scannerMetricLastRealtime
+func (p *scannerMetrics) lastMinute(m scannerMetric) AccElem {
+	if m < 0 || m >= scannerMetricLastRealtime {
+		return AccElem{}
+	}
+	val := p.latency[m].total()
+	return val
+}
+
+// lifetimeActions returns the lifetime count of the specified ilm metric.
+func (p *scannerMetrics) lifetimeActions(a lifecycle.Action) uint64 {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return 0
+	}
+	val := atomic.LoadUint64(&p.actions[a])
+	return val
+}
+
+// lastMinuteActions returns the last minute statistics of an ilm metric.
+func (p *scannerMetrics) lastMinuteActions(a lifecycle.Action) AccElem {
+	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
+		return AccElem{}
+	}
+	val := p.actionsLatency[a].total()
+	return val
+}
+
+// setCycle updates the current cycle metrics.
+func (p *scannerMetrics) setCycle(c *currentScannerCycle) {
+	if c != nil {
+		c2 := c.clone()
+		c = &c2
+	}
+	p.cycleInfoMu.Lock()
+	p.cycleInfo = c
+	p.cycleInfoMu.Unlock()
+}
+
+// getCycle returns the current cycle metrics.
+// If not nil, the returned value can safely be modified.
+func (p *scannerMetrics) getCycle() *currentScannerCycle {
+	p.cycleInfoMu.Lock()
+	defer p.cycleInfoMu.Unlock()
+	if p.cycleInfo == nil {
+		return nil
+	}
+	c := p.cycleInfo.clone()
+	return &c
+}
+
+func (p *scannerMetrics) report() madmin.ScannerMetrics {
+	var m madmin.ScannerMetrics
+	cycle := p.getCycle()
+	if cycle != nil {
+		m.CurrentCycle = cycle.current
+		m.CyclesCompletedAt = cycle.cycleCompleted
+		m.CurrentStarted = cycle.started
+	}
+	m.CollectedAt = time.Now()
+	m.ActivePaths = p.getCurrentPaths()
+	m.LifeTimeOps = make(map[string]uint64, scannerMetricLast)
+	for i := scannerMetric(0); i < scannerMetricLast; i++ {
+		if n := atomic.LoadUint64(&p.operations[i]); n > 0 {
+			m.LifeTimeOps[i.String()] = n
+		}
+	}
+	if len(m.LifeTimeOps) == 0 {
+		m.LifeTimeOps = nil
+	}
+
+	m.LastMinute.Actions = make(map[string]madmin.TimedAction, scannerMetricLastRealtime)
+	for i := scannerMetric(0); i < scannerMetricLastRealtime; i++ {
+		lm := p.lastMinute(i)
+		if lm.N > 0 {
+			m.LastMinute.Actions[i.String()] = lm.asTimedAction()
+		}
+	}
+	if len(m.LastMinute.Actions) == 0 {
+		m.LastMinute.Actions = nil
+	}
+
+	// ILM
+	m.LifeTimeILM = make(map[string]uint64)
+	for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ {
+		if n := atomic.LoadUint64(&p.actions[i]); n > 0 {
+			m.LifeTimeILM[i.String()] = n
+		}
+	}
+	if len(m.LifeTimeILM) == 0 {
+		m.LifeTimeILM = nil
+	}
+
+	if len(m.LifeTimeILM) > 0 {
+		m.LastMinute.ILM = make(map[string]madmin.TimedAction, len(m.LifeTimeILM))
+		for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ {
+			lm := p.lastMinuteActions(i)
+			if lm.N > 0 {
+				m.LastMinute.ILM[i.String()] = madmin.TimedAction{Count: uint64(lm.N), AccTime: uint64(lm.Total)}
+			}
+		}
+		if len(m.LastMinute.ILM) == 0 {
+			m.LastMinute.ILM = nil
+		}
+	}
+	return m
+}

cmd/data-scanner.go

@@ -31,7 +31,6 @@ import (
 	"path"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/bits-and-blooms/bloom/v3"
@@ -49,12 +48,13 @@ import (
 )
 
 const (
-	dataScannerSleepPerFolder     = time.Millisecond                 // Time to wait between folders.
-	dataUsageUpdateDirCycles      = 16                               // Visit all folders every n cycles.
-	dataScannerCompactLeastObject = 500                              // Compact when there is less than this many objects in a branch.
-	dataScannerCompactAtChildren  = 10000                            // Compact when there are this many children in a branch.
-	dataScannerCompactAtFolders   = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
-	dataScannerStartDelay         = 1 * time.Minute                  // Time to wait on startup and between cycles.
+	dataScannerSleepPerFolder        = time.Millisecond                 // Time to wait between folders.
+	dataUsageUpdateDirCycles         = 16                               // Visit all folders every n cycles.
+	dataScannerCompactLeastObject    = 500                              // Compact when there is less than this many objects in a branch.
+	dataScannerCompactAtChildren     = 10000                            // Compact when there are this many children in a branch.
+	dataScannerCompactAtFolders      = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
+	dataScannerForceCompactAtFolders = 1_000_000                        // Compact when this many subfolders in a single folder (even top level).
+	dataScannerStartDelay            = 1 * time.Minute                  // Time to wait on startup and between cycles.
 
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
@@ -66,7 +66,7 @@ var (
 
 	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper = newDynamicSleeper(10, 10*time.Second)
+	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
 	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
 )
 
@@ -162,35 +162,45 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 	// No unlock for "leader" lock.
 
 	// Load current bloom cycle
-	nextBloomCycle := intDataUpdateTracker.current() + 1
+	var cycleInfo currentScannerCycle
+	cycleInfo.next = intDataUpdateTracker.current() + 1
 
 	buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
-	if len(buf) >= 8 {
-		if err = binary.Read(bytes.NewReader(buf), binary.LittleEndian, &nextBloomCycle); err != nil {
-			logger.LogIf(ctx, err)
-		}
+	if len(buf) == 8 {
+		cycleInfo.next = binary.LittleEndian.Uint64(buf)
+	} else if len(buf) > 8 {
+		cycleInfo.next = binary.LittleEndian.Uint64(buf[:8])
+		buf = buf[8:]
+		_, err = cycleInfo.UnmarshalMsg(buf)
+		logger.LogIf(ctx, err)
 	}
 
 	scannerTimer := time.NewTimer(scannerCycle.Load())
 	defer scannerTimer.Stop()
+	defer globalScannerMetrics.setCycle(nil)
 
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-scannerTimer.C:
-			if intDataUpdateTracker.debug {
-				console.Debugln("starting scanner cycle")
-			}
+			// Reset the timer for next cycle.
+			// If scanner takes longer we start at once.
+			scannerTimer.Reset(scannerCycle.Load())
+
+			stopFn := globalScannerMetrics.log(scannerMetricScanCycle)
+			cycleInfo.current = cycleInfo.next
+			cycleInfo.started = time.Now()
+			globalScannerMetrics.setCycle(&cycleInfo)
 
 			bgHealInfo := readBackgroundHealInfo(ctx, objAPI)
-			scanMode := getCycleScanMode(nextBloomCycle, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
+			scanMode := getCycleScanMode(cycleInfo.current, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
 			if bgHealInfo.CurrentScanMode != scanMode {
 				newHealInfo := bgHealInfo
 				newHealInfo.CurrentScanMode = scanMode
 				if scanMode == madmin.HealDeepScan {
 					newHealInfo.BitrotStartTime = time.Now().UTC()
-					newHealInfo.BitrotStartCycle = nextBloomCycle
+					newHealInfo.BitrotStartCycle = cycleInfo.current
 				}
 				saveBackgroundHealInfo(ctx, objAPI, newHealInfo)
 			}
@@ -198,22 +208,27 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
+			bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle), scanMode)
+			err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
+			stopFn()
 			if err == nil {
 				// Store new cycle...
-				nextBloomCycle++
-				var tmp [8]byte
-				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
-				if err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp[:]); err != nil {
-					logger.LogIf(ctx, err)
+				cycleInfo.next++
+				cycleInfo.current = 0
+				cycleInfo.cycleCompleted = append(cycleInfo.cycleCompleted, time.Now())
+				if len(cycleInfo.cycleCompleted) > dataUsageUpdateDirCycles {
+					cycleInfo.cycleCompleted = cycleInfo.cycleCompleted[len(cycleInfo.cycleCompleted)-dataUsageUpdateDirCycles:]
 				}
+				globalScannerMetrics.setCycle(&cycleInfo)
+				tmp := make([]byte, 8, 8+cycleInfo.Msgsize())
+				// Cycle for backward compat.
+				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
+				tmp, _ = cycleInfo.MarshalMsg(tmp)
+				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
+				logger.LogIf(ctx, err)
 			}
-
-			// Reset the timer for next cycle.
-			scannerTimer.Reset(scannerCycle.Load())
 		}
 	}
 }
@@ -244,24 +259,11 @@ type folderScanner struct {
 	// Will not be closed when returned.
 	updates    chan<- dataUsageEntry
 	lastUpdate time.Time
+
+	// updateCurrentPath should be called whenever a new path is scanned.
+	updateCurrentPath func(string)
 }
 
-type scannerStats struct {
-	// All fields must be accessed atomically and aligned.
-
-	accTotalObjects  uint64
-	accTotalVersions uint64
-	accFolders       uint64
-	bucketsStarted   uint64
-	bucketsFinished  uint64
-	ilmChecks        uint64
-
-	// actions records actions performed.
-	actions [lifecycle.ActionCount]uint64
-}
-
-var globalScannerStats scannerStats
-
 // Cache structure and compaction:
 //
 // A cache structure will be kept with a tree of usages.
@@ -305,24 +307,14 @@ var globalScannerStats scannerStats
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
 func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
-	t := UTCNow()
-
 	logPrefix := color.Green("data-usage: ")
-	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
-	atomic.AddUint64(&globalScannerStats.bucketsStarted, 1)
-	defer func() {
-		atomic.AddUint64(&globalScannerStats.bucketsFinished, 1)
-	}()
-	if intDataUpdateTracker.debug {
-		defer func() {
-			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
-		}()
-	}
 
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
 	}
+	updatePath, closeDisk := globalScannerMetrics.currentPathUpdater(basePath, cache.Info.Name)
+	defer closeDisk()
 
 	s := folderScanner{
 		root:                  basePath,
@@ -335,6 +327,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		healObjectSelect:      0,
 		scanMode:              scanMode,
 		updates:               cache.Info.updates,
+		updateCurrentPath:     updatePath,
 	}
 
 	// Add disks for set healing.
@@ -366,14 +359,8 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 			s.withFilter = nil
 		}
 	}
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Start scanning. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
-	}
 
 	done := ctx.Done()
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
-	}
 
 	// Read top level in bucket.
 	select {
@@ -389,9 +376,6 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		return cache, err
 	}
 
-	if s.dataUsageScannerDebug {
-		console.Debugf(logPrefix+"Finished scanner, %v entries (%+v) %s \n", len(s.newCache.Cache), *s.newCache.sizeRecursive(s.newCache.Info.Name), logSuffix)
-	}
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle = cache.Info.NextCycle
 	return s.newCache, nil
@@ -420,10 +404,10 @@ func (f *folderScanner) sendUpdate() {
 func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, into *dataUsageEntry) error {
 	done := ctx.Done()
 	scannerLogPrefix := color.Green("folder-scanner:")
+
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
-	atomic.AddUint64(&globalScannerStats.accFolders, 1)
 
 	for {
 		select {
@@ -583,9 +567,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		}
 
 		// If we have many subfolders, compact ourself.
-		if !into.Compacted &&
-			f.newCache.Info.Name != folder.name &&
-			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders {
+		shouldCompact := f.newCache.Info.Name != folder.name &&
+			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
+			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
+
+		if !into.Compacted && shouldCompact {
 			into.Compacted = true
 			newFolders = append(newFolders, existingFolders...)
 			existingFolders = nil
@@ -648,7 +634,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					f.updateCache.replaceHashed(h, &thisHash, dataUsageEntry{})
 				}
 			}
+			f.updateCurrentPath(folder.name)
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
+			stopFn()
 			// Add new folders if this is new and we don't have existing.
 			if !into.Compacted {
 				parent := f.updateCache.find(thisHash.Key())
@@ -676,7 +665,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
+			f.updateCurrentPath(folder.name)
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name, "EXISTING")
 			scanFolder(folder)
+			stopFn()
 		}
 
 		// Scan for healing
@@ -717,9 +709,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		healObjectsPrefix := color.Green("healObjects:")
 		for k := range abandonedChildren {
 			bucket, prefix := path2BucketObject(k)
-			if f.dataUsageScannerDebug {
-				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
-			}
+			stopFn := globalScannerMetrics.time(scannerMetricCheckMissing)
+			f.updateCurrentPath(k)
 
 			if bucket != resolver.bucket {
 				// Bucket might be missing as well with abandoned children.
@@ -746,11 +737,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 				},
 				// Some disks have data for this.
-				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-					if f.dataUsageScannerDebug {
-						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
-					}
-
+				partial: func(entries metaCacheEntries, errs []error) {
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						// check if we can get one entry atleast
@@ -807,6 +794,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				},
 			})
 
+			stopFn()
 			if f.dataUsageScannerDebug && err != nil && err != errFileNotFound {
 				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
@@ -814,7 +802,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// Add unless healing returned an error.
 			if foundObjs {
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
+				stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, this.name, "HEALED")
 				scanFolder(this)
+				stopFn()
 			}
 		}
 		break
@@ -964,7 +954,6 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 		return false, size
 	}
 
-	atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
 	action := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi, false)
@@ -975,7 +964,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
 		}
 	}
-	atomic.AddUint64(&globalScannerStats.actions[action], 1)
+	defer globalScannerMetrics.timeILM(action)
 
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
@@ -1092,16 +1081,23 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 // The metadata will be compared to consensus on the object layer before any changes are applied.
 // If no metadata is supplied, -1 is returned if no action is taken.
 func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) int64 {
+	done := globalScannerMetrics.time(scannerMetricILM)
 	applied, size := i.applyLifecycle(ctx, o, oi)
+	done()
+
 	// For instance, an applied lifecycle means we remove/transitioned an object
 	// from the current deployment, which means we don't have to call healing
 	// routine even if we are asked to do via heal flag.
 	if !applied {
 		if i.heal.enabled {
+			done := globalScannerMetrics.time(scannerMetricHealCheck)
 			size = i.applyHealing(ctx, o, oi)
+			done()
 		}
 		// replicate only if lifecycle rules are not applied.
+		done := globalScannerMetrics.time(scannerMetricCheckReplication)
 		i.healReplication(ctx, o, oi.Clone(), sizeS)
+		done()
 	}
 	return size
 }
@@ -1282,12 +1278,14 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 
 	switch oi.ReplicationStatus {
 	case replication.Pending, replication.Failed:
+		roi.EventType = ReplicateHeal
 		globalReplicationPool.queueReplicaTask(roi)
 		return
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
 	if roi.ExistingObjResync.mustResync() {
+		roi.EventType = ReplicateExisting
 		globalReplicationPool.queueReplicaTask(roi)
 	}
 }
@@ -1313,11 +1311,13 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 				DeleteMarker:          roi.DeleteMarker,
 			},
-			Bucket: roi.Bucket,
-			OpType: replication.HealReplicationType,
+			Bucket:    roi.Bucket,
+			OpType:    replication.HealReplicationType,
+			EventType: ReplicateHealDelete,
 		}
 		if roi.ExistingObjResync.mustResync() {
 			doi.OpType = replication.ExistingObjectReplicationType
+			doi.EventType = ReplicateExistingDelete
 			queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
 			return
 		}
@@ -1341,15 +1341,20 @@ type dynamicSleeper struct {
 
 	// cycle will be closed
 	cycle chan struct{}
+
+	// isScanner should be set when this is used by the scanner
+	// to record metrics.
+	isScanner bool
 }
 
 // newDynamicSleeper
-func newDynamicSleeper(factor float64, maxWait time.Duration) *dynamicSleeper {
+func newDynamicSleeper(factor float64, maxWait time.Duration, isScanner bool) *dynamicSleeper {
 	return &dynamicSleeper{
-		factor:   factor,
-		cycle:    make(chan struct{}),
-		maxSleep: maxWait,
-		minSleep: 100 * time.Microsecond,
+		factor:    factor,
+		cycle:     make(chan struct{}),
+		maxSleep:  maxWait,
+		minSleep:  100 * time.Microsecond,
+		isScanner: isScanner,
 	}
 }
 
@@ -1379,15 +1384,24 @@ func (d *dynamicSleeper) Timer(ctx context.Context) func() {
 			select {
 			case <-ctx.Done():
 				if !timer.Stop() {
+					if d.isScanner {
+						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+					}
 					<-timer.C
 				}
 				return
 			case <-timer.C:
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 				return
 			case <-cycle:
 				if !timer.Stop() {
 					// We expired.
 					<-timer.C
+					if d.isScanner {
+						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+					}
 					return
 				}
 			}
@@ -1418,14 +1432,23 @@ func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) {
 		case <-ctx.Done():
 			if !timer.Stop() {
 				<-timer.C
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 			}
 			return
 		case <-timer.C:
+			if d.isScanner {
+				globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+			}
 			return
 		case <-cycle:
 			if !timer.Stop() {
 				// We expired.
 				<-timer.C
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 				return
 			}
 		}
@@ -1457,9 +1480,9 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
 	var apiName string
-	switch trigger {
+	switch event {
 	case ILMExpiry:
 		apiName = "ILMExpiry"
 	case ILMFreeVersionDelete:
@@ -1468,7 +1491,7 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
 		apiName = "ILMTransition"
 	}
 	auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
-		Trigger:   trigger,
+		Event:     event,
 		APIName:   apiName,
 		VersionID: oi.VersionID,
 	})

cmd/data-update-tracker_test.go

@@ -63,7 +63,7 @@ func (t *testingLogger) Type() types.TargetType {
 	return types.TargetHTTP
 }
 
-func (t *testingLogger) Send(entry interface{}, errKind string) error {
+func (t *testingLogger) Send(entry interface{}) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.t == nil {
@@ -75,7 +75,7 @@ func (t *testingLogger) Send(entry interface{}, errKind string) error {
 	}
 
 	t.t.Helper()
-	t.t.Log(e.Level, ":", errKind, e.Message)
+	t.t.Log(e.Level, ":", e.Message)
 	return nil
 }
 

cmd/data-usage-cache.go

@@ -1323,3 +1323,19 @@ func (z dataUsageHashMap) Msgsize() (s int) {
 	}
 	return
 }
+
+//msgp:encode ignore currentScannerCycle
+//msgp:decode ignore currentScannerCycle
+
+type currentScannerCycle struct {
+	current        uint64
+	next           uint64
+	started        time.Time
+	cycleCompleted []time.Time
+}
+
+// clone returns a clone.
+func (z currentScannerCycle) clone() currentScannerCycle {
+	z.cycleCompleted = append(make([]time.Time, 0, len(z.cycleCompleted)), z.cycleCompleted...)
+	return z
+}

cmd/data-usage-cache_gen.go

@@ -3,6 +3,8 @@ package cmd
 // Code generated by github.com/tinylib/msgp DO NOT EDIT.
 
 import (
+	"time"
+
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -284,6 +286,101 @@ func (z *allTierStats) Msgsize() (s int) {
 	return
 }
 
+// MarshalMsg implements msgp.Marshaler
+func (z *currentScannerCycle) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 4
+	// string "current"
+	o = append(o, 0x84, 0xa7, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74)
+	o = msgp.AppendUint64(o, z.current)
+	// string "next"
+	o = append(o, 0xa4, 0x6e, 0x65, 0x78, 0x74)
+	o = msgp.AppendUint64(o, z.next)
+	// string "started"
+	o = append(o, 0xa7, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
+	o = msgp.AppendTime(o, z.started)
+	// string "cycleCompleted"
+	o = append(o, 0xae, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.cycleCompleted)))
+	for za0001 := range z.cycleCompleted {
+		o = msgp.AppendTime(o, z.cycleCompleted[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *currentScannerCycle) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "current":
+			z.current, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "current")
+				return
+			}
+		case "next":
+			z.next, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "next")
+				return
+			}
+		case "started":
+			z.started, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "started")
+				return
+			}
+		case "cycleCompleted":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "cycleCompleted")
+				return
+			}
+			if cap(z.cycleCompleted) >= int(zb0002) {
+				z.cycleCompleted = (z.cycleCompleted)[:zb0002]
+			} else {
+				z.cycleCompleted = make([]time.Time, zb0002)
+			}
+			for za0001 := range z.cycleCompleted {
+				z.cycleCompleted[za0001], bts, err = msgp.ReadTimeBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "cycleCompleted", za0001)
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *currentScannerCycle) Msgsize() (s int) {
+	s = 1 + 8 + msgp.Uint64Size + 5 + msgp.Uint64Size + 8 + msgp.TimeSize + 15 + msgp.ArrayHeaderSize + (len(z.cycleCompleted) * (msgp.TimeSize))
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/data-usage-cache_gen_test.go

@@ -122,6 +122,64 @@ func BenchmarkDecodeallTierStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalcurrentScannerCycle(t *testing.T) {
+	v := currentScannerCycle{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgcurrentScannerCycle(b *testing.B) {
+	v := currentScannerCycle{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgcurrentScannerCycle(b *testing.B) {
+	v := currentScannerCycle{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalcurrentScannerCycle(b *testing.B) {
+	v := currentScannerCycle{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaldataUsageCache(t *testing.T) {
 	v := dataUsageCache{}
 	bts, err := v.MarshalMsg(nil)

cmd/disk-cache-backend.go

@@ -797,25 +797,25 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 	return bytesWritten, base64.StdEncoding.EncodeToString(md5sumCurr), nil
 }
 
-func newCacheEncryptReader(content io.Reader, bucket, object string, metadata map[string]string) (r io.Reader, err error) {
-	objectEncryptionKey, err := newCacheEncryptMetadata(bucket, object, metadata)
+func newCacheEncryptReader(ctx context.Context, content io.Reader, bucket, object string, metadata map[string]string) (r io.Reader, err error) {
+	objectEncryptionKey, err := newCacheEncryptMetadata(ctx, bucket, object, metadata)
 	if err != nil {
 		return nil, err
 	}
 
-	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey, MinVersion: sio.Version20, CipherSuites: fips.CipherSuitesDARE()})
+	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey, MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
 	if err != nil {
 		return nil, crypto.ErrInvalidCustomerKey
 	}
 	return reader, nil
 }
 
-func newCacheEncryptMetadata(bucket, object string, metadata map[string]string) ([]byte, error) {
+func newCacheEncryptMetadata(ctx context.Context, bucket, object string, metadata map[string]string) ([]byte, error) {
 	var sealedKey crypto.SealedKey
 	if globalCacheKMS == nil {
 		return nil, errKMSNotConfigured
 	}
-	key, err := globalCacheKMS.GenerateKey("", kms.Context{bucket: pathJoin(bucket, object)})
+	key, err := globalCacheKMS.GenerateKey(ctx, "", kms.Context{bucket: pathJoin(bucket, object)})
 	if err != nil {
 		return nil, err
 	}
@@ -891,7 +891,7 @@ func (c *diskCache) put(ctx context.Context, bucket, object string, data io.Read
 	reader := data
 	actualSize := uint64(size)
 	if globalCacheKMS != nil {
-		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
+		reader, err = newCacheEncryptReader(ctx, data, bucket, object, metadata)
 		if err != nil {
 			removeAll(cachePath)
 			return oi, err
@@ -948,7 +948,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 	// objSize is the actual size of object (with encryption overhead if any)
 	objSize := uint64(size)
 	if globalCacheKMS != nil {
-		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
+		reader, err = newCacheEncryptReader(ctx, data, bucket, object, metadata)
 		if err != nil {
 			return err
 		}
@@ -1301,7 +1301,7 @@ func (c *diskCache) NewMultipartUpload(ctx context.Context, bucket, object, uID
 	m.Stat.ModTime = UTCNow()
 	if globalCacheKMS != nil {
 		m.Meta[ReservedMetadataPrefix+"Encrypted-Multipart"] = ""
-		if _, err := newCacheEncryptMetadata(bucket, object, m.Meta); err != nil {
+		if _, err := newCacheEncryptMetadata(ctx, bucket, object, m.Meta); err != nil {
 			return uploadID, err
 		}
 	}
@@ -1454,7 +1454,7 @@ func newCachePartEncryptReader(ctx context.Context, bucket, object string, partI
 		return nil, err
 	}
 
-	reader, err := sio.EncryptReader(content, sio.Config{Key: partEncryptionKey[:], MinVersion: sio.Version20, CipherSuites: fips.CipherSuitesDARE()})
+	reader, err := sio.EncryptReader(content, sio.Config{Key: partEncryptionKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
 	if err != nil {
 		return nil, crypto.ErrInvalidCustomerKey
 	}

cmd/encryption-v1.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"crypto/hmac"
 	"crypto/rand"
-	"crypto/sha256"
 	"crypto/subtle"
 	"encoding/binary"
 	"encoding/hex"
@@ -38,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -255,7 +255,7 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
 }
 
 // This function rotates old to new key.
-func rotateKey(oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, ctx kms.Context) error {
+func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) error {
 	kind, _ := crypto.IsEncrypted(metadata)
 	switch kind {
 	case crypto.S3:
@@ -275,7 +275,7 @@ func rotateKey(oldKey []byte, newKeyID string, newKey []byte, bucket, object str
 			return err
 		}
 
-		newKey, err := GlobalKMS.GenerateKey("", kms.Context{bucket: path.Join(bucket, object)})
+		newKey, err := GlobalKMS.GenerateKey(ctx, "", kms.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return err
 		}
@@ -291,8 +291,8 @@ func rotateKey(oldKey []byte, newKeyID string, newKey []byte, bucket, object str
 			return err
 		}
 
-		if len(ctx) == 0 {
-			_, _, _, ctx, err = crypto.S3KMS.ParseMetadata(metadata)
+		if len(cryptoCtx) == 0 {
+			_, _, _, cryptoCtx, err = crypto.S3KMS.ParseMetadata(metadata)
 			if err != nil {
 				return err
 			}
@@ -305,19 +305,19 @@ func rotateKey(oldKey []byte, newKeyID string, newKey []byte, bucket, object str
 		// of the client provided context and add the bucket
 		// key, if not present.
 		kmsCtx := kms.Context{}
-		for k, v := range ctx {
+		for k, v := range cryptoCtx {
 			kmsCtx[k] = v
 		}
 		if _, ok := kmsCtx[bucket]; !ok {
 			kmsCtx[bucket] = path.Join(bucket, object)
 		}
-		newKey, err := GlobalKMS.GenerateKey(newKeyID, kmsCtx)
+		newKey, err := GlobalKMS.GenerateKey(ctx, newKeyID, kmsCtx)
 		if err != nil {
 			return err
 		}
 
 		sealedKey := objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)
-		crypto.S3KMS.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey, ctx)
+		crypto.S3KMS.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey, cryptoCtx)
 		return nil
 	case crypto.SSEC:
 		sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
@@ -344,14 +344,14 @@ func rotateKey(oldKey []byte, newKeyID string, newKey []byte, bucket, object str
 	}
 }
 
-func newEncryptMetadata(kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, ctx kms.Context) (crypto.ObjectKey, error) {
+func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) {
 	var sealedKey crypto.SealedKey
 	switch kind {
 	case crypto.S3:
 		if GlobalKMS == nil {
 			return crypto.ObjectKey{}, errKMSNotConfigured
 		}
-		key, err := GlobalKMS.GenerateKey("", kms.Context{bucket: path.Join(bucket, object)})
+		key, err := GlobalKMS.GenerateKey(ctx, "", kms.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return crypto.ObjectKey{}, err
 		}
@@ -372,13 +372,13 @@ func newEncryptMetadata(kind crypto.Type, keyID string, key []byte, bucket, obje
 		// of the client provided context and add the bucket
 		// key, if not present.
 		kmsCtx := kms.Context{}
-		for k, v := range ctx {
+		for k, v := range cryptoCtx {
 			kmsCtx[k] = v
 		}
 		if _, ok := kmsCtx[bucket]; !ok {
 			kmsCtx[bucket] = path.Join(bucket, object)
 		}
-		key, err := GlobalKMS.GenerateKey(keyID, kmsCtx)
+		key, err := GlobalKMS.GenerateKey(ctx, keyID, kmsCtx)
 		if err != nil {
 			if errors.Is(err, kes.ErrKeyNotFound) {
 				return crypto.ObjectKey{}, errKMSKeyNotFound
@@ -388,7 +388,7 @@ func newEncryptMetadata(kind crypto.Type, keyID string, key []byte, bucket, obje
 
 		objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader)
 		sealedKey = objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)
-		crypto.S3KMS.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey, ctx)
+		crypto.S3KMS.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey, cryptoCtx)
 		return objectKey, nil
 	case crypto.SSEC:
 		objectKey := crypto.GenerateKey(key, rand.Reader)
@@ -400,13 +400,13 @@ func newEncryptMetadata(kind crypto.Type, keyID string, key []byte, bucket, obje
 	}
 }
 
-func newEncryptReader(content io.Reader, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, ctx kms.Context) (io.Reader, crypto.ObjectKey, error) {
-	objectEncryptionKey, err := newEncryptMetadata(kind, keyID, key, bucket, object, metadata, ctx)
+func newEncryptReader(ctx context.Context, content io.Reader, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (io.Reader, crypto.ObjectKey, error) {
+	objectEncryptionKey, err := newEncryptMetadata(ctx, kind, keyID, key, bucket, object, metadata, cryptoCtx)
 	if err != nil {
 		return nil, crypto.ObjectKey{}, err
 	}
 
-	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20, CipherSuites: fips.CipherSuitesDARE()})
+	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20, CipherSuites: fips.DARECiphers()})
 	if err != nil {
 		return nil, crypto.ObjectKey{}, crypto.ErrInvalidCustomerKey
 	}
@@ -418,9 +418,9 @@ func newEncryptReader(content io.Reader, kind crypto.Type, keyID string, key []b
 // SSE-S3
 func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[string]string) (err error) {
 	var (
-		key   []byte
-		keyID string
-		ctx   kms.Context
+		key    []byte
+		keyID  string
+		kmsCtx kms.Context
 	)
 	kind, _ := crypto.IsRequested(r.Header)
 	switch kind {
@@ -430,12 +430,12 @@ func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[
 			return err
 		}
 	case crypto.S3KMS:
-		keyID, ctx, err = crypto.S3KMS.ParseHTTP(r.Header)
+		keyID, kmsCtx, err = crypto.S3KMS.ParseHTTP(r.Header)
 		if err != nil {
 			return err
 		}
 	}
-	_, err = newEncryptMetadata(kind, keyID, key, bucket, object, metadata, ctx)
+	_, err = newEncryptMetadata(r.Context(), kind, keyID, key, bucket, object, metadata, kmsCtx)
 	return
 }
 
@@ -468,7 +468,7 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
 			return nil, crypto.ObjectKey{}, err
 		}
 	}
-	return newEncryptReader(content, kind, keyID, key, bucket, object, metadata, ctx)
+	return newEncryptReader(r.Context(), content, kind, keyID, key, bucket, object, metadata, ctx)
 }
 
 func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
@@ -553,7 +553,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
 	reader, err := sio.DecryptReader(client, sio.Config{
 		Key:            objectEncryptionKey,
 		SequenceNumber: seqNumber,
-		CipherSuites:   fips.CipherSuitesDARE(),
+		CipherSuites:   fips.DARECiphers(),
 	})
 	if err != nil {
 		return nil, crypto.ErrInvalidCustomerKey

cmd/endpoint-ellipses.go

@@ -42,7 +42,7 @@ type endpointSet struct {
 
 // Supported set sizes this is used to find the optimal
 // single set size.
-var setSizes = []uint64{4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+var setSizes = []uint64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
 
 // getDivisibleSize - returns a greatest common divisor of
 // all the ellipses sizes.

cmd/endpoint-ellipses_test.go

@@ -201,18 +201,6 @@ func TestGetSetIndexes(t *testing.T) {
 		success    bool
 	}{
 		// Invalid inputs.
-		{
-			[]string{"data{1...3}"},
-			[]uint64{3},
-			nil,
-			false,
-		},
-		{
-			[]string{"data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"},
-			[]uint64{2, 4, 8},
-			nil,
-			false,
-		},
 		{
 			[]string{"data{1...17}/export{1...52}"},
 			[]uint64{14144},
@@ -220,6 +208,18 @@ func TestGetSetIndexes(t *testing.T) {
 			false,
 		},
 		// Valid inputs.
+		{
+			[]string{"data{1...3}"},
+			[]uint64{3},
+			[][]uint64{{3}},
+			true,
+		},
+		{
+			[]string{"data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"},
+			[]uint64{2, 4, 8},
+			[][]uint64{{2}, {2, 2}, {2, 2, 2, 2}},
+			true,
+		},
 		{
 			[]string{"data{1...27}"},
 			[]uint64{27},

cmd/erasure-common.go

@@ -19,7 +19,11 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"sync"
+
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
@@ -119,3 +123,84 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 	// Return disks which have maximum disk usage common.
 	return newDisks[max]
 }
+
+// readMultipleFiles Reads raw data from all specified files from all disks.
+func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultipleReq, readQuorum int) ([]ReadMultipleResp, error) {
+	resps := make([]chan ReadMultipleResp, len(disks))
+	for i := range resps {
+		resps[i] = make(chan ReadMultipleResp, len(req.Files))
+	}
+	g := errgroup.WithNErrs(len(disks))
+	// Read files in parallel across disks.
+	for index := range disks {
+		index := index
+		g.Go(func() (err error) {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			return disks[index].ReadMultiple(ctx, req, resps[index])
+		}, index)
+	}
+
+	dataArray := make([]ReadMultipleResp, 0, len(req.Files))
+	// Merge results. They should come in order from each.
+	for _, wantFile := range req.Files {
+		quorum := 0
+		toAdd := ReadMultipleResp{
+			Bucket: req.Bucket,
+			Prefix: req.Prefix,
+			File:   wantFile,
+		}
+		for i := range resps {
+			if disks[i] == nil {
+				continue
+			}
+			select {
+			case <-ctx.Done():
+			case gotFile, ok := <-resps[i]:
+				if !ok {
+					continue
+				}
+				if gotFile.Error != "" || !gotFile.Exists {
+					continue
+				}
+				if gotFile.File != wantFile || gotFile.Bucket != req.Bucket || gotFile.Prefix != req.Prefix {
+					continue
+				}
+				quorum++
+				if toAdd.Modtime.After(gotFile.Modtime) || len(gotFile.Data) < len(toAdd.Data) {
+					// Pick latest, or largest to avoid possible truncated entries.
+					continue
+				}
+				toAdd = gotFile
+			}
+		}
+		if quorum < readQuorum {
+			toAdd.Exists = false
+			toAdd.Error = errErasureReadQuorum.Error()
+			toAdd.Data = nil
+		}
+		dataArray = append(dataArray, toAdd)
+	}
+
+	errs := g.Wait()
+	for index, err := range errs {
+		if err == nil {
+			continue
+		}
+		if !IsErr(err, []error{
+			errFileNotFound,
+			errVolumeNotFound,
+			errFileVersionNotFound,
+			errDiskNotFound,
+			errUnformattedDisk,
+		}...) {
+			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
+				disks[index], req.Bucket, req.Prefix, err),
+				disks[index].String())
+		}
+	}
+
+	// Return all the metadata.
+	return dataArray, nil
+}

cmd/erasure-encode_test.go

@@ -196,7 +196,10 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 			if disk == OfflineDisk {
 				continue
 			}
-			disk.Delete(context.Background(), "testbucket", "object", false)
+			disk.Delete(context.Background(), "testbucket", "object", DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)

cmd/erasure-healing-common.go

@@ -180,9 +180,15 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error)
 }
 
 // Returns the latest updated FileInfo files and error in case of failure.
-func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []error) (FileInfo, error) {
+func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultParityCount int, errs []error) (FileInfo, error) {
 	// There should be atleast half correct entries, if not return failure
-	reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, len(partsMetadata)/2)
+	expectedRQuorum := len(partsMetadata) / 2
+	if defaultParityCount == 0 {
+		// if parity count is '0', we expected all entries to be present.
+		expectedRQuorum = len(partsMetadata)
+	}
+
+	reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, expectedRQuorum)
 	if reducedErr != nil {
 		return FileInfo{}, reducedErr
 	}

cmd/erasure-healing-common_test.go

@@ -194,7 +194,7 @@ func TestListOnlineDisks(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -218,7 +218,10 @@ func TestListOnlineDisks(t *testing.T) {
 					// and check if that disk
 					// appears in outDatedDisks.
 					tamperedIndex = index
-					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+						Recursive: false,
+						Force:     false,
+					})
 					if dErr != nil {
 						t.Fatalf("Failed to delete %s - %v", filepath.Join(object, "part.1"), dErr)
 					}
@@ -367,7 +370,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			_, err = getLatestFileInfo(ctx, partsMetadata, errs)
+			_, err = getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -395,7 +398,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 					// and check if that disk
 					// appears in outDatedDisks.
 					tamperedIndex = index
-					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+					dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+						Recursive: false,
+						Force:     false,
+					})
 					if dErr != nil {
 						t.Fatalf("Failed to delete %s - %v", pathJoin(object, xlStorageFormatFile), dErr)
 					}
@@ -423,7 +429,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 
 			}
 			partsMetadata, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if !errors.Is(err, errErasureReadQuorum) {
 				t.Fatalf("Failed to getLatestFileInfo, expected %v, got %v", errErasureReadQuorum, err)
 			}
@@ -489,7 +495,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to read xl meta data %v", err)
 	}
 
-	fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+	fi, err := getLatestFileInfo(ctx, partsMetadata, s.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to get quorum consistent fileInfo %v", err)
 	}

cmd/erasure-healing.go

@@ -500,7 +500,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ {
 			partSize := latestMeta.Parts[partIndex].Size
 			partActualSize := latestMeta.Parts[partIndex].ActualSize
+			partModTime := latestMeta.Parts[partIndex].ModTime
 			partNumber := latestMeta.Parts[partIndex].Number
+			partIdx := latestMeta.Parts[partIndex].Index
 			tillOffset := erasure.ShardFileOffset(0, partSize, partSize)
 			readers := make([]io.ReaderAt, len(latestDisks))
 			checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm
@@ -550,7 +552,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 
 				partsMetadata[i].DataDir = dstDataDir
-				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize)
+				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize, partModTime, partIdx)
 				partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 					PartNumber: partNumber,
 					Algorithm:  checksumAlgo,
@@ -655,7 +657,10 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 				wg.Add(1)
 				go func(index int, disk StorageAPI) {
 					defer wg.Done()
-					_ = disk.Delete(ctx, bucket, object, false)
+					_ = disk.Delete(ctx, bucket, object, DeleteOptions{
+						Recursive: false,
+						Force:     false,
+					})
 				}(index, disk)
 			}
 			wg.Wait()

cmd/erasure-healing_test.go

@@ -580,7 +580,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 
 	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
 	if err != nil {
-		t.Fatalf("Failed to complete multipart upload - %v", err)
+		t.Fatalf("Failed to complete multipart upload - got: %v", err)
 	}
 
 	cfgFile := pathJoin(bucketMetaPrefix, bucket, ".test.bin")
@@ -601,7 +601,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 		erasureDisks := er.getDisks()
 
 		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 		if errors.Is(err, errFileNotFound) {
 			continue
 		}
@@ -610,7 +610,10 @@ func TestHealCorrectQuorum(t *testing.T) {
 		}
 
 		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
-			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 		}
 
 		// Try healing now, it should heal the content properly.
@@ -625,7 +628,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 		}
 
 		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false)
-		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+		nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 		if errors.Is(err, errFileNotFound) {
 			continue
 		}
@@ -634,7 +637,10 @@ func TestHealCorrectQuorum(t *testing.T) {
 		}
 
 		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
-			erasureDisks[i].Delete(context.Background(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), false)
+			erasureDisks[i].Delete(context.Background(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 		}
 
 		// Try healing now, it should heal the content properly.
@@ -714,7 +720,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	er := set.sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -725,7 +734,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 
 	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -734,7 +743,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
@@ -750,7 +762,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -761,7 +773,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
@@ -778,7 +793,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -792,7 +807,10 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	// Test 4: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
 	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		})
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -879,12 +897,15 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 
 	// Test 1: Remove the object backend files from the first disk.
 	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -899,7 +920,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi1, err := getLatestFileInfo(ctx, fileInfos, errs)
+	nfi1, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -922,7 +943,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi2, err := getLatestFileInfo(ctx, fileInfos, errs)
+	nfi2, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -936,7 +957,10 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	// Test 3: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
 	for i := 0; i <= nfi2.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{
+			Recursive: false,
+			Force:     false,
+		})
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -1017,7 +1041,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	secondDisk := erasureDisks[1]
 
 	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -1033,7 +1057,10 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	}
 
 	// Test 1, remove part.1
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -1078,7 +1105,10 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	err = secondDisk.Delete(context.Background(), bucket, object, true)
+	err = secondDisk.Delete(context.Background(), bucket, object, DeleteOptions{
+		Recursive: true,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -1166,7 +1196,10 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	// Delete the whole object folder
-	err = firstDisk.Delete(context.Background(), bucket, object, true)
+	err = firstDisk.Delete(context.Background(), bucket, object, DeleteOptions{
+		Recursive: true,
+		Force:     false,
+	})
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}

cmd/erasure-metadata-utils.go

@@ -151,6 +151,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 			errVolumeNotFound,
 			errFileVersionNotFound,
 			errDiskNotFound,
+			errUnformattedDisk,
 		}...) {
 			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], bucket, object, err),

cmd/erasure-metadata.go

@@ -237,12 +237,14 @@ func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
 }
 
 // AddObjectPart - add a new object part in order.
-func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize int64, actualSize int64) {
+func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, actualSize int64, modTime time.Time, idx []byte) {
 	partInfo := ObjectPartInfo{
 		Number:     partNumber,
 		ETag:       partETag,
 		Size:       partSize,
 		ActualSize: actualSize,
+		ModTime:    modTime,
+		Index:      idx,
 	}
 
 	// Update part info if it already exists.
@@ -398,17 +400,17 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 // readQuorum is the min required disks to read data.
 // writeQuorum is the min required disks to write data.
 func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
-	if defaultParityCount == 0 {
-		return 1, 1, nil
-	}
-
 	// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
-	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs)
+	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, defaultParityCount, errs)
 	if err != nil {
 		return 0, 0, err
 	}
 
 	if latestFileInfo.Deleted {
+		// special case when parity is '0'
+		if defaultParityCount == 0 {
+			return len(partsMetaData), len(partsMetaData), nil
+		}
 		// For delete markers do not use 'defaultParityCount' as it is not expected to be the case.
 		// Use maximum allowed read quorum instead, writeQuorum+1 is returned for compatibility sake
 		// but there are no callers that shall be using this.
@@ -417,7 +419,7 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 	}
 
 	parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass])
-	if parityBlocks <= 0 {
+	if parityBlocks < 0 {
 		parityBlocks = defaultParityCount
 	}
 

cmd/erasure-metadata_test.go

@@ -23,7 +23,7 @@ import (
 	"testing"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 const ActualSize = 1000
@@ -58,7 +58,7 @@ func TestAddObjectPart(t *testing.T) {
 	for _, testCase := range testCases {
 		if testCase.expectedIndex > -1 {
 			partNumString := strconv.Itoa(testCase.partNum)
-			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil)
 		}
 
 		if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
@@ -91,7 +91,7 @@ func TestObjectPartIndex(t *testing.T) {
 	// Add some parts for testing.
 	for _, testCase := range testCases {
 		partNumString := strconv.Itoa(testCase.partNum)
-		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize)
+		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil)
 	}
 
 	// Add failure test case.
@@ -121,7 +121,7 @@ func TestObjectToPartOffset(t *testing.T) {
 	// Total size of all parts is 5,242,899 bytes.
 	for _, partNum := range []int{1, 2, 4, 5, 7} {
 		partNumString := strconv.Itoa(partNum)
-		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize)
+		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize, UTCNow(), nil)
 	}
 
 	testCases := []struct {
@@ -160,7 +160,7 @@ func TestObjectToPartOffset(t *testing.T) {
 func TestFindFileInfoInQuorum(t *testing.T) {
 	getNFInfo := func(n int, quorum int, t int64, dataDir string) []FileInfo {
 		fi := newFileInfo("test", 8, 8)
-		fi.AddObjectPart(1, "etag", 100, 100)
+		fi.AddObjectPart(1, "etag", 100, 100, UTCNow(), nil)
 		fi.ModTime = time.Unix(t, 0)
 		fi.DataDir = dataDir
 		fis := make([]FileInfo, n)

cmd/erasure-multipart.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -48,7 +49,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
 // checkUploadIDExists - verify if a given uploadID exists and is valid.
 func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string) (err error) {
 	defer func() {
-		if err == errFileNotFound {
+		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
 			err = errUploadIDNotFound
 		}
 	}()
@@ -75,6 +76,30 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	return err
 }
 
+// Removes part.meta given by partName belonging to a mulitpart upload from minioMetaBucket
+func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string, partNumber int) {
+	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
+	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
+	storageDisks := er.getDisks()
+
+	g := errgroup.WithNErrs(len(storageDisks))
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
+
+			return nil
+		}, index)
+	}
+	g.Wait()
+}
+
 // Removes part given by partName belonging to a mulitpart upload from minioMetaBucket
 func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir string, partNumber int) {
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
@@ -91,7 +116,15 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, false)
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
+
 			return nil
 		}, index)
 	}
@@ -138,7 +171,10 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 		wg.Add(1)
 		go func(disk StorageAPI) {
 			defer wg.Done()
-			disk.Delete(ctx, bucket, prefix, true)
+			disk.Delete(ctx, bucket, prefix, DeleteOptions{
+				Recursive: true,
+				Force:     false,
+			})
 		}(disk)
 	}
 	wg.Wait()
@@ -202,10 +238,10 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 	for _, disk = range er.getLoadBalancedDisks(true) {
 		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
-			if err == errDiskNotFound {
+			if errors.Is(err, errDiskNotFound) {
 				continue
 			}
-			if err == errFileNotFound {
+			if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
 				return result, nil
 			}
 			logger.LogIf(ctx, err)
@@ -230,7 +266,11 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		}
 		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "", false)
 		if err != nil {
-			return result, toObjectErr(err, bucket, object)
+			if !IsErrIgnored(err, errFileNotFound, errDiskNotFound) {
+				logger.LogIf(ctx, err)
+			}
+			// Ignore this invalid upload-id since we are listing here
+			continue
 		}
 		populatedUploadIds.Add(uploadID)
 		uploads = append(uploads, MultipartInfo{
@@ -290,7 +330,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 
 	onlineDisks := er.getDisks()
 	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
-	if parityDrives <= 0 {
+	if parityDrives < 0 {
 		parityDrives = er.defaultParityCount
 	}
 
@@ -442,6 +482,47 @@ func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, ds
 	return evalDisks(disks, errs), err
 }
 
+// writeAllDisks - writes 'b' to all provided disks.
+// If write cannot reach quorum, the files will be deleted from all disks.
+func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry string, b []byte, writeQuorum int) ([]StorageAPI, error) {
+	g := errgroup.WithNErrs(len(disks))
+
+	// Write file to all underlying storage disks.
+	for index := range disks {
+		index := index
+		g.Go(func() error {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			return disks[index].WriteAll(ctx, dstBucket, dstEntry, b)
+		}, index)
+	}
+
+	// Wait for all renames to finish.
+	errs := g.Wait()
+
+	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
+	// otherwise return failure. Cleanup successful renames.
+	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	if err == errErasureWriteQuorum {
+		// Remove all written
+		g := errgroup.WithNErrs(len(disks))
+		for index := range disks {
+			if disks[index] == nil || errs[index] != nil {
+				continue
+			}
+			index := index
+			g.Go(func() error {
+				return disks[index].Delete(ctx, dstBucket, dstEntry, DeleteOptions{Force: true})
+			}, index)
+		}
+		// Ignore these errors.
+		g.WaitErr()
+	}
+
+	return evalDisks(disks, errs), err
+}
+
 // PutObjectPart - reads incoming stream and internally erasure codes
 // them. This call is similar to single put operation but it is part
 // of the multipart transaction.
@@ -468,11 +549,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return PartInfo{}, err
 	}
 	rctx := rlkctx.Context()
-	defer func() {
-		if uploadIDRLock != nil {
-			uploadIDRLock.RUnlock(rlkctx.Cancel)
-		}
-	}()
+	defer uploadIDRLock.RUnlock(rlkctx.Cancel)
 
 	data := r.Reader
 	// Validate input data size and it can never be less than zero.
@@ -496,10 +573,6 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	partsMetadata, errs = readAllFileInfo(rctx, storageDisks, minioMetaMultipartBucket,
 		uploadIDPath, "", false)
 
-	// Unlock upload id locks before, so others can get it.
-	uploadIDRLock.RUnlock(rlkctx.Cancel)
-	uploadIDRLock = nil
-
 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(pctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
@@ -606,79 +679,50 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		}
 	}
 
-	// Acquire write lock to update metadata.
-	uploadIDWLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	wlkctx, err := uploadIDWLock.GetLock(pctx, globalOperationTimeout)
-	if err != nil {
-		return PartInfo{}, err
-	}
-	wctx := wlkctx.Context()
-	defer uploadIDWLock.Unlock(wlkctx.Cancel)
-
-	// Validates if upload ID exists.
-	if err = er.checkUploadIDExists(wctx, bucket, object, uploadID); err != nil {
-		return pi, toObjectErr(err, bucket, object, uploadID)
-	}
-
 	// Rename temporary part file to its final location.
 	partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix)
-	onlineDisks, err = renamePart(wctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum)
+	onlineDisks, err = renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
-	// Read metadata again because it might be updated with parallel upload of another part.
-	partsMetadata, errs = readAllFileInfo(wctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
-	reducedErr = reduceWriteQuorumErrs(wctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if reducedErr == errErasureWriteQuorum {
-		return pi, toObjectErr(reducedErr, bucket, object)
-	}
-
-	// Get current highest version based on re-read partsMetadata.
-	onlineDisks, modTime = listOnlineDisks(onlineDisks, partsMetadata, errs)
-
-	// Pick one from the first valid metadata.
-	fi, err = pickValidFileInfo(wctx, partsMetadata, modTime, writeQuorum)
-	if err != nil {
-		return pi, err
-	}
-
-	// Once part is successfully committed, proceed with updating erasure metadata.
-	fi.ModTime = UTCNow()
-
 	md5hex := r.MD5CurrentHexString()
-
-	// Add the current part.
-	fi.AddObjectPart(partID, md5hex, n, data.ActualSize())
-
-	for i, disk := range onlineDisks {
-		if disk == OfflineDisk {
-			continue
-		}
-		partsMetadata[i].Size = fi.Size
-		partsMetadata[i].ModTime = fi.ModTime
-		partsMetadata[i].Parts = fi.Parts
-		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
-			PartNumber: partID,
-			Algorithm:  DefaultBitrotAlgorithm,
-			Hash:       bitrotWriterSum(writers[i]),
-		})
+	if opts.PreserveETag != "" {
+		md5hex = opts.PreserveETag
 	}
 
-	// Writes update `xl.meta` format for each disk.
-	if _, err = writeUniqueFileInfo(wctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
-		return pi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
 	}
 
-	online = countOnlineDisks(onlineDisks)
+	part := ObjectPartInfo{
+		Number:     partID,
+		ETag:       md5hex,
+		Size:       n,
+		ActualSize: data.ActualSize(),
+		ModTime:    UTCNow(),
+		Index:      index,
+	}
+
+	partMsg, err := part.MarshalMsg(nil)
+	if err != nil {
+		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
+	}
+
+	// Write part metadata to all disks.
+	onlineDisks, err = writeAllDisks(ctx, onlineDisks, minioMetaMultipartBucket, partPath+".meta", partMsg, writeQuorum)
+	if err != nil {
+		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
+	}
 
 	// Return success.
 	return PartInfo{
-		PartNumber:   partID,
-		ETag:         md5hex,
-		LastModified: fi.ModTime,
-		Size:         n,
-		ActualSize:   data.ActualSize(),
+		PartNumber:   part.Number,
+		ETag:         part.ETag,
+		LastModified: part.ModTime,
+		Size:         part.Size,
+		ActualSize:   part.ActualSize,
 	}, nil
 }
 
@@ -686,6 +730,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 // by callers to verify object states
 // - encrypted
 // - compressed
+// Does not contain currently uploaded parts by design.
 func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
 	auditObjectErasureSet(ctx, object, &er)
 
@@ -777,7 +822,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		return result, toObjectErr(reducedErr, minioMetaMultipartBucket, uploadIDPath)
 	}
 
-	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
+	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// Pick one from the first valid metadata.
 	fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, writeQuorum)
@@ -785,6 +830,41 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		return result, err
 	}
 
+	if maxParts == 0 {
+		return result, nil
+	}
+
+	if partNumberMarker < 0 {
+		partNumberMarker = 0
+	}
+
+	// Limit output to maxPartsList.
+	if maxParts > maxPartsList-partNumberMarker {
+		maxParts = maxPartsList - partNumberMarker
+	}
+
+	// Read Part info for all parts
+	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
+	req := ReadMultipleReq{
+		Bucket:     minioMetaMultipartBucket,
+		Prefix:     partPath,
+		MaxSize:    1 << 20, // Each part should realistically not be > 1MiB.
+		MaxResults: maxParts + 1,
+	}
+
+	start := partNumberMarker + 1
+	end := start + maxParts
+
+	// Parts are 1 based, so index 0 is part one, etc.
+	for i := start; i <= end; i++ {
+		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
+	}
+
+	partInfoFiles, err := readMultipleFiles(ctx, onlineDisks, req, writeQuorum)
+	if err != nil {
+		return result, err
+	}
+
 	// Populate the result stub.
 	result.Bucket = bucket
 	result.Object = object
@@ -794,34 +874,49 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	result.UserDefined = cloneMSS(fi.Metadata)
 
 	// For empty number of parts or maxParts as zero, return right here.
-	if len(fi.Parts) == 0 || maxParts == 0 {
+	if len(partInfoFiles) == 0 || maxParts == 0 {
 		return result, nil
 	}
 
-	// Limit output to maxPartsList.
-	if maxParts > maxPartsList {
-		maxParts = maxPartsList
+	var partI ObjectPartInfo
+	for i, part := range partInfoFiles {
+		partN := i + partNumberMarker + 1
+		if part.Error != "" || !part.Exists {
+			continue
+		}
+
+		_, err := partI.UnmarshalMsg(part.Data)
+		if err != nil {
+			// Maybe crash or similar.
+			logger.LogIf(ctx, err)
+			continue
+		}
+
+		if partN != partI.Number {
+			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", i+1, i+1, partI.Number))
+			continue
+		}
+
+		// Add the current part.
+		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index)
 	}
 
 	// Only parts with higher part numbers will be listed.
-	partIdx := objectPartIndex(fi.Parts, partNumberMarker)
 	parts := fi.Parts
-	if partIdx != -1 {
-		parts = fi.Parts[partIdx+1:]
-	}
-	count := maxParts
+	result.Parts = make([]PartInfo, 0, len(parts))
 	for _, part := range parts {
 		result.Parts = append(result.Parts, PartInfo{
 			PartNumber:   part.Number,
 			ETag:         part.ETag,
-			LastModified: fi.ModTime,
+			LastModified: part.ModTime,
+			ActualSize:   part.ActualSize,
 			Size:         part.Size,
 		})
-		count--
-		if count == 0 {
+		if len(result.Parts) >= maxParts {
 			break
 		}
 	}
+
 	// If listed entries are more than maxParts, we set IsTruncated as true.
 	if len(parts) > len(result.Parts) {
 		result.IsTruncated = true
@@ -842,17 +937,17 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)
 
-	// Hold read-locks to verify uploaded parts, also disallows
-	// parallel part uploads as well.
+	// Hold write locks to verify uploaded parts, also disallows any
+	// parallel PutObjectPart() requests.
 	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
-	rlkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
+	wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return oi, err
 	}
-	rctx := rlkctx.Context()
-	defer uploadIDLock.RUnlock(rlkctx.Cancel)
+	wctx := wlkctx.Context()
+	defer uploadIDLock.Unlock(wlkctx.Cancel)
 
-	if err = er.checkUploadIDExists(rctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(wctx, bucket, object, uploadID); err != nil {
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -861,15 +956,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(rctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
+	partsMetadata, errs := readAllFileInfo(wctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(rctx, partsMetadata, errs, er.defaultParityCount)
+	_, writeQuorum, err := objectQuorumFromMeta(wctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
 
-	reducedErr := reduceWriteQuorumErrs(rctx, errs, objectOpIgnoredErrs, writeQuorum)
+	reducedErr := reduceWriteQuorumErrs(wctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if reducedErr == errErasureWriteQuorum {
 		return oi, toObjectErr(reducedErr, bucket, object)
 	}
@@ -877,11 +972,63 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// Pick one from the first valid metadata.
-	fi, err := pickValidFileInfo(rctx, partsMetadata, modTime, writeQuorum)
+	fi, err := pickValidFileInfo(wctx, partsMetadata, modTime, writeQuorum)
 	if err != nil {
 		return oi, err
 	}
 
+	// Read Part info for all parts
+	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
+	req := ReadMultipleReq{
+		Bucket:     minioMetaMultipartBucket,
+		Prefix:     partPath,
+		MaxSize:    1 << 20, // Each part should realistically not be > 1MiB.
+		Files:      make([]string, 0, len(parts)),
+		AbortOn404: true,
+	}
+	for _, part := range parts {
+		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", part.PartNumber))
+	}
+	partInfoFiles, err := readMultipleFiles(ctx, onlineDisks, req, writeQuorum)
+	if err != nil {
+		return oi, err
+	}
+	if len(partInfoFiles) != len(parts) {
+		// Should only happen through internal error
+		err := fmt.Errorf("unexpected part result count: %d, want %d", len(partInfoFiles), len(parts))
+		logger.LogIf(ctx, err)
+		return oi, toObjectErr(err, bucket, object)
+	}
+
+	var partI ObjectPartInfo
+	for i, part := range partInfoFiles {
+		partID := parts[i].PartNumber
+		if part.Error != "" || !part.Exists {
+			return oi, InvalidPart{
+				PartNumber: partID,
+			}
+		}
+
+		_, err := partI.UnmarshalMsg(part.Data)
+		if err != nil {
+			// Maybe crash or similar.
+			logger.LogIf(ctx, err)
+			return oi, InvalidPart{
+				PartNumber: partID,
+			}
+		}
+
+		if partID != partI.Number {
+			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
+			return oi, InvalidPart{
+				PartNumber: partID,
+			}
+		}
+
+		// Add the current part.
+		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index)
+	}
+
 	// Calculate full object size.
 	var objectSize int64
 
@@ -921,7 +1068,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			return oi, invp
 		}
 
-		// All parts except the last part has to be atleast 5MB.
+		// All parts except the last part has to be at least 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(currentFI.Parts[partIdx].ActualSize) {
 			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
@@ -941,6 +1088,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			Number:     part.PartNumber,
 			Size:       currentFI.Parts[partIdx].Size,
 			ActualSize: currentFI.Parts[partIdx].ActualSize,
+			ModTime:    currentFI.Parts[partIdx].ModTime,
+			Index:      currentFI.Parts[partIdx].Index,
 		}
 	}
 
@@ -999,6 +1148,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	// Remove part.meta which is not needed anymore.
+	for _, part := range fi.Parts {
+		er.removePartMeta(bucket, object, uploadID, fi.DataDir, part.Number)
+	}
+
 	// Rename the multipart object to final location.
 	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
 		partsMetadata, bucket, object, writeQuorum); err != nil {

cmd/erasure-object.go

@@ -736,7 +736,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	storageDisks := er.getDisks()
 	// Get parity and data drive count based on storage class metadata
 	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
-	if parityDrives <= 0 {
+	if parityDrives < 0 {
 		parityDrives = er.defaultParityCount
 	}
 	dataDrives := len(storageDisks) - parityDrives
@@ -815,6 +815,12 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	if n < data.Size() {
 		return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key}
 	}
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
+	modTime := UTCNow()
 
 	for i, w := range writers {
 		if w == nil {
@@ -823,7 +829,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		partsMetadata[i].Data = inlineBuffers[i].Bytes()
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -831,8 +837,6 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		})
 	}
 
-	modTime := UTCNow()
-
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
@@ -885,7 +889,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	if !opts.MaxParity {
 		// Get parity and data drive count based on storage class metadata
 		parityDrives = globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
-		if parityDrives <= 0 {
+		if parityDrives < 0 {
 			parityDrives = er.defaultParityCount
 		}
 
@@ -972,14 +976,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	switch size := data.Size(); {
 	case size == 0:
 		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
-	case size == -1:
-		if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
-			buffer = make([]byte, data.ActualSize()+256, data.ActualSize()*2+512)
-		} else {
-			buffer = er.bp.Get()
-			defer er.bp.Put(buffer)
-		}
-	case size >= fi.Erasure.BlockSize:
+	case size >= fi.Erasure.BlockSize || size == -1:
 		buffer = er.bp.Get()
 		defer er.bp.Put(buffer)
 	case size < fi.Erasure.BlockSize:
@@ -1071,6 +1068,10 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
+	var compIndex []byte
+	if opts.IndexCB != nil {
+		compIndex = opts.IndexCB()
+	}
 	if !opts.NoLock {
 		lk := er.NewNSLock(bucket, object)
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -1081,6 +1082,11 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		defer lk.Unlock(lkctx.Cancel)
 	}
 
+	modTime := opts.MTime
+	if opts.MTime.IsZero() {
+		modTime = UTCNow()
+	}
+
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -1091,15 +1097,19 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		} else {
 			partsMetadata[i].Data = nil
 		}
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, compIndex)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
 			Hash:       bitrotWriterSum(w),
 		})
 	}
+
 	if userDefined["etag"] == "" {
 		userDefined["etag"] = r.MD5CurrentHexString()
+		if opts.PreserveETag != "" {
+			userDefined["etag"] = opts.PreserveETag
+		}
 	}
 
 	// Guess content-type from the extension if possible.
@@ -1107,11 +1117,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
-	modTime := opts.MTime
-	if opts.MTime.IsZero() {
-		modTime = UTCNow()
-	}
-
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
@@ -1367,8 +1372,14 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 			// Deletes
 			// - The prefix and its children
 			// - The prefix__XLDIR__
-			defer disks[index].Delete(ctx, bucket, dirPrefix, true)
-			return disks[index].Delete(ctx, bucket, prefix, true)
+			defer disks[index].Delete(ctx, bucket, dirPrefix, DeleteOptions{
+				Recursive: true,
+				Force:     true,
+			})
+			return disks[index].Delete(ctx, bucket, prefix, DeleteOptions{
+				Recursive: true,
+				Force:     true,
+			})
 		}, index)
 	}
 	for _, err := range g.Wait() {

cmd/erasure-object_test.go

@@ -528,7 +528,10 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		files, _ := disk.ListDir(ctx, bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
-				disk.Delete(ctx, bucket, pathJoin(object, file), true)
+				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
+					Recursive: true,
+					Force:     false,
+				})
 			}
 		}
 	}
@@ -629,7 +632,10 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 		files, _ := disk.ListDir(ctx, bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
-				disk.Delete(ctx, bucket, pathJoin(object, file), true)
+				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
+					Recursive: true,
+					Force:     false,
+				})
 			}
 		}
 	}
@@ -893,6 +899,14 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 
 	// Object for test case 1 - No StorageClass defined, no MetaData in PutObject
 	object1 := "object1"
+	globalStorageClass = storageclass.Config{
+		RRS: storageclass.StorageClass{
+			Parity: 2,
+		},
+		Standard: storageclass.StorageClass{
+			Parity: 4,
+		},
+	}
 	_, err = obj.PutObject(ctx, bucket, object1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
@@ -964,17 +978,16 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "", false)
-	parts5SC := storageclass.Config{
-		RRS: storageclass.StorageClass{
-			Parity: 2,
-		},
-	}
+	parts5SC := globalStorageClass
 
 	// Object for test case 6 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting Standard Storage Class
 	object6 := "object6"
 	metadata6 := make(map[string]string)
 	metadata6["x-amz-storage-class"] = storageclass.STANDARD
 	globalStorageClass = storageclass.Config{
+		Standard: storageclass.StorageClass{
+			Parity: 4,
+		},
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
@@ -1035,7 +1048,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		tt := tt
 		t.(*testing.T).Run("", func(t *testing.T) {
 			globalStorageClass = tt.storageClassCfg
-			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, getDefaultParityBlocks(len(erasureDisks)))
+			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, storageclass.DefaultParityBlocks(len(erasureDisks)))
 			if tt.expectedError != nil && err == nil {
 				t.Errorf("Expected %s, got %s", tt.expectedError, err)
 			}

cmd/erasure-server-pool-decom.go

@@ -22,6 +22,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"sort"
 	"strconv"
@@ -30,6 +31,8 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
@@ -53,6 +56,9 @@ type PoolDecommissionInfo struct {
 
 	// Last bucket/object decommissioned.
 	Bucket string `json:"-" msg:"bkt"`
+	// Captures prefix that is currently being
+	// decommissioned inside the 'Bucket'
+	Prefix string `json:"-" msg:"pfx"`
 	Object string `json:"-" msg:"obj"`
 
 	// Verbose information
@@ -73,6 +79,7 @@ func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
 			// Clear tracker info.
 			if pd.Bucket == bucket {
 				pd.Bucket = "" // empty this out for next bucket
+				pd.Prefix = "" // empty this out for the next bucket
 				pd.Object = "" // empty this out for next object
 			}
 			return
@@ -80,12 +87,6 @@ func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
 	}
 }
 
-func (pd *PoolDecommissionInfo) bucketsToDecommission() []string {
-	queuedBuckets := make([]string, len(pd.QueuedBuckets))
-	copy(queuedBuckets, pd.QueuedBuckets)
-	return queuedBuckets
-}
-
 func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
 	for _, b := range pd.DecommissionedBuckets {
 		if b == bucket {
@@ -95,17 +96,18 @@ func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
 	return false
 }
 
-func (pd *PoolDecommissionInfo) bucketPush(bucket string) {
+func (pd *PoolDecommissionInfo) bucketPush(bucket decomBucketInfo) {
 	for _, b := range pd.QueuedBuckets {
 		if pd.isBucketDecommissioned(b) {
 			return
 		}
-		if b == bucket {
+		if b == bucket.String() {
 			return
 		}
 	}
-	pd.QueuedBuckets = append(pd.QueuedBuckets, bucket)
-	pd.Bucket = bucket
+	pd.QueuedBuckets = append(pd.QueuedBuckets, bucket.String())
+	pd.Bucket = bucket.Name
+	pd.Prefix = bucket.Prefix
 }
 
 // PoolStatus captures current pool status
@@ -183,12 +185,12 @@ func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool {
 	return p.Pools[idx].Decommission.isBucketDecommissioned(bucket)
 }
 
-func (p *poolMeta) BucketDone(idx int, bucket string) {
+func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) {
 	if p.Pools[idx].Decommission == nil {
 		// Decommission not in progress.
 		return
 	}
-	p.Pools[idx].Decommission.bucketPop(bucket)
+	p.Pools[idx].Decommission.bucketPop(bucket.String())
 }
 
 func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) {
@@ -208,19 +210,38 @@ func (p *poolMeta) TrackCurrentBucketObject(idx int, bucket string, object strin
 	p.Pools[idx].Decommission.Object = object
 }
 
-func (p *poolMeta) PendingBuckets(idx int) []string {
+func (p *poolMeta) PendingBuckets(idx int) []decomBucketInfo {
 	if p.Pools[idx].Decommission == nil {
 		// Decommission not in progress.
 		return nil
 	}
 
-	return p.Pools[idx].Decommission.bucketsToDecommission()
+	decomBuckets := make([]decomBucketInfo, len(p.Pools[idx].Decommission.QueuedBuckets))
+	for i := range decomBuckets {
+		bucket, prefix := path2BucketObject(p.Pools[idx].Decommission.QueuedBuckets[i])
+		decomBuckets[i] = decomBucketInfo{
+			Name:   bucket,
+			Prefix: prefix,
+		}
+	}
+
+	return decomBuckets
 }
 
-func (p *poolMeta) QueueBuckets(idx int, buckets []BucketInfo) {
+//msgp:ignore decomBucketInfo
+type decomBucketInfo struct {
+	Name   string
+	Prefix string
+}
+
+func (db decomBucketInfo) String() string {
+	return pathJoin(db.Name, db.Prefix)
+}
+
+func (p *poolMeta) QueueBuckets(idx int, buckets []decomBucketInfo) {
 	// add new queued buckets
 	for _, bucket := range buckets {
-		p.Pools[idx].Decommission.bucketPush(bucket.Name)
+		p.Pools[idx].Decommission.bucketPush(bucket)
 	}
 }
 
@@ -505,14 +526,26 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 			}
 			if globalEndpoints[idx].Endpoints[0].IsLocal {
 				go func(pool PoolStatus) {
-					switch err := z.Decommission(ctx, pool.ID); err {
-					case nil:
-						// we already started decommission
-					case errDecommissionAlreadyRunning:
-						// A previous decommission running found restart it.
-						z.doDecommissionInRoutine(ctx, idx)
-					default:
-						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+					r := rand.New(rand.NewSource(time.Now().UnixNano()))
+					for {
+						if err := z.Decommission(ctx, pool.ID); err != nil {
+							switch err {
+							// we already started decommission
+							case errDecommissionAlreadyRunning:
+								// A previous decommission running found restart it.
+								z.doDecommissionInRoutine(ctx, idx)
+								return
+							default:
+								if configRetriableErrors(err) {
+									logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w: retrying..", pool, err))
+									time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
+									continue
+								}
+								logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+								return
+							}
+						}
+						break
 					}
 				}(pool)
 			}
@@ -548,6 +581,11 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		auditLogDecom(ctx, "DecomCopyData", objInfo.Bucket, objInfo.Name, objInfo.VersionID, err)
 	}()
 
+	actualSize, err := objInfo.GetActualSize()
+	if err != nil {
+		return err
+	}
+
 	if objInfo.isMultipart() {
 		uploadID, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:   objInfo.VersionID,
@@ -555,21 +593,26 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			UserDefined: objInfo.UserDefined,
 		})
 		if err != nil {
-			return err
+			return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err)
 		}
 		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, uploadID, ObjectOptions{})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
-			hr, err := hash.NewReader(gr, part.Size, "", "", part.Size)
+			hr, err := hash.NewReader(gr, part.Size, "", "", part.ActualSize)
 			if err != nil {
-				return err
+				return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 			}
 			pi, err := z.PutObjectPart(ctx, bucket, objInfo.Name, uploadID,
 				part.Number,
 				NewPutObjReader(hr),
-				ObjectOptions{})
+				ObjectOptions{
+					PreserveETag: part.ETag, // Preserve original ETag to ensure same metadata.
+					IndexCB: func() []byte {
+						return part.Index // Preserve part Index to ensure decompression works.
+					},
+				})
 			if err != nil {
-				return err
+				return fmt.Errorf("decommissionObject: PutObjectPart() %w", err)
 			}
 			parts[i] = CompletePart{
 				ETag:       pi.ETag,
@@ -579,21 +622,32 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, uploadID, parts, ObjectOptions{
 			MTime: objInfo.ModTime,
 		})
+		if err != nil {
+			err = fmt.Errorf("decommissionObject: CompleteMultipartUpload() %w", err)
+		}
 		return err
 	}
-	hr, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size)
+
+	hr, err := hash.NewReader(gr, objInfo.Size, "", "", actualSize)
 	if err != nil {
-		return err
+		return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 	}
 	_, err = z.PutObject(ctx,
 		bucket,
 		objInfo.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
-			VersionID:   objInfo.VersionID,
-			MTime:       objInfo.ModTime,
-			UserDefined: objInfo.UserDefined,
+			VersionID:    objInfo.VersionID,
+			MTime:        objInfo.ModTime,
+			UserDefined:  objInfo.UserDefined,
+			PreserveETag: objInfo.ETag, // Preserve original ETag to ensure same metadata.
+			IndexCB: func() []byte {
+				return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works.
+			},
 		})
+	if err != nil {
+		err = fmt.Errorf("decommissionObject: PutObject() %w", err)
+	}
 	return err
 }
 
@@ -607,7 +661,7 @@ func (v versionsSorter) reverse() {
 	})
 }
 
-func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bName string) error {
+func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
 
 	var wg sync.WaitGroup
@@ -628,7 +682,34 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			continue
 		}
 
-		vc, _ := globalBucketVersioningSys.Get(bName)
+		vc, _ := globalBucketVersioningSys.Get(bi.Name)
+
+		// Check if the current bucket has a configured lifecycle policy
+		lc, _ := globalLifecycleSys.Get(bi.Name)
+
+		// Check if bucket is object locked.
+		lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+
+		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
+			if lc == nil {
+				return false
+			}
+			versioned := vc != nil && vc.Versioned(object)
+			objInfo := fi.ToObjectInfo(bucket, object, versioned)
+			action := evalActionFromLifecycle(ctx, *lc, lr, objInfo, false)
+			switch action {
+			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+				// Skip this entry.
+				return true
+			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				// Skip this entry.
+				return true
+			}
+			return false
+		}
+
 		decommissionEntry := func(entry metaCacheEntry) {
 			defer func() {
 				<-parallelWorkers
@@ -639,7 +720,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				return
 			}
 
-			fivs, err := entry.fileInfoVersions(bName)
+			fivs, err := entry.fileInfoVersions(bi.Name)
 			if err != nil {
 				return
 			}
@@ -652,25 +733,35 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			for _, version := range fivs.Versions {
 				// TODO: Skip transitioned objects for now.
 				if version.IsRemote() {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bName, version.Name))
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bi.Name, version.Name))
 					continue
 				}
+
+				// Apply lifecycle rules on the objects that are expired.
+				if filterLifecycle(bi.Name, version.Name, version) {
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s (%s) expired object based on ILM rules, skipping and scheduled for deletion", bi.Name, version.Name, version.VersionID))
+					continue
+				}
+
 				// We will skip decommissioning delete markers
 				// with single version, its as good as there
 				// is no data associated with the object.
 				if version.Deleted && len(fivs.Versions) == 1 {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no content left", bName, version.Name))
+					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no content left", bi.Name, version.Name))
 					continue
 				}
+
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
-						bName,
+						bi.Name,
 						version.Name,
 						ObjectOptions{
-							Versioned:         vc.PrefixEnabled(version.Name),
-							VersionID:         version.VersionID,
-							MTime:             version.ModTime,
-							DeleteReplication: version.ReplicationState,
+							Versioned:          vc.PrefixEnabled(version.Name),
+							VersionID:          version.VersionID,
+							MTime:              version.ModTime,
+							DeleteReplication:  version.ReplicationState,
+							DeleteMarker:       true, // make sure we create a delete marker
+							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 						})
 					var failure bool
 					if err != nil {
@@ -680,10 +771,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					z.poolMetaMutex.Lock()
 					z.poolMeta.CountItem(idx, 0, failure)
 					z.poolMetaMutex.Unlock()
-					if failure {
-						break // break out on first error
+					if !failure {
+						// Success keep a count.
+						decommissionedCount++
 					}
-					decommissionedCount++
 					continue
 				}
 
@@ -691,13 +782,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
 					gr, err := set.GetObjectNInfo(ctx,
-						bName,
+						bi.Name,
 						encodeDirObject(version.Name),
 						nil,
 						http.Header{},
 						noLock, // all mutations are blocked reads are safe without locks.
 						ObjectOptions{
-							VersionID: version.VersionID,
+							VersionID:    version.VersionID,
+							NoDecryption: true,
 						})
 					if isErrObjectNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
@@ -708,7 +800,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						logger.LogIf(ctx, err)
 						continue
 					}
-					if err = z.decommissionObject(ctx, bName, gr); err != nil {
+					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
 						failure = true
 						logger.LogIf(ctx, err)
 						continue
@@ -728,19 +820,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// if all versions were decommissioned, then we can delete the object versions.
 			if decommissionedCount == len(fivs.Versions) {
 				_, err := set.DeleteObject(ctx,
-					bName,
+					bi.Name,
 					encodeDirObject(entry.name),
 					ObjectOptions{
 						DeletePrefix: true, // use prefix delete to delete all versions at once.
 					},
 				)
-				auditLogDecom(ctx, "DecomDeleteObject", bName, entry.name, "", err)
+				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 				}
 			}
 			z.poolMetaMutex.Lock()
-			z.poolMeta.TrackCurrentBucketObject(idx, bName, entry.name)
+			z.poolMeta.TrackCurrentBucketObject(idx, bi.Name, entry.name)
 			ok, err := z.poolMeta.updateAfter(ctx, idx, z.serverPools, 30*time.Second)
 			logger.LogIf(ctx, err)
 			if ok {
@@ -753,7 +845,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		resolver := metadataResolutionParams{
 			dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
 			objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
-			bucket:    bName,
+			bucket:    bi.Name,
 		}
 
 		wg.Add(1)
@@ -761,7 +853,8 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			defer wg.Done()
 			err := listPathRaw(ctx, listPathRawOptions{
 				disks:          disks,
-				bucket:         bName,
+				bucket:         bi.Name,
+				path:           bi.Prefix,
 				recursive:      true,
 				forwardTo:      "",
 				minDisks:       len(disks) / 2, // to capture all quorum ratios
@@ -771,7 +864,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					wg.Add(1)
 					go decommissionEntry(entry)
 				},
-				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+				partial: func(entries metaCacheEntries, _ []error) {
 					entry, ok := entries.resolve(&resolver)
 					if ok {
 						parallelWorkers <- struct{}{}
@@ -791,7 +884,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
 	pool := z.serverPools[idx]
 	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
-		if z.poolMeta.isBucketDecommissioned(idx, bucket) {
+		if z.poolMeta.isBucketDecommissioned(idx, bucket.String()) {
 			if serverDebugLog {
 				console.Debugln("decommission: already done, moving on", bucket)
 			}
@@ -803,7 +896,7 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 			continue
 		}
 		if serverDebugLog {
-			console.Debugln("decommission: currently on bucket", bucket)
+			console.Debugln("decommission: currently on bucket", bucket.Name)
 		}
 		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
 			return err
@@ -892,15 +985,9 @@ func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpace
 	if idx+1 > len(z.serverPools) {
 		return pi, errInvalidArgument
 	}
+
 	info, _ := z.serverPools[idx].StorageInfo(context.Background())
 	info.Backend = z.BackendInfo()
-	for _, disk := range info.Disks {
-		if disk.Healing {
-			return pi, decomError{
-				Err: fmt.Sprintf("%s drive is healing, decommission will not be started", disk.Endpoint),
-			}
-		}
-	}
 
 	usableTotal := int64(GetTotalUsableCapacity(info.Disks, info))
 	usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info))
@@ -921,7 +1008,7 @@ func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, e
 
 	pi, err := z.getDecommissionPoolSpaceInfo(idx)
 	if err != nil {
-		return PoolStatus{}, errInvalidArgument
+		return PoolStatus{}, err
 	}
 
 	poolInfo := z.poolMeta.Pools[idx]
@@ -964,7 +1051,9 @@ func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (e
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionCancel(idx) {
-		defer z.decommissionCancelers[idx]() // cancel any active thread.
+		if fn := z.decommissionCancelers[idx]; fn != nil {
+			defer fn() // cancel any active thread.
+		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -986,7 +1075,9 @@ func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (e
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionFailed(idx) {
-		defer z.decommissionCancelers[idx]() // cancel any active thread.
+		if fn := z.decommissionCancelers[idx]; fn != nil {
+			defer fn() // cancel any active thread.
+		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -1008,7 +1099,9 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionComplete(idx) {
-		defer z.decommissionCancelers[idx]() // cancel any active thread.
+		if fn := z.decommissionCancelers[idx]; fn != nil {
+			defer fn() // cancel any active thread.
+		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -1031,8 +1124,22 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		return err
 	}
 
-	// TODO: Support decommissioning transition tiers.
+	// Make sure to heal the buckets to ensure the new
+	// pool has the new buckets, this is to avoid
+	// failures later.
 	for _, bucket := range buckets {
+		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
+	}
+
+	decomBuckets := make([]decomBucketInfo, len(buckets))
+	for i := range buckets {
+		decomBuckets[i] = decomBucketInfo{
+			Name: buckets[i].Name,
+		}
+	}
+
+	// TODO: Support decommissioning transition tiers.
+	for _, bucket := range decomBuckets {
 		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
 			if lc.HasTransition() {
 				return decomError{
@@ -1059,11 +1166,13 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
-	buckets = append(buckets, BucketInfo{
-		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: minioConfigPrefix,
 	})
-	buckets = append(buckets, BucketInfo{
-		Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: bucketMetaPrefix,
 	})
 
 	var pool *erasureSets
@@ -1089,7 +1198,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 	if err = z.poolMeta.Decommission(idx, pi); err != nil {
 		return err
 	}
-	z.poolMeta.QueueBuckets(idx, buckets)
+	z.poolMeta.QueueBuckets(idx, decomBuckets)
 	if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 		return err
 	}
@@ -1103,7 +1212,7 @@ func auditLogDecom(ctx context.Context, apiName, bucket, object, versionID strin
 		errStr = err.Error()
 	}
 	auditLogInternal(ctx, bucket, object, AuditLogOptions{
-		Trigger:   "decommissioning",
+		Event:     "decommission",
 		APIName:   apiName,
 		VersionID: versionID,
 		Error:     errStr,

cmd/erasure-server-pool-decom_gen.go

@@ -110,6 +110,12 @@ func (z *PoolDecommissionInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Bucket")
 				return
 			}
+		case "pfx":
+			z.Prefix, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
 		case "obj":
 			z.Object, err = dc.ReadString()
 			if err != nil {
@@ -153,9 +159,9 @@ func (z *PoolDecommissionInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *PoolDecommissionInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 15
+	// map header, size 16
 	// write "st"
-	err = en.Append(0x8f, 0xa2, 0x73, 0x74)
+	err = en.Append(0xde, 0x0, 0x10, 0xa2, 0x73, 0x74)
 	if err != nil {
 		return
 	}
@@ -268,6 +274,16 @@ func (z *PoolDecommissionInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Bucket")
 		return
 	}
+	// write "pfx"
+	err = en.Append(0xa3, 0x70, 0x66, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Prefix)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
 	// write "obj"
 	err = en.Append(0xa3, 0x6f, 0x62, 0x6a)
 	if err != nil {
@@ -324,9 +340,9 @@ func (z *PoolDecommissionInfo) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *PoolDecommissionInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 15
+	// map header, size 16
 	// string "st"
-	o = append(o, 0x8f, 0xa2, 0x73, 0x74)
+	o = append(o, 0xde, 0x0, 0x10, 0xa2, 0x73, 0x74)
 	o = msgp.AppendTime(o, z.StartTime)
 	// string "ss"
 	o = append(o, 0xa2, 0x73, 0x73)
@@ -361,6 +377,9 @@ func (z *PoolDecommissionInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "bkt"
 	o = append(o, 0xa3, 0x62, 0x6b, 0x74)
 	o = msgp.AppendString(o, z.Bucket)
+	// string "pfx"
+	o = append(o, 0xa3, 0x70, 0x66, 0x78)
+	o = msgp.AppendString(o, z.Prefix)
 	// string "obj"
 	o = append(o, 0xa3, 0x6f, 0x62, 0x6a)
 	o = msgp.AppendString(o, z.Object)
@@ -483,6 +502,12 @@ func (z *PoolDecommissionInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Bucket")
 				return
 			}
+		case "pfx":
+			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
 		case "obj":
 			z.Object, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
@@ -527,7 +552,7 @@ func (z *PoolDecommissionInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *PoolDecommissionInfo) Msgsize() (s int) {
-	s = 1 + 3 + msgp.TimeSize + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.BoolSize + 3 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.ArrayHeaderSize
+	s = 3 + 3 + msgp.TimeSize + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.BoolSize + 3 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.QueuedBuckets {
 		s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001])
 	}
@@ -535,7 +560,7 @@ func (z *PoolDecommissionInfo) Msgsize() (s int) {
 	for za0002 := range z.DecommissionedBuckets {
 		s += msgp.StringPrefixSize + len(z.DecommissionedBuckets[za0002])
 	}
-	s += 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
+	s += 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Prefix) + 4 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size
 	return
 }
 

cmd/erasure-server-pool.go

@@ -119,7 +119,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		}
 
 		if deploymentID == "" {
-			// all zones should have same deployment ID
+			// all pools should have same deployment ID
 			deploymentID = formats[i].ID
 		}
 
@@ -238,6 +238,7 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 	return nil
 }
 
+// Return the count of disks in each pool
 func (z *erasureServerPools) SetDriveCounts() []int {
 	setDriveCounts := make([]int, len(z.serverPools))
 	for i := range z.serverPools {
@@ -426,8 +427,9 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 	})
 
 	for _, pinfo := range poolObjInfos {
-		// skip all objects from suspended pools for mutating calls.
-		if z.IsSuspended(pinfo.PoolIndex) && opts.Mutate {
+		// skip all objects from suspended pools if asked by the
+		// caller.
+		if z.IsSuspended(pinfo.PoolIndex) && opts.SkipDecommissioned {
 			continue
 		}
 
@@ -457,11 +459,11 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 // If the object exists, but the latest version is a delete marker, the index with it is still returned.
 // If the object does not exist ObjectNotFound error is returned.
 // If any other error is found, it is returned.
-// The check is skipped if there is only one zone, and 0, nil is always returned in that case.
+// The check is skipped if there is only one pool, and 0, nil is always returned in that case.
 func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
 	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{
-		NoLock: true,
-		Mutate: true,
+		NoLock:             true,
+		SkipDecommissioned: true,
 	})
 }
 
@@ -485,7 +487,7 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
 // if none are found falls back to most available space pool, this function is
 // designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
 func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
-	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{Mutate: true})
+	idx, err = z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{SkipDecommissioned: true})
 	if err != nil && !isErrObjectNotFound(err) {
 		return idx, err
 	}
@@ -525,7 +527,7 @@ func (z *erasureServerPools) BackendInfo() (b madmin.BackendInfo) {
 	b.Type = madmin.Erasure
 
 	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-	if scParity <= 0 {
+	if scParity < 0 {
 		scParity = z.serverPools[0].defaultParityCount
 	}
 	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
@@ -881,7 +883,7 @@ func (z *erasureServerPools) getLatestObjectInfoWithIdx(ctx context.Context, buc
 	sort.Slice(results, func(i, j int) bool {
 		a, b := results[i], results[j]
 		if a.oi.ModTime.Equal(b.oi.ModTime) {
-			// On tiebreak, select the lowest zone index.
+			// On tiebreak, select the lowest pool index.
 			return a.zIdx < b.zIdx
 		}
 		return a.oi.ModTime.After(b.oi.ModTime)
@@ -975,13 +977,8 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 }
 
 func (z *erasureServerPools) deletePrefix(ctx context.Context, bucket string, prefix string) error {
-	for idx, zone := range z.serverPools {
-		if z.IsSuspended(idx) {
-			logger.LogIf(ctx, fmt.Errorf("pool %d is suspended, all writes are suspended", idx+1))
-			continue
-		}
-		_, err := zone.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
-		if err != nil {
+	for _, pool := range z.serverPools {
+		if _, err := pool.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true}); err != nil {
 			return err
 		}
 	}
@@ -1003,7 +1000,6 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return objInfo, err
@@ -1053,7 +1049,9 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		j := j
 		obj := obj
 		eg.Go(func() error {
-			idx, err := z.getPoolIdxExistingNoLock(ctx, bucket, obj.ObjectName)
+			idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, obj.ObjectName, ObjectOptions{
+				NoLock: true,
+			})
 			if err != nil {
 				derrs[j] = err
 				return nil
@@ -1323,7 +1321,10 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
 	poolResult.KeyMarker = keyMarker
 	poolResult.Prefix = prefix
 	poolResult.Delimiter = delimiter
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
 			delimiter, maxUploads)
 		if err != nil {
@@ -1348,6 +1349,9 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
 		if err != nil {
 			return "", err
@@ -1390,7 +1394,10 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
@@ -1419,7 +1426,10 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 	if z.SinglePool() {
 		return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		mi, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return mi, nil
@@ -1448,7 +1458,10 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 	if z.SinglePool() {
 		return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
@@ -1476,7 +1489,10 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
 
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
@@ -1505,7 +1521,10 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 
-	for _, pool := range z.serverPools {
+	for idx, pool := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
 		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
@@ -1650,7 +1669,10 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketI
 	if z.SinglePool() {
 		buckets, err = z.serverPools[0].ListBuckets(ctx)
 	} else {
-		for _, pool := range z.serverPools {
+		for idx, pool := range z.serverPools {
+			if z.IsSuspended(idx) {
+				continue
+			}
 			buckets, err = pool.ListBuckets(ctx)
 			if err != nil {
 				logger.LogIf(ctx, err)
@@ -1820,7 +1842,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 						minDisks:       1,
 						reportNotFound: false,
 						agreed:         loadEntry,
-						partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+						partial: func(entries metaCacheEntries, _ []error) {
 							entry, ok := entries.resolve(&resolver)
 							if !ok {
 								// check if we can get one entry atleast
@@ -1887,7 +1909,7 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 				cancel()
 			}
 		},
-		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+		partial: func(entries metaCacheEntries, _ []error) {
 			entry, ok := entries.resolve(&resolver)
 			if !ok {
 				// check if we can get one entry atleast
@@ -2208,7 +2230,6 @@ func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, obje
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2225,7 +2246,6 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 	}
 
 	// We don't know the size here set 1GiB atleast.
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2241,7 +2261,6 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -2257,7 +2276,6 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = false
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, err
@@ -2273,7 +2291,8 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 		return z.serverPools[0].TransitionObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
+	// Avoid transitioning an object from a pool being decommissioned.
+	opts.SkipDecommissioned = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err
@@ -2289,7 +2308,8 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
 
-	opts.Mutate = true
+	// Avoid restoring object from a pool being decommissioned.
+	opts.SkipDecommissioned = true
 	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return err

cmd/erasure-sets.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"context"
 	"encoding/binary"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"hash/crc32"
@@ -92,8 +91,6 @@ type erasureSets struct {
 	distributionAlgo string
 	deploymentID     [16]byte
 
-	disksStorageInfoCache timedValue
-
 	lastConnectDisksOpTime time.Time
 }
 
@@ -468,7 +465,7 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 				getDisks:              s.GetDisks(i),
 				getLockers:            s.GetLockers(i),
 				getEndpoints:          s.GetEndpoints(i),
-				deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
+				deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second, false),
 				nsMutex:               mutex,
 				bp:                    bp,
 				bpOld:                 bpOld,
@@ -551,29 +548,13 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 	}
 }
 
-const objectErasureMapKey = "objectErasureMap"
-
 type auditObjectOp struct {
+	Name  string   `json:"name"`
 	Pool  int      `json:"poolId"`
 	Set   int      `json:"setId"`
 	Disks []string `json:"disks"`
 }
 
-type auditObjectErasureMap struct {
-	sync.Map
-}
-
-// Define how to marshal auditObjectErasureMap so it can be
-// printed in the audit webhook notification request.
-func (a *auditObjectErasureMap) MarshalJSON() ([]byte, error) {
-	mapCopy := make(map[string]auditObjectOp)
-	a.Range(func(k, v interface{}) bool {
-		mapCopy[k.(string)] = v.(auditObjectOp)
-		return true
-	})
-	return json.Marshal(mapCopy)
-}
-
 // Add erasure set information to the current context
 func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjects) {
 	if len(logger.AuditTargets()) == 0 {
@@ -581,33 +562,20 @@ func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjec
 	}
 
 	object = decodeDirObject(object)
-
-	var disksEndpoints []string
-	for _, endpoint := range set.getEndpoints() {
+	endpoints := set.getEndpoints()
+	disksEndpoints := make([]string, 0, len(endpoints))
+	for _, endpoint := range endpoints {
 		disksEndpoints = append(disksEndpoints, endpoint.String())
 	}
 
 	op := auditObjectOp{
+		Name:  object,
 		Pool:  set.poolIndex + 1,
 		Set:   set.setIndex + 1,
 		Disks: disksEndpoints,
 	}
 
-	var objectErasureSetTag *auditObjectErasureMap
-	reqInfo := logger.GetReqInfo(ctx)
-	for _, kv := range reqInfo.GetTags() {
-		if kv.Key == objectErasureMapKey {
-			objectErasureSetTag = kv.Val.(*auditObjectErasureMap)
-			break
-		}
-	}
-
-	if objectErasureSetTag == nil {
-		objectErasureSetTag = &auditObjectErasureMap{}
-	}
-
-	objectErasureSetTag.Store(object, op)
-	reqInfo.SetTags(objectErasureMapKey, objectErasureSetTag)
+	logger.GetReqInfo(ctx).AppendTags("objectLocation", op)
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
@@ -629,47 +597,6 @@ func (s *erasureSets) ParityCount() int {
 	return s.defaultParityCount
 }
 
-// StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets.
-// This only returns disk usage info for ServerPools to perform placement decision, this call
-// is not implemented in Object interface and is not meant to be used by other object
-// layer implementations.
-func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
-	storageUsageInfo := func() StorageInfo {
-		var storageInfo StorageInfo
-		storageInfos := make([]StorageInfo, len(s.sets))
-		storageInfo.Backend.Type = madmin.Erasure
-
-		g := errgroup.WithNErrs(len(s.sets))
-		for index := range s.sets {
-			index := index
-			g.Go(func() error {
-				// ignoring errors on purpose
-				storageInfos[index], _ = s.sets[index].StorageInfo(ctx)
-				return nil
-			}, index)
-		}
-
-		// Wait for the go routines.
-		g.Wait()
-
-		for _, lstorageInfo := range storageInfos {
-			storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...)
-		}
-
-		return storageInfo
-	}
-
-	s.disksStorageInfoCache.Once.Do(func() {
-		s.disksStorageInfoCache.TTL = time.Second
-		s.disksStorageInfoCache.Update = func() (interface{}, error) {
-			return storageUsageInfo(), nil
-		}
-	})
-
-	v, _ := s.disksStorageInfoCache.Get()
-	return v.(StorageInfo)
-}
-
 // StorageInfo - combines output of StorageInfo across all erasure coded object sets.
 func (s *erasureSets) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 	var storageInfo madmin.StorageInfo
@@ -1047,11 +974,13 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 		// Version ID is set for the destination and source == destination version ID.
 		// perform an in-place update.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
+			srcInfo.Reader.Close() // We are not interested in the reader stream at this point close it.
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		// Destination is not versioned and source version ID is empty
 		// perform an in-place update.
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
+			srcInfo.Reader.Close() // We are not interested in the reader stream at this point close it.
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		// CopyObject optimization where we don't create an entire copy
@@ -1060,6 +989,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 		// that we actually create a new dataDir for legacy objects.
 		if dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID && !srcInfo.Legacy {
 			srcInfo.versionOnly = true
+			srcInfo.Reader.Close() // We are not interested in the reader stream at this point close it.
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 	}
@@ -1235,8 +1165,12 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 		// Do nothing
 		return
 	}
-	infos, _ := getHealDiskInfos(storageDisks, errs)
+	infos, ierrs := getHealDiskInfos(storageDisks, errs)
 	for i := range storageDisks {
+		if ierrs[i] != nil && ierrs[i] != errUnformattedDisk {
+			storageDisks[i] = nil
+			continue
+		}
 		if storageDisks[i] != nil && infos[i].RootDisk {
 			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 			// defective drive we should not heal a path on the root disk.

cmd/erasure-single-drive.go

@@ -90,7 +90,7 @@ func newErasureSingle(ctx context.Context, storageDisk StorageAPI, format *forma
 		format:                format,
 		nsMutex:               newNSLock(false),
 		bp:                    bp,
-		deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second),
+		deletedCleanupSleeper: newDynamicSleeper(10, 2*time.Second, false),
 	}
 
 	// start cleanup stale uploads go-routine.
@@ -184,6 +184,10 @@ func (es *erasureSingle) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+func (es *erasureSingle) SetDriveCounts() []int {
+	return []int{1}
+}
+
 func (es *erasureSingle) BackendInfo() (b madmin.BackendInfo) {
 	b.Type = madmin.Erasure
 
@@ -318,6 +322,10 @@ func (es *erasureSingle) DeleteBucket(ctx context.Context, bucket string, opts D
 	defer NSUpdated(bucket, slashSeparator)
 
 	err := es.disk.DeleteVol(ctx, bucket, opts.Force)
+	// Purge the entire bucket metadata entirely.
+	deleteBucketMetadata(ctx, es, bucket)
+	globalBucketMetadataSys.Remove(bucket)
+
 	return toObjectErr(err, bucket)
 }
 
@@ -845,6 +853,13 @@ func (es *erasureSingle) putMetacacheObject(ctx context.Context, key string, r *
 		return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key}
 	}
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
+	modTime := UTCNow()
+
 	for i, w := range writers {
 		if w == nil {
 			// Make sure to avoid writing to disks which we couldn't complete in erasure.Encode()
@@ -852,7 +867,7 @@ func (es *erasureSingle) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		partsMetadata[i].Data = inlineBuffers[i].Bytes()
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -860,8 +875,6 @@ func (es *erasureSingle) putMetacacheObject(ctx context.Context, key string, r *
 		})
 	}
 
-	modTime := UTCNow()
-
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
@@ -1078,6 +1091,16 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		defer lk.Unlock(lkctx.Cancel)
 	}
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
+	modTime := opts.MTime
+	if opts.MTime.IsZero() {
+		modTime = UTCNow()
+	}
+
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -1088,7 +1111,7 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		} else {
 			partsMetadata[i].Data = nil
 		}
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, index)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -1104,11 +1127,6 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 		opts.UserDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
-	modTime := opts.MTime
-	if opts.MTime.IsZero() {
-		modTime = UTCNow()
-	}
-
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
@@ -1317,8 +1335,14 @@ func (es *erasureSingle) DeleteObjects(ctx context.Context, bucket string, objec
 
 func (es *erasureSingle) deletePrefix(ctx context.Context, bucket, prefix string) error {
 	dirPrefix := encodeDirObject(prefix)
-	defer es.disk.Delete(ctx, bucket, dirPrefix, true)
-	return es.disk.Delete(ctx, bucket, prefix, true)
+	defer es.disk.Delete(ctx, bucket, dirPrefix, DeleteOptions{
+		Recursive: true,
+		Force:     true,
+	})
+	return es.disk.Delete(ctx, bucket, prefix, DeleteOptions{
+		Recursive: true,
+		Force:     true,
+	})
 }
 
 // DeleteObject - deletes an object, this call doesn't necessary reply
@@ -1912,7 +1936,10 @@ func (es *erasureSingle) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, false)
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{
+				Recursive: false,
+				Force:     false,
+			})
 			return nil
 		}, index)
 	}
@@ -1950,7 +1977,10 @@ func (es *erasureSingle) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		}
 		wait := es.deletedCleanupSleeper.Timer(ctx)
 		if now.Sub(vi.Created) > expiry {
-			disk.Delete(ctx, minioMetaTmpBucket, tmpDir, true)
+			disk.Delete(ctx, minioMetaTmpBucket, tmpDir, DeleteOptions{
+				Recursive: true,
+				Force:     false,
+			})
 		}
 		wait()
 		return nil
@@ -2353,8 +2383,13 @@ func (es *erasureSingle) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	md5hex := r.MD5CurrentHexString()
 
+	var index []byte
+	if opts.IndexCB != nil {
+		index = opts.IndexCB()
+	}
+
 	// Add the current part.
-	fi.AddObjectPart(partID, md5hex, n, data.ActualSize())
+	fi.AddObjectPart(partID, md5hex, n, data.ActualSize(), fi.ModTime, index)
 
 	for i, disk := range onlineDisks {
 		if disk == OfflineDisk {
@@ -2652,6 +2687,7 @@ func (es *erasureSingle) CompleteMultipartUpload(ctx context.Context, bucket str
 			Number:     part.PartNumber,
 			Size:       currentFI.Parts[partIdx].Size,
 			ActualSize: currentFI.Parts[partIdx].ActualSize,
+			Index:      currentFI.Parts[partIdx].Index,
 		}
 	}
 
@@ -2980,7 +3016,7 @@ func (es *erasureSingle) Walk(ctx context.Context, bucket, prefix string, result
 				minDisks:       1,
 				reportNotFound: false,
 				agreed:         loadEntry,
-				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+				partial: func(entries metaCacheEntries, _ []error) {
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						// check if we can get one entry atleast

cmd/erasure.go

@@ -213,11 +213,13 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				}
 			}
 			di.Metrics = &madmin.DiskMetrics{
-				APILatencies: make(map[string]interface{}),
-				APICalls:     make(map[string]uint64),
+				LastMinute: make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),
+				APICalls:   make(map[string]uint64, len(info.Metrics.APICalls)),
 			}
-			for k, v := range info.Metrics.APILatencies {
-				di.Metrics.APILatencies[k] = v
+			for k, v := range info.Metrics.LastMinute {
+				if v.N > 0 {
+					di.Metrics.LastMinute[k] = v.asTimedAction()
+				}
 			}
 			for k, v := range info.Metrics.APICalls {
 				di.Metrics.APICalls[k] = v

cmd/format-disk-cache.go

@@ -359,7 +359,7 @@ func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile
 
 	actualSize := uint64(st.Size())
 	if globalCacheKMS != nil {
-		reader, err = newCacheEncryptReader(readCloser, bucket, object, metadata)
+		reader, err = newCacheEncryptReader(ctx, readCloser, bucket, object, metadata)
 		if err != nil {
 			return err
 		}

cmd/format-erasure.go

@@ -367,7 +367,10 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	tmpFormat := mustGetUUID()
 
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, false)
+	defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, DeleteOptions{
+		Recursive: false,
+		Force:     false,
+	})
 
 	// write to unique file.
 	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatBytes); err != nil {
@@ -643,9 +646,8 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 		}, index)
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
 	// Wait for the routines to finish.
-	return reduceWriteQuorumErrs(ctx, g.Wait(), nil, writeQuorum)
+	return reduceWriteQuorumErrs(ctx, g.Wait(), nil, len(storageDisks))
 }
 
 // relinquishes the underlying connection for all storage disks.
@@ -805,26 +807,13 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 	return getFormatErasureInQuorum(formats)
 }
 
-func getDefaultParityBlocks(drive int) int {
-	switch drive {
-	case 3, 2:
-		return 1
-	case 4, 5:
-		return 2
-	case 6, 7:
-		return 3
-	default:
-		return 4
-	}
-}
-
 // ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
 // It will attempt to read it from env variable and fall back to drives/2.
 func ecDrivesNoConfig(setDriveCount int) int {
 	sc, _ := storageclass.LookupConfig(config.KVS{}, setDriveCount)
 	ecDrives := sc.GetParityForSC(storageclass.STANDARD)
-	if ecDrives <= 0 {
-		ecDrives = getDefaultParityBlocks(setDriveCount)
+	if ecDrives < 0 {
+		ecDrives = storageclass.DefaultParityBlocks(setDriveCount)
 	}
 	return ecDrives
 }

cmd/fs-v1-rwpool.go

@@ -187,6 +187,8 @@ func (fsi *fsIOPool) Create(path string) (wlk *lock.LockedFile, err error) {
 			return nil, errFileAccessDenied
 		case isSysErrIsDir(err):
 			return nil, errIsNotRegular
+		case isSysErrNotDir(err):
+			return nil, errFileAccessDenied
 		case isSysErrPathNotFound(err):
 			return nil, errFileAccessDenied
 		default:

cmd/fs-v1.go

@@ -31,7 +31,6 @@ import (
 	"sort"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -210,7 +209,13 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
 
 // BackendInfo - returns backend information
 func (fs *FSObjects) BackendInfo() madmin.BackendInfo {
-	return madmin.BackendInfo{Type: madmin.FS}
+	return madmin.BackendInfo{
+		Type:             madmin.FS,
+		StandardSCData:   []int{1},
+		StandardSCParity: 0,
+		RRSCData:         []int{1},
+		RRSCParity:       0,
+	}
 }
 
 // LocalStorageInfo - returns underlying storage statistics.
@@ -235,7 +240,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 			},
 		},
 	}
-	storageInfo.Backend.Type = madmin.FS
+	storageInfo.Backend = fs.BackendInfo()
 	return storageInfo, nil
 }
 
@@ -349,6 +354,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 // A partially updated bucket may be returned.
 func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUsageCache) (dataUsageCache, error) {
 	defer close(cache.Info.updates)
+	defer globalScannerMetrics.log(scannerMetricScanBucketDisk, fs.fsPath, bucket)()
 
 	// Get bucket policy
 	// Check if the current bucket has a configured lifecycle policy
@@ -363,6 +369,16 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 	// Load bucket info.
 	cache, err = scanDataFolder(ctx, -1, -1, fs.fsPath, cache, func(item scannerItem) (sizeSummary, error) {
 		bucket, object := item.bucket, item.objectPath()
+		stopFn := globalScannerMetrics.log(scannerMetricScanObject, fs.fsPath, PathJoin(item.bucket, item.objectPath()))
+		defer stopFn()
+
+		var fsMetaBytes []byte
+		done := globalScannerMetrics.timeSize(scannerMetricReadMetadata)
+		defer func() {
+			if done != nil {
+				done(len(fsMetaBytes))
+			}
+		}()
 		fsMetaBytes, err := xioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile))
 		if err != nil && !osIsNotExist(err) {
 			if intDataUpdateTracker.debug {
@@ -391,11 +407,16 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 			}
 			return sizeSummary{}, errSkipFile
 		}
+		done(len(fsMetaBytes))
+		done = nil
+
+		// FS has no "all versions". Increment the counter, though
+		globalScannerMetrics.incNoTime(scannerMetricApplyAll)
 
 		oi := fsMeta.ToObjectInfo(bucket, object, fi)
-		atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
-		atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
+		doneVer := globalScannerMetrics.time(scannerMetricApplyVersion)
 		sz := item.applyActions(ctx, fs, oi, &sizeSummary{})
+		doneVer()
 		if sz >= 0 {
 			return sizeSummary{totalSize: sz, versions: 1}, nil
 		}

cmd/gateway-common.go

@@ -297,6 +297,10 @@ func ErrorRespToObjectError(err error, params ...string) error {
 	}
 
 	switch minioErr.Code {
+	case "PreconditionFailed":
+		err = PreConditionFailed{}
+	case "InvalidRange":
+		err = InvalidRange{}
 	case "BucketAlreadyOwnedByYou":
 		err = BucketAlreadyOwnedByYou{}
 	case "BucketNotEmpty":

cmd/gateway-main.go

@@ -32,7 +32,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -221,10 +220,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Set when gateway is enabled
 	globalIsGateway = true
 
-	// TODO: We need to move this code with globalConfigSys.Init()
-	// for now keep it here such that "s3" gateway layer initializes
-	// itself properly when KMS is set.
-
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
@@ -384,15 +379,5 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.Info("======")
 	}
 
-	// TODO: remove the following line by June 1st.
-	logger.Info(
-		color.RedBold(`
-===================================================================================
-**** WARNING: MinIO Gateway will be removed by June 1st from MinIO repository *****
-
-Please read https://github.com/minio/minio/issues/14331
-===================================================================================
-`))
-
 	<-globalOSSignalCh
 }

cmd/gateway-startup-msg.go

@@ -27,6 +27,12 @@ import (
 
 // Prints the formatted startup message.
 func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
+	if len(globalSubnetConfig.APIKey) == 0 {
+		var builder strings.Builder
+		startupBanner(&builder)
+		logger.Info("\n" + builder.String())
+	}
+
 	strippedAPIEndpoints := stripStandardPorts(apiEndPoints, globalMinioHost)
 	// If cache layer is enabled, print cache capacity.
 	cacheAPI := newCachedObjectLayerFn()
@@ -42,12 +48,6 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 
 	// Prints documentation message.
 	printObjectAPIMsg()
-
-	if globalMinioConsolePortAuto && globalBrowserEnabled {
-		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
-			globalMinioConsolePort)
-		logger.Info(color.RedBold(msg))
-	}
 }
 
 // Prints common server startup message. Prints credential, region and browser access.

cmd/generic-handlers.go

@@ -97,12 +97,25 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
 // Limits body and header to specific allowed maximum limits as per S3/MinIO API requirements.
 func setRequestLimitHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		// Reject unsupported reserved metadata first before validation.
 		if containsReservedMetadata(r.Header) {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL)
 			return
 		}
+
 		if isHTTPHeaderSizeTooLarge(r.Header) {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1)
 			return
@@ -373,7 +386,14 @@ func hasMultipleAuth(r *http.Request) bool {
 // any malicious requests.
 func setRequestValidityHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+
 		if err := hasBadHost(r.Host); err != nil {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (%s)", invalidReq.Description, err)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -383,6 +403,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 
 		// Check for bad components in URL path.
 		if hasBadPathComponent(r.URL.Path) {
+			if ok {
+				tc.funcName = "handler.ValidRequest"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 			return
@@ -391,6 +416,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 		for _, vv := range r.Form {
 			for _, v := range vv {
 				if hasBadPathComponent(v) {
+					if ok {
+						tc.funcName = "handler.ValidRequest"
+						tc.responseRecorder.LogErrBody = true
+					}
+
 					writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 					atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 					return
@@ -398,6 +428,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 			}
 		}
 		if hasMultipleAuth(r) {
+			if ok {
+				tc.funcName = "handler.Auth"
+				tc.responseRecorder.LogErrBody = true
+			}
+
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (request has multiple authentication types, please use one)", invalidReq.Description)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -408,6 +443,11 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 		bucketName, _ := request2BucketObjectName(r)
 		if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
 			if !guessIsRPCReq(r) && !guessIsBrowserReq(r) && !guessIsHealthCheckReq(r) && !guessIsMetricsReq(r) && !isAdminReq(r) {
+				if ok {
+					tc.funcName = "handler.ValidRequest"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
 				return
 			}
@@ -415,8 +455,18 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 		// Deny SSE-C requests if not made over TLS
 		if !globalIsTLS && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
 			if r.Method == http.MethodHead {
+				if ok {
+					tc.funcName = "handler.ValidRequest"
+					tc.responseRecorder.LogErrBody = false
+				}
+
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
 			} else {
+				if ok {
+					tc.funcName = "handler.ValidRequest"
+					tc.responseRecorder.LogErrBody = true
+				}
+
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL)
 			}
 			return

cmd/global-heal.go

@@ -171,6 +171,16 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	healBuckets := make([]string, len(buckets))
 	copy(healBuckets, buckets)
 
+	// Heal all buckets first in this erasure set - this is useful
+	// for new objects upload in different buckets to be successful
+	for _, bucket := range healBuckets {
+		_, err := er.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
+		if err != nil {
+			// Log bucket healing error if any, we shall retry again.
+			logger.LogIf(ctx, err)
+		}
+	}
+
 	var retErr error
 	// Heal all buckets with all objects
 	for _, bucket := range healBuckets {
@@ -189,7 +199,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 		tracker.Object = ""
 		tracker.Bucket = bucket
-		// Heal current bucket
+		// Heal current bucket again in case if it is failed
+		// in the  being of erasure set healing
 		if _, err := er.HealBucket(ctx, bucket, madmin.HealOpts{
 			ScanMode: scanMode,
 		}); err != nil {
@@ -258,8 +269,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				return
 			}
 
+			// erasureObjects layer needs object names to be encoded
+			encodedEntryName := encodeDirObject(entry.name)
+
 			for _, version := range fivs.Versions {
-				if _, err := er.HealObject(ctx, bucket, version.Name,
+				if _, err := er.HealObject(ctx, bucket, encodedEntryName,
 					version.VersionID, madmin.HealOpts{
 						ScanMode: scanMode,
 						Remove:   healDeleteDangling,
@@ -302,7 +316,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			minDisks:       1,
 			reportNotFound: false,
 			agreed:         healEntry,
-			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+			partial: func(entries metaCacheEntries, _ []error) {
 				entry, ok := entries.resolve(&resolver)
 				if !ok {
 					// check if we can get one entry atleast

cmd/globals.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/minio/console/restapi"
+	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/config"
@@ -173,9 +174,8 @@ var (
 	globalMinioAddr = ""
 
 	// MinIO default port, can be changed through command line.
-	globalMinioPort            = GlobalMinioDefaultPort
-	globalMinioConsolePort     = "13333"
-	globalMinioConsolePortAuto = false
+	globalMinioPort        = GlobalMinioDefaultPort
+	globalMinioConsolePort = "13333"
 
 	// Holds the host that was passed using --address
 	globalMinioHost = ""
@@ -230,6 +230,7 @@ var (
 	globalTrace = pubsub.New(8)
 
 	// global Listen system to send S3 API events to registered listeners
+	// Objects are expected to be event.Event
 	globalHTTPListen = pubsub.New(0)
 
 	// global console system to send console logs to
@@ -338,6 +339,8 @@ var (
 
 	globalProxyTransport http.RoundTripper
 
+	globalRemoteTargetTransport http.RoundTripper
+
 	globalDNSCache = &dnscache.Resolver{
 		Timeout: 5 * time.Second,
 	}
@@ -371,6 +374,12 @@ var (
 	globalObjectPerfBucket       = "minio-perf-test-tmp-bucket"
 	globalObjectPerfUserMetadata = "X-Amz-Meta-Minio-Object-Perf" // Clients can set this to bypass S3 API service freeze. Used by object pref tests.
 
+	// MinIO version unix timestamp
+	globalVersionUnix uint64
+
+	// MinIO client
+	globalMinioClient *minio.Client
+
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -116,7 +116,9 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
 		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
 		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
-		logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
+		if globalIsDistErasure {
+			logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
+		}
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
 		if len(globalEndpoints.Hostnames()) > 0 {

cmd/handler-utils.go

@@ -27,7 +27,6 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
-	"net/url"
 	"regexp"
 	"strings"
 
@@ -197,16 +196,6 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
 	return nil
 }
 
-// The Query string for the redirect URL the client is
-// redirected on successful upload.
-func getRedirectPostRawQuery(objInfo ObjectInfo) string {
-	redirectValues := make(url.Values)
-	redirectValues.Set("bucket", objInfo.Bucket)
-	redirectValues.Set("key", objInfo.Name)
-	redirectValues.Set("etag", "\""+objInfo.ETag+"\"")
-	return redirectValues.Encode()
-}
-
 // Returns access credentials in the request Authorization header.
 func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
 	cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)
@@ -357,30 +346,6 @@ func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (fil
 	return filePart, fileName, fileSize, formValues, nil
 }
 
-// Log headers and body.
-func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if globalTrace.NumSubscribers() == 0 {
-			f.ServeHTTP(w, r)
-			return
-		}
-		trace := Trace(f, true, w, r)
-		globalTrace.Publish(trace)
-	}
-}
-
-// Log only the headers.
-func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if globalTrace.NumSubscribers() == 0 {
-			f.ServeHTTP(w, r)
-			return
-		}
-		trace := Trace(f, false, w, r)
-		globalTrace.Publish(trace)
-	}
-}
-
 func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		globalHTTPStats.currentS3Requests.Inc(api)

cmd/http-tracer.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"io"
 	"net"
 	"net/http"
@@ -43,8 +44,6 @@ type recordRequest struct {
 	logBody bool
 	// Internal recording buffer
 	buf bytes.Buffer
-	// request headers
-	headers http.Header
 	// total bytes read including header size
 	bytesRead int
 }
@@ -67,12 +66,8 @@ func (r *recordRequest) Read(p []byte) (n int, err error) {
 	return n, err
 }
 
-func (r *recordRequest) Size() int {
-	sz := r.bytesRead
-	for k, v := range r.headers {
-		sz += len(k) + len(v)
-	}
-	return sz
+func (r *recordRequest) BodySize() int {
+	return r.bytesRead
 }
 
 // Return the bytes that were recorded.
@@ -102,96 +97,161 @@ func getOpName(name string) (op string) {
 	op = strings.TrimSuffix(op, "Handler-fm")
 	op = strings.Replace(op, "objectAPIHandlers", "s3", 1)
 	op = strings.Replace(op, "adminAPIHandlers", "admin", 1)
-	op = strings.Replace(op, "(*webAPIHandlers)", "web", 1)
-	op = strings.Replace(op, "(*storageRESTServer)", "internal", 1)
-	op = strings.Replace(op, "(*peerRESTServer)", "internal", 1)
-	op = strings.Replace(op, "(*lockRESTServer)", "internal", 1)
+	op = strings.Replace(op, "(*storageRESTServer)", "storageR", 1)
+	op = strings.Replace(op, "(*peerRESTServer)", "peer", 1)
+	op = strings.Replace(op, "(*lockRESTServer)", "lockR", 1)
 	op = strings.Replace(op, "(*stsAPIHandlers)", "sts", 1)
-	op = strings.Replace(op, "LivenessCheckHandler", "healthcheck", 1)
-	op = strings.Replace(op, "ReadinessCheckHandler", "healthcheck", 1)
+	op = strings.Replace(op, "ClusterCheckHandler", "health.Cluster", 1)
+	op = strings.Replace(op, "ClusterReadCheckHandler", "health.ClusterRead", 1)
+	op = strings.Replace(op, "LivenessCheckHandler", "health.Liveness", 1)
+	op = strings.Replace(op, "ReadinessCheckHandler", "health.Readiness", 1)
 	op = strings.Replace(op, "-fm", "", 1)
 	return op
 }
 
-// Trace gets trace of http request
-func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Request) madmin.TraceInfo {
-	name := getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
+type contextTraceReqType string
 
-	// Setup a http request body recorder
-	reqHeaders := r.Header.Clone()
-	reqHeaders.Set("Host", r.Host)
-	if len(r.TransferEncoding) == 0 {
-		reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
-	} else {
-		reqHeaders.Set("Transfer-Encoding", strings.Join(r.TransferEncoding, ","))
-	}
+const contextTraceReqKey = contextTraceReqType("request-trace-info")
 
-	reqBodyRecorder := &recordRequest{Reader: r.Body, logBody: logBody, headers: reqHeaders}
-	r.Body = reqBodyRecorder
-
-	now := time.Now().UTC()
-	t := madmin.TraceInfo{TraceType: madmin.TraceHTTP, FuncName: name, Time: now}
-
-	t.NodeName = r.Host
-	if globalIsDistErasure {
-		t.NodeName = globalLocalNodeName
-	}
-
-	if t.NodeName == "" {
-		t.NodeName = globalLocalNodeName
-	}
-
-	// strip only standard port from the host address
-	if host, port, err := net.SplitHostPort(t.NodeName); err == nil {
-		if port == "443" || port == "80" {
-			t.NodeName = host
-		}
-	}
-
-	rq := madmin.TraceRequestInfo{
-		Time:     now,
-		Proto:    r.Proto,
-		Method:   r.Method,
-		RawQuery: redactLDAPPwd(r.URL.RawQuery),
-		Client:   handlers.GetSourceIP(r),
-		Headers:  reqHeaders,
-	}
-
-	path := r.URL.RawPath
-	if path == "" {
-		path = r.URL.Path
-	}
-	rq.Path = path
-
-	rw := logger.NewResponseWriter(w)
-	rw.LogErrBody = true
-	rw.LogAllBody = logBody
-
-	// Execute call.
-	f(rw, r)
-
-	rs := madmin.TraceResponseInfo{
-		Time:       time.Now().UTC(),
-		Headers:    rw.Header().Clone(),
-		StatusCode: rw.StatusCode,
-		Body:       rw.Body(),
-	}
-
-	// Transfer request body
-	rq.Body = reqBodyRecorder.Data()
-
-	if rs.StatusCode == 0 {
-		rs.StatusCode = http.StatusOK
-	}
-
-	t.ReqInfo = rq
-	t.RespInfo = rs
-
-	t.CallStats = madmin.TraceCallStats{
-		Latency:         rs.Time.Sub(rw.StartTime),
-		InputBytes:      reqBodyRecorder.Size(),
-		OutputBytes:     rw.Size(),
-		TimeToFirstByte: rw.TimeToFirstByte,
-	}
-	return t
+// Hold related tracing data of a http request, any handler
+// can modify this struct to modify the trace information .
+type traceCtxt struct {
+	requestRecorder  *recordRequest
+	responseRecorder *logger.ResponseWriter
+	funcName         string
+}
+
+// If trace is enabled, execute the request if it is traced by other handlers
+// otherwise, generate a trace event with request information but no response.
+func httpTracer(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if globalTrace.NumSubscribers(madmin.TraceS3|madmin.TraceInternal) == 0 {
+			h.ServeHTTP(w, r)
+			return
+		}
+
+		// Create tracing data structure and associate it to the request context
+		tc := traceCtxt{}
+		ctx := context.WithValue(r.Context(), contextTraceReqKey, &tc)
+		r = r.WithContext(ctx)
+
+		// Setup a http request and response body recorder
+		reqRecorder := &recordRequest{Reader: r.Body}
+		respRecorder := logger.NewResponseWriter(w)
+
+		tc.requestRecorder = reqRecorder
+		tc.responseRecorder = respRecorder
+
+		// Execute call.
+		r.Body = reqRecorder
+
+		reqStartTime := time.Now().UTC()
+		h.ServeHTTP(respRecorder, r)
+		reqEndTime := time.Now().UTC()
+
+		tt := madmin.TraceInternal
+		if strings.HasPrefix(tc.funcName, "s3.") {
+			tt = madmin.TraceS3
+		}
+		// No need to continue if no subscribers for actual type...
+		if globalTrace.NumSubscribers(tt) == 0 {
+			return
+		}
+		// Calculate input body size with headers
+		reqHeaders := r.Header.Clone()
+		reqHeaders.Set("Host", r.Host)
+		if len(r.TransferEncoding) == 0 {
+			reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
+		} else {
+			reqHeaders.Set("Transfer-Encoding", strings.Join(r.TransferEncoding, ","))
+		}
+		inputBytes := reqRecorder.BodySize()
+		for k, v := range reqHeaders {
+			inputBytes += len(k) + len(v)
+		}
+
+		// Calculate node name
+		nodeName := r.Host
+		if globalIsDistErasure {
+			nodeName = globalLocalNodeName
+		}
+		if host, port, err := net.SplitHostPort(nodeName); err == nil {
+			if port == "443" || port == "80" {
+				nodeName = host
+			}
+		}
+
+		// Calculate reqPath
+		reqPath := r.URL.RawPath
+		if reqPath == "" {
+			reqPath = r.URL.Path
+		}
+
+		// Calculate function name
+		funcName := tc.funcName
+		if funcName == "" {
+			funcName = "<unknown>"
+		}
+
+		t := madmin.TraceInfo{
+			TraceType: tt,
+			FuncName:  funcName,
+			NodeName:  nodeName,
+			Time:      reqStartTime,
+			Duration:  reqEndTime.Sub(respRecorder.StartTime),
+			Path:      reqPath,
+			HTTP: &madmin.TraceHTTPStats{
+				ReqInfo: madmin.TraceRequestInfo{
+					Time:     reqStartTime,
+					Proto:    r.Proto,
+					Method:   r.Method,
+					RawQuery: redactLDAPPwd(r.URL.RawQuery),
+					Client:   handlers.GetSourceIP(r),
+					Headers:  reqHeaders,
+					Path:     reqPath,
+					Body:     reqRecorder.Data(),
+				},
+				RespInfo: madmin.TraceResponseInfo{
+					Time:       reqEndTime,
+					Headers:    respRecorder.Header().Clone(),
+					StatusCode: respRecorder.StatusCode,
+					Body:       respRecorder.Body(),
+				},
+				CallStats: madmin.TraceCallStats{
+					Latency:         reqEndTime.Sub(respRecorder.StartTime),
+					InputBytes:      inputBytes,
+					OutputBytes:     respRecorder.Size(),
+					TimeToFirstByte: respRecorder.TimeToFirstByte,
+				},
+			},
+		}
+
+		globalTrace.Publish(t)
+	})
+}
+
+func httpTrace(f http.HandlerFunc, logBody bool) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt)
+		if !ok {
+			// Tracing is not enabled for this request
+			f.ServeHTTP(w, r)
+			return
+		}
+
+		tc.funcName = getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
+		tc.requestRecorder.logBody = logBody
+		tc.responseRecorder.LogAllBody = logBody
+		tc.responseRecorder.LogErrBody = true
+
+		f.ServeHTTP(w, r)
+	}
+}
+
+func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
+	return httpTrace(f, true)
+}
+
+func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
+	return httpTrace(f, false)
 }

cmd/iam-dummy-store.go

@@ -20,8 +20,6 @@ package cmd
 import (
 	"context"
 	"sync"
-
-	"github.com/minio/minio/internal/auth"
 )
 
 type iamDummyStore struct {
@@ -79,7 +77,7 @@ func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]Polic
 	return nil
 }
 
-func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	u, ok := ids.iamUsersMap[user]
 	if !ok {
 		return errNoSuchUser
@@ -88,7 +86,7 @@ func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IA
 	return nil
 }
 
-func (ids *iamDummyStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ids *iamDummyStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
 	for k, v := range ids.iamUsersMap {
 		m[k] = v
 	}

cmd/iam-etcd-store.go

@@ -336,7 +336,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]Policy
 	return nil
 }
 
-func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]auth.Credentials, basePrefix string) error {
+func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
 	var u UserIdentity
 	err := getIAMConfig(&u, userkv.Value, string(userkv.Key))
 	if err != nil {
@@ -349,7 +349,7 @@ func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue,
 	return ies.addUser(ctx, user, userType, u, m)
 }
 
-func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]UserIdentity) error {
 	if u.Credentials.IsExpired() {
 		// Delete expired identity.
 		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
@@ -359,11 +359,11 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
-	m[user] = u.Credentials
+	m[user] = u
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	var u UserIdentity
 	err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
 	if err != nil {
@@ -375,7 +375,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
 	return ies.addUser(ctx, user, userType, u, m)
 }
 
-func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
 	var basePrefix string
 	switch userType {
 	case svcUser:

cmd/iam-object-store.go

@@ -287,7 +287,7 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]Po
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	var u UserIdentity
 	err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
 	if err != nil {
@@ -308,11 +308,11 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 		u.Credentials.AccessKey = user
 	}
 
-	m[user] = u.Credentials
+	m[user] = u
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error {
 	var basePrefix string
 	switch userType {
 	case svcUser:

cmd/iam-store.go

@@ -249,7 +249,7 @@ type iamCache struct {
 	// map of policy names to policy definitions
 	iamPolicyDocsMap map[string]PolicyDoc
 	// map of usernames to credentials
-	iamUsersMap map[string]auth.Credentials
+	iamUsersMap map[string]UserIdentity
 	// map of group names to group info
 	iamGroupsMap map[string]GroupInfo
 	// map of user names to groups they are a member of
@@ -263,7 +263,7 @@ type iamCache struct {
 func newIamCache() *iamCache {
 	return &iamCache{
 		iamPolicyDocsMap:        map[string]PolicyDoc{},
-		iamUsersMap:             map[string]auth.Credentials{},
+		iamUsersMap:             map[string]UserIdentity{},
 		iamGroupsMap:            map[string]GroupInfo{},
 		iamUserGroupMemberships: map[string]set.StringSet{},
 		iamUserPolicyMap:        map[string]MappedPolicy{},
@@ -346,16 +346,16 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 	var parentName string
 	u, ok := c.iamUsersMap[name]
 	if ok {
-		if !u.IsValid() {
+		if !u.Credentials.IsValid() {
 			return nil, time.Time{}, nil
 		}
-		parentName = u.ParentUser
+		parentName = u.Credentials.ParentUser
 	}
 
 	mp, ok := c.iamUserPolicyMap[name]
 	if !ok {
 		// Service accounts with root credentials, inherit parent permissions
-		if parentName == globalActiveCred.AccessKey && u.IsServiceAccount() {
+		if parentName == globalActiveCred.AccessKey && u.Credentials.IsServiceAccount() {
 			// even if this is set, the claims present in the service
 			// accounts apply the final permissions if any.
 			return []string{"consoleAdmin"}, mp.UpdatedAt, nil
@@ -395,8 +395,8 @@ type IAMStorageAPI interface {
 	getUsersSysType() UsersSysType
 	loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error
 	loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error
-	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error
-	loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error
+	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error
+	loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error
 	loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error
 	loadGroups(ctx context.Context, m map[string]GroupInfo) error
 	loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
@@ -526,12 +526,12 @@ func (store *IAMStoreSys) HasWatcher() bool {
 }
 
 // GetUser - fetches credential from memory.
-func (store *IAMStoreSys) GetUser(user string) (auth.Credentials, bool) {
+func (store *IAMStoreSys) GetUser(user string) (UserIdentity, bool) {
 	cache := store.rlock()
 	defer store.runlock()
 
-	c, ok := cache.iamUsersMap[user]
-	return c, ok
+	u, ok := cache.iamUsersMap[user]
+	return u, ok
 }
 
 // GetMappedPolicy - fetches mapped policy from memory.
@@ -614,9 +614,9 @@ func (store *IAMStoreSys) PolicyDBGet(name string, isGroup bool, groups ...strin
 }
 
 // AddUsersToGroup - adds users to group, creating the group if needed.
-func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, members []string) error {
+func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if group == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -624,12 +624,13 @@ func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, mem
 
 	// Validate that all members exist.
 	for _, member := range members {
-		cr, ok := cache.iamUsersMap[member]
+		u, ok := cache.iamUsersMap[member]
 		if !ok {
-			return errNoSuchUser
+			return updatedAt, errNoSuchUser
 		}
+		cr := u.Credentials
 		if cr.IsTemp() || cr.IsServiceAccount() {
-			return errIAMActionNotAllowed
+			return updatedAt, errIAMActionNotAllowed
 		}
 	}
 
@@ -640,10 +641,11 @@ func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, mem
 		gi = newGroupInfo(members)
 	} else {
 		gi.Members = set.CreateStringSet(append(gi.Members, members...)...).ToSlice()
+		gi.UpdatedAt = UTCNow()
 	}
 
 	if err := store.saveGroupInfo(ctx, group, gi); err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	cache.iamGroupsMap[group] = gi
@@ -660,16 +662,15 @@ func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, mem
 	}
 
 	cache.updatedAt = time.Now()
-
-	return nil
+	return gi.UpdatedAt, nil
 }
 
 // helper function - does not take any locks. Updates only cache if
 // updateCacheOnly is set.
-func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamCache, group string, members []string, updateCacheOnly bool) error {
+func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamCache, group string, members []string, updateCacheOnly bool) (updatedAt time.Time, err error) {
 	gi, ok := cache.iamGroupsMap[group]
 	if !ok {
-		return errNoSuchGroup
+		return updatedAt, errNoSuchGroup
 	}
 
 	s := set.CreateStringSet(gi.Members...)
@@ -679,9 +680,10 @@ func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamC
 	if !updateCacheOnly {
 		err := store.saveGroupInfo(ctx, group, gi)
 		if err != nil {
-			return err
+			return updatedAt, err
 		}
 	}
+	gi.UpdatedAt = UTCNow()
 	cache.iamGroupsMap[group] = gi
 
 	// update user-group membership map
@@ -695,13 +697,13 @@ func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamC
 	}
 
 	cache.updatedAt = time.Now()
-	return nil
+	return gi.UpdatedAt, nil
 }
 
 // RemoveUsersFromGroup - removes users from group, deleting it if it is empty.
-func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) error {
+func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if group == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -709,23 +711,24 @@ func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string
 
 	// Validate that all members exist.
 	for _, member := range members {
-		cr, ok := cache.iamUsersMap[member]
+		u, ok := cache.iamUsersMap[member]
 		if !ok {
-			return errNoSuchUser
+			return updatedAt, errNoSuchUser
 		}
+		cr := u.Credentials
 		if cr.IsTemp() || cr.IsServiceAccount() {
-			return errIAMActionNotAllowed
+			return updatedAt, errIAMActionNotAllowed
 		}
 	}
 
 	gi, ok := cache.iamGroupsMap[group]
 	if !ok {
-		return errNoSuchGroup
+		return updatedAt, errNoSuchGroup
 	}
 
 	// Check if attempting to delete a non-empty group.
 	if len(members) == 0 && len(gi.Members) != 0 {
-		return errGroupNotEmpty
+		return updatedAt, errGroupNotEmpty
 	}
 
 	if len(members) == 0 {
@@ -734,26 +737,26 @@ func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string
 		// Remove the group from storage. First delete the
 		// mapped policy. No-mapped-policy case is ignored.
 		if err := store.deleteMappedPolicy(ctx, group, regUser, true); err != nil && err != errNoSuchPolicy {
-			return err
+			return updatedAt, err
 		}
 		if err := store.deleteGroupInfo(ctx, group); err != nil && err != errNoSuchGroup {
-			return err
+			return updatedAt, err
 		}
 
 		// Delete from server memory
 		delete(cache.iamGroupsMap, group)
 		delete(cache.iamGroupPolicyMap, group)
 		cache.updatedAt = time.Now()
-		return nil
+		return cache.updatedAt, nil
 	}
 
 	return removeMembersFromGroup(ctx, store, cache, group, members, false)
 }
 
 // SetGroupStatus - updates group status
-func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enabled bool) error {
+func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enabled bool) (updatedAt time.Time, err error) {
 	if group == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -761,7 +764,7 @@ func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enab
 
 	gi, ok := cache.iamGroupsMap[group]
 	if !ok {
-		return errNoSuchGroup
+		return updatedAt, errNoSuchGroup
 	}
 
 	if enabled {
@@ -769,15 +772,15 @@ func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enab
 	} else {
 		gi.Status = statusDisabled
 	}
-
+	gi.UpdatedAt = UTCNow()
 	if err := store.saveGroupInfo(ctx, group, gi); err != nil {
-		return err
+		return gi.UpdatedAt, err
 	}
 
 	cache.iamGroupsMap[group] = gi
 	cache.updatedAt = time.Now()
 
-	return nil
+	return gi.UpdatedAt, nil
 }
 
 // GetGroupDescription - builds up group description
@@ -851,9 +854,9 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 
 // PolicyDBSet - update the policy mapping for the given user or group in
 // storage and in cache.
-func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) error {
+func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) (updatedAt time.Time, err error) {
 	if name == "" {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -863,11 +866,11 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 	if store.getUsersSysType() == MinIOUsersSysType {
 		if !isGroup {
 			if _, ok := cache.iamUsersMap[name]; !ok {
-				return errNoSuchUser
+				return updatedAt, errNoSuchUser
 			}
 		} else {
 			if _, ok := cache.iamGroupsMap[name]; !ok {
-				return errNoSuchGroup
+				return updatedAt, errNoSuchGroup
 			}
 		}
 	}
@@ -882,7 +885,7 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 		}
 		err := store.deleteMappedPolicy(ctx, name, userType, isGroup)
 		if err != nil && err != errNoSuchPolicy {
-			return err
+			return updatedAt, err
 		}
 		if !isGroup {
 			delete(cache.iamUserPolicyMap, name)
@@ -890,21 +893,19 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 			delete(cache.iamGroupPolicyMap, name)
 		}
 		cache.updatedAt = time.Now()
-
-		return nil
+		return cache.updatedAt, nil
 	}
 
 	// Handle policy mapping set/update
 	mp := newMappedPolicy(policy)
 	for _, p := range mp.toSlice() {
 		if _, found := cache.iamPolicyDocsMap[p]; !found {
-			logger.LogIf(GlobalContext, fmt.Errorf("%w: (%s)", errNoSuchPolicy, p))
-			return errNoSuchPolicy
+			return updatedAt, errNoSuchPolicy
 		}
 	}
 
 	if err := store.saveMappedPolicy(ctx, name, userType, isGroup, mp); err != nil {
-		return err
+		return updatedAt, err
 	}
 	if !isGroup {
 		cache.iamUserPolicyMap[name] = mp
@@ -912,7 +913,7 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 		cache.iamGroupPolicyMap[name] = mp
 	}
 	cache.updatedAt = time.Now()
-	return nil
+	return mp.UpdatedAt, nil
 }
 
 // PolicyNotificationHandler - loads given policy from storage. If not present,
@@ -1000,12 +1001,6 @@ func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string) error
 		}
 	}
 	if len(users) != 0 || len(groups) != 0 {
-		// error out when a policy could not be deleted as it was in use.
-		loggedErr := fmt.Errorf("policy could not be deleted as it is use (users=%s; groups=%s)",
-			fmt.Sprintf("[%s]", strings.Join(users, ",")),
-			fmt.Sprintf("[%s]", strings.Join(groups, ",")),
-		)
-		logger.LogIf(GlobalContext, loggedErr)
 		return errPolicyInUse
 	}
 
@@ -1068,9 +1063,9 @@ func (store *IAMStoreSys) GetPolicyDoc(name string) (r PolicyDoc, err error) {
 }
 
 // SetPolicy - creates a policy with name.
-func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iampolicy.Policy) error {
+func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iampolicy.Policy) (time.Time, error) {
 	if policy.IsEmpty() || name == "" {
-		return errInvalidArgument
+		return time.Time{}, errInvalidArgument
 	}
 
 	cache := store.lock()
@@ -1087,13 +1082,13 @@ func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iam
 	}
 
 	if err := store.savePolicyDoc(ctx, name, d); err != nil {
-		return err
+		return d.UpdateDate, err
 	}
 
 	cache.iamPolicyDocsMap[name] = d
 	cache.updatedAt = time.Now()
 
-	return nil
+	return d.UpdateDate, nil
 }
 
 // ListPolicies - fetches all policies from storage and updates cache as well.
@@ -1198,7 +1193,8 @@ func (store *IAMStoreSys) GetBucketUsers(bucket string) (map[string]madmin.UserI
 
 	result := map[string]madmin.UserInfo{}
 	for k, v := range cache.iamUsersMap {
-		if v.IsTemp() || v.IsServiceAccount() {
+		c := v.Credentials
+		if c.IsTemp() || c.IsServiceAccount() {
 			continue
 		}
 		var policies []string
@@ -1216,7 +1212,7 @@ func (store *IAMStoreSys) GetBucketUsers(bucket string) (map[string]madmin.UserI
 			result[k] = madmin.UserInfo{
 				PolicyName: matchedPolicies,
 				Status: func() madmin.AccountStatus {
-					if v.IsValid() {
+					if c.IsValid() {
 						return madmin.AccountEnabled
 					}
 					return madmin.AccountDisabled
@@ -1235,7 +1231,9 @@ func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo {
 	defer store.runlock()
 
 	result := map[string]madmin.UserInfo{}
-	for k, v := range cache.iamUsersMap {
+	for k, u := range cache.iamUsersMap {
+		v := u.Credentials
+
 		if v.IsTemp() || v.IsServiceAccount() {
 			continue
 		}
@@ -1281,8 +1279,8 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 		// return that info. Otherwise we return error.
 		var groups []string
 		for _, v := range cache.iamUsersMap {
-			if v.ParentUser == name {
-				groups = v.Groups
+			if v.Credentials.ParentUser == name {
+				groups = v.Credentials.Groups
 				break
 			}
 		}
@@ -1297,11 +1295,11 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 		}, nil
 	}
 
-	cred, found := cache.iamUsersMap[name]
+	ui, found := cache.iamUsersMap[name]
 	if !found {
 		return u, errNoSuchUser
 	}
-
+	cred := ui.Credentials
 	if cred.IsTemp() || cred.IsServiceAccount() {
 		return u, errIAMActionNotAllowed
 	}
@@ -1363,7 +1361,7 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		if store.getUsersSysType() == MinIOUsersSysType {
 			memberOf := cache.iamUserGroupMemberships[accessKey].ToSlice()
 			for _, group := range memberOf {
-				removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, true)
+				_, removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, true)
 				if removeErr == errNoSuchGroup {
 					removeErr = nil
 				}
@@ -1376,11 +1374,11 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		// 2. Remove any derived credentials from memory
 		if userType == regUser {
 			for _, u := range cache.iamUsersMap {
-				if u.IsServiceAccount() && u.ParentUser == accessKey {
-					delete(cache.iamUsersMap, u.AccessKey)
+				if u.Credentials.IsServiceAccount() && u.Credentials.ParentUser == accessKey {
+					delete(cache.iamUsersMap, u.Credentials.AccessKey)
 				}
-				if u.IsTemp() && u.ParentUser == accessKey {
-					delete(cache.iamUsersMap, u.AccessKey)
+				if u.Credentials.IsTemp() && u.Credentials.ParentUser == accessKey {
+					delete(cache.iamUsersMap, u.Credentials.AccessKey)
 				}
 			}
 		}
@@ -1412,8 +1410,9 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 	// This mapping is necessary to ensure that valid credentials
 	// have necessary ParentUser present - this is mainly for only
 	// webIdentity based STS tokens.
-	cred, ok := cache.iamUsersMap[accessKey]
+	u, ok := cache.iamUsersMap[accessKey]
 	if ok {
+		cred := u.Credentials
 		if cred.IsTemp() && cred.ParentUser != "" && cred.ParentUser != globalActiveCred.AccessKey {
 			if _, ok := cache.iamUserPolicyMap[cred.ParentUser]; !ok {
 				cache.iamUserPolicyMap[cred.ParentUser] = cache.iamUserPolicyMap[accessKey]
@@ -1439,7 +1438,7 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	if store.getUsersSysType() == MinIOUsersSysType && userType == regUser {
 		memberOf := cache.iamUserGroupMemberships[accessKey].ToSlice()
 		for _, group := range memberOf {
-			removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, false)
+			_, removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, false)
 			if removeErr != nil {
 				return removeErr
 			}
@@ -1451,7 +1450,8 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	// Delete any STS and service account derived from this credential
 	// first.
 	if userType == regUser {
-		for _, u := range cache.iamUsersMap {
+		for _, ui := range cache.iamUsersMap {
+			u := ui.Credentials
 			if u.IsServiceAccount() && u.ParentUser == accessKey {
 				_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
 				delete(cache.iamUsersMap, u.AccessKey)
@@ -1483,9 +1483,9 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 // SetTempUser - saves temporary (STS) credential to storage and cache. If a
 // policy name is given, it is associated with the parent user specified in the
 // credential.
-func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
+func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) (time.Time, error) {
 	if accessKey == "" || !cred.IsTemp() || cred.IsExpired() || cred.ParentUser == "" {
-		return errInvalidArgument
+		return time.Time{}, errInvalidArgument
 	}
 
 	ttl := int64(cred.Expiration.Sub(UTCNow()).Seconds())
@@ -1498,12 +1498,12 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 		_, combinedPolicyStmt := filterPolicies(cache, mp.Policies, "")
 
 		if combinedPolicyStmt.IsEmpty() {
-			return fmt.Errorf("specified policy %s, not found %w", policyName, errNoSuchPolicy)
+			return time.Time{}, fmt.Errorf("specified policy %s, not found %w", policyName, errNoSuchPolicy)
 		}
 
 		err := store.saveMappedPolicy(ctx, cred.ParentUser, stsUser, false, mp, options{ttl: ttl})
 		if err != nil {
-			return err
+			return time.Time{}, err
 		}
 
 		cache.iamUserPolicyMap[cred.ParentUser] = mp
@@ -1512,14 +1512,14 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 	u := newUserIdentity(cred)
 	err := store.saveUserIdentity(ctx, accessKey, stsUser, u, options{ttl: ttl})
 	if err != nil {
-		return err
+		return time.Time{}, err
 	}
 
-	cache.iamUsersMap[accessKey] = cred
+	cache.iamUsersMap[accessKey] = u
 
 	cache.updatedAt = time.Now()
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // DeleteUsers - given a set of users or access keys, deletes them along with
@@ -1531,8 +1531,10 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 
 	var deleted bool
 	usersToDelete := set.CreateStringSet(users...)
-	for user, cred := range cache.iamUsersMap {
+	for user, ui := range cache.iamUsersMap {
 		userType := regUser
+		cred := ui.Credentials
+
 		if cred.IsServiceAccount() {
 			userType = svcUser
 		} else if cred.IsTemp() {
@@ -1575,7 +1577,8 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 	defer store.runlock()
 
 	res := map[string]ParentUserInfo{}
-	for _, cred := range cache.iamUsersMap {
+	for _, ui := range cache.iamUsersMap {
+		cred := ui.Credentials
 		// Only consider service account or STS credentials with
 		// non-empty session tokens.
 		if !(cred.IsServiceAccount() || cred.IsTemp()) ||
@@ -1633,21 +1636,22 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 }
 
 // SetUserStatus - sets current user status.
-func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) error {
+func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) (updatedAt time.Time, err error) {
 	if accessKey != "" && status != madmin.AccountEnabled && status != madmin.AccountDisabled {
-		return errInvalidArgument
+		return updatedAt, errInvalidArgument
 	}
 
 	cache := store.lock()
 	defer store.unlock()
 
-	cred, ok := cache.iamUsersMap[accessKey]
+	ui, ok := cache.iamUsersMap[accessKey]
 	if !ok {
-		return errNoSuchUser
+		return updatedAt, errNoSuchUser
 	}
+	cred := ui.Credentials
 
 	if cred.IsTemp() || cred.IsServiceAccount() {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
 	uinfo := newUserIdentity(auth.Credentials{
@@ -1663,17 +1667,17 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
 	})
 
 	if err := store.saveUserIdentity(ctx, accessKey, regUser, uinfo); err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[accessKey] = uinfo.Credentials
+	cache.iamUsersMap[accessKey] = uinfo
 	cache.updatedAt = time.Now()
 
-	return nil
+	return uinfo.UpdatedAt, nil
 }
 
 // AddServiceAccount - add a new service account
-func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Credentials) error {
+func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Credentials) (updatedAt time.Time, err error) {
 	cache := store.lock()
 	defer store.unlock()
 
@@ -1683,44 +1687,45 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
 	// Found newly requested service account, to be an existing account -
 	// reject such operation (updates to the service account are handled in
 	// a different API).
-	if scred, found := cache.iamUsersMap[accessKey]; found {
+	if su, found := cache.iamUsersMap[accessKey]; found {
+		scred := su.Credentials
 		if scred.ParentUser != parentUser {
-			return errIAMServiceAccountUsed
+			return updatedAt, errIAMServiceAccountUsed
 		}
-		return errIAMServiceAccount
+		return updatedAt, errIAMServiceAccount
 	}
 
 	// Parent user must not be a service account.
-	if cr, found := cache.iamUsersMap[parentUser]; found && cr.IsServiceAccount() {
-		return errIAMServiceAccount
+	if u, found := cache.iamUsersMap[parentUser]; found && u.Credentials.IsServiceAccount() {
+		return updatedAt, errIAMServiceAccount
 	}
 
 	u := newUserIdentity(cred)
-	err := store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u)
+	err = store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+	cache.iamUsersMap[u.Credentials.AccessKey] = u
 	cache.updatedAt = time.Now()
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // UpdateServiceAccount - updates a service account on storage.
-func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) error {
+func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) (updatedAt time.Time, err error) {
 	cache := store.lock()
 	defer store.unlock()
 
-	cr, ok := cache.iamUsersMap[accessKey]
-	if !ok || !cr.IsServiceAccount() {
-		return errNoSuchServiceAccount
+	ui, ok := cache.iamUsersMap[accessKey]
+	if !ok || !ui.Credentials.IsServiceAccount() {
+		return updatedAt, errNoSuchServiceAccount
 	}
-
+	cr := ui.Credentials
 	currentSecretKey := cr.SecretKey
 	if opts.secretKey != "" {
 		if !auth.IsSecretKeyValid(opts.secretKey) {
-			return auth.ErrInvalidSecretKeyLength
+			return updatedAt, auth.ErrInvalidSecretKeyLength
 		}
 		cr.SecretKey = opts.secretKey
 	}
@@ -1736,12 +1741,12 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 	case auth.AccountOn, auth.AccountOff:
 		cr.Status = opts.status
 	default:
-		return errors.New("unknown account status value")
+		return updatedAt, errors.New("unknown account status value")
 	}
 
 	m, err := getClaimsFromTokenWithSecret(cr.SessionToken, currentSecretKey)
 	if err != nil {
-		return fmt.Errorf("unable to get svc acc claims: %v", err)
+		return updatedAt, fmt.Errorf("unable to get svc acc claims: %v", err)
 	}
 
 	// Extracted session policy name string can be removed as its not useful
@@ -1757,16 +1762,16 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 
 	if opts.sessionPolicy != nil {
 		if err := opts.sessionPolicy.Validate(); err != nil {
-			return err
+			return updatedAt, err
 		}
 
 		policyBuf, err := json.Marshal(opts.sessionPolicy)
 		if err != nil {
-			return err
+			return updatedAt, err
 		}
 
 		if len(policyBuf) > 16*humanize.KiByte {
-			return fmt.Errorf("Session policy should not exceed 16 KiB characters")
+			return updatedAt, fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
 
 		// Overwrite session policy claims.
@@ -1776,41 +1781,41 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 
 	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	u := newUserIdentity(cr)
 	if err := store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u); err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+	cache.iamUsersMap[u.Credentials.AccessKey] = u
 	cache.updatedAt = time.Now()
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // ListTempAccounts - lists only temporary accounts from the cache.
-func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string) ([]UserIdentity, error) {
 	cache := store.rlock()
 	defer store.runlock()
 
 	userExists := false
-	var tempAccounts []auth.Credentials
+	var tempAccounts []UserIdentity
 	for _, v := range cache.iamUsersMap {
 		isDerived := false
-		if v.IsServiceAccount() || v.IsTemp() {
+		if v.Credentials.IsServiceAccount() || v.Credentials.IsTemp() {
 			isDerived = true
 		}
 
-		if !isDerived && v.AccessKey == accessKey {
+		if !isDerived && v.Credentials.AccessKey == accessKey {
 			userExists = true
-		} else if isDerived && v.ParentUser == accessKey {
+		} else if isDerived && v.Credentials.ParentUser == accessKey {
 			userExists = true
-			if v.IsTemp() {
+			if v.Credentials.IsTemp() {
 				// Hide secret key & session key here
-				v.SecretKey = ""
-				v.SessionToken = ""
+				v.Credentials.SecretKey = ""
+				v.Credentials.SessionToken = ""
 				tempAccounts = append(tempAccounts, v)
 			}
 		}
@@ -1830,8 +1835,9 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 
 	userExists := false
 	var serviceAccounts []auth.Credentials
-	for _, v := range cache.iamUsersMap {
+	for _, u := range cache.iamUsersMap {
 		isDerived := false
+		v := u.Credentials
 		if v.IsServiceAccount() || v.IsTemp() {
 			isDerived = true
 		}
@@ -1857,17 +1863,17 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 }
 
 // AddUser - adds/updates long term user account to storage.
-func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
+func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) (updatedAt time.Time, err error) {
 	cache := store.lock()
 	defer store.unlock()
 
 	cache.updatedAt = time.Now()
 
-	cr, ok := cache.iamUsersMap[accessKey]
+	ui, ok := cache.iamUsersMap[accessKey]
 
 	// It is not possible to update an STS account.
-	if ok && cr.IsTemp() {
-		return errIAMActionNotAllowed
+	if ok && ui.Credentials.IsTemp() {
+		return updatedAt, errIAMActionNotAllowed
 	}
 
 	u := newUserIdentity(auth.Credentials{
@@ -1883,12 +1889,12 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
 	})
 
 	if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil {
-		return err
+		return updatedAt, err
 	}
 
-	cache.iamUsersMap[accessKey] = u.Credentials
+	cache.iamUsersMap[accessKey] = u
 
-	return nil
+	return u.UpdatedAt, nil
 }
 
 // UpdateUserSecretKey - sets user secret key to storage.
@@ -1898,18 +1904,18 @@ func (store *IAMStoreSys) UpdateUserSecretKey(ctx context.Context, accessKey, se
 
 	cache.updatedAt = time.Now()
 
-	cred, ok := cache.iamUsersMap[accessKey]
+	ui, ok := cache.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
 	}
-
+	cred := ui.Credentials
 	cred.SecretKey = secretKey
 	u := newUserIdentity(cred)
 	if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil {
 		return err
 	}
 
-	cache.iamUsersMap[accessKey] = cred
+	cache.iamUsersMap[accessKey] = u
 	return nil
 }
 
@@ -1919,7 +1925,8 @@ func (store *IAMStoreSys) GetSTSAndServiceAccounts() []auth.Credentials {
 	defer store.runlock()
 
 	var res []auth.Credentials
-	for _, cred := range cache.iamUsersMap {
+	for _, u := range cache.iamUsersMap {
+		cred := u.Credentials
 		if cred.IsTemp() || cred.IsServiceAccount() {
 			res = append(res, cred)
 		}
@@ -1940,13 +1947,13 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
 	} else if cred.IsTemp() {
 		userType = stsUser
 	}
-
+	ui := newUserIdentity(cred)
 	// Overwrite the user identity here. As store should be
 	// atomic, it shouldn't cause any corruption.
-	if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, newUserIdentity(cred)); err != nil {
+	if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, ui); err != nil {
 		return err
 	}
-	cache.iamUsersMap[cred.AccessKey] = cred
+	cache.iamUsersMap[cred.AccessKey] = ui
 	return nil
 }
 
@@ -1969,9 +1976,9 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 			if svc, found := cache.iamUsersMap[accessKey]; found {
 				// Load parent user and mapped policies.
 				if store.getUsersSysType() == MinIOUsersSysType {
-					store.loadUser(ctx, svc.ParentUser, regUser, cache.iamUsersMap)
+					store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
 				}
-				store.loadMappedPolicy(ctx, svc.ParentUser, regUser, false, cache.iamUserPolicyMap)
+				store.loadMappedPolicy(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap)
 			} else {
 				// check for STS account
 				store.loadUser(ctx, accessKey, stsUser, cache.iamUsersMap)

cmd/iam.go

@@ -223,8 +223,6 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
-	iamInitStart := time.Now()
-
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -297,8 +295,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
-	iamLoadStart := time.Now()
-
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx); err != nil {
@@ -370,9 +366,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	}
 
 	sys.printIAMRoles()
-
-	now := time.Now()
-	logger.Info("Finished loading IAM sub-system (took %.1fs of %.1fs to load data).", now.Sub(iamLoadStart).Seconds(), now.Sub(iamInitStart).Seconds())
 }
 
 func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[arn.ARN]string) {
@@ -606,14 +599,14 @@ func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[s
 }
 
 // SetPolicy - sets a new named policy.
-func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy.Policy) (time.Time, error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return time.Time{}, errServerNotInitialized
 	}
 
-	err := sys.store.SetPolicy(ctx, policyName, p)
+	updatedAt, err := sys.store.SetPolicy(ctx, policyName, p)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	if !sys.HasWatcher() {
@@ -625,7 +618,7 @@ func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy
 			}
 		}
 	}
-	return nil
+	return updatedAt, nil
 }
 
 // DeleteUser - delete user (only for long-term users not STS users).
@@ -714,9 +707,9 @@ func (sys *IAMSys) notifyForUser(ctx context.Context, accessKey string, isTemp b
 // elsewhere), the AssumeRole case (because the parent user is real and their
 // policy is associated via policy-set API) and the AssumeRoleWithLDAP case
 // (because the policy association is made via policy-set API).
-func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
+func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) (time.Time, error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return time.Time{}, errServerNotInitialized
 	}
 
 	if newGlobalAuthZPluginFn() != nil {
@@ -724,14 +717,14 @@ func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.
 		policyName = ""
 	}
 
-	err := sys.store.SetTempUser(ctx, accessKey, cred, policyName)
+	updatedAt, err := sys.store.SetTempUser(ctx, accessKey, cred, policyName)
 	if err != nil {
-		return err
+		return time.Time{}, err
 	}
 
 	sys.notifyForUser(ctx, cred.AccessKey, true)
 
-	return nil
+	return updatedAt, nil
 }
 
 // ListBucketUsers - list all users who can access this 'bucket'
@@ -789,11 +782,11 @@ func (sys *IAMSys) IsTempUser(name string) (bool, string, error) {
 		return false, "", errServerNotInitialized
 	}
 
-	cred, found := sys.store.GetUser(name)
+	u, found := sys.store.GetUser(name)
 	if !found {
 		return false, "", errNoSuchUser
 	}
-
+	cred := u.Credentials
 	if cred.IsTemp() {
 		return true, cred.ParentUser, nil
 	}
@@ -807,11 +800,11 @@ func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
 		return false, "", errServerNotInitialized
 	}
 
-	cred, found := sys.store.GetUser(name)
+	u, found := sys.store.GetUser(name)
 	if !found {
 		return false, "", errNoSuchUser
 	}
-
+	cred := u.Credentials
 	if cred.IsServiceAccount() {
 		return true, cred.ParentUser, nil
 	}
@@ -835,22 +828,22 @@ func (sys *IAMSys) GetUserInfo(ctx context.Context, name string) (u madmin.UserI
 }
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
-func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) error {
+func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.SetUserStatus(ctx, accessKey, status)
+	updatedAt, err = sys.store.SetUserStatus(ctx, accessKey, status)
 	if err != nil {
-		return err
+		return
 	}
 
 	sys.notifyForUser(ctx, accessKey, false)
-	return nil
+	return updatedAt, nil
 }
 
 func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string) {
@@ -874,34 +867,34 @@ type newServiceAccountOpts struct {
 }
 
 // NewServiceAccount - create a new service account
-func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, groups []string, opts newServiceAccountOpts) (auth.Credentials, error) {
+func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, groups []string, opts newServiceAccountOpts) (auth.Credentials, time.Time, error) {
 	if !sys.Initialized() {
-		return auth.Credentials{}, errServerNotInitialized
+		return auth.Credentials{}, time.Time{}, errServerNotInitialized
 	}
 
 	if parentUser == "" {
-		return auth.Credentials{}, errInvalidArgument
+		return auth.Credentials{}, time.Time{}, errInvalidArgument
 	}
 
 	var policyBuf []byte
 	if opts.sessionPolicy != nil {
 		err := opts.sessionPolicy.Validate()
 		if err != nil {
-			return auth.Credentials{}, err
+			return auth.Credentials{}, time.Time{}, err
 		}
 		policyBuf, err = json.Marshal(opts.sessionPolicy)
 		if err != nil {
-			return auth.Credentials{}, err
+			return auth.Credentials{}, time.Time{}, err
 		}
 		if len(policyBuf) > 16*humanize.KiByte {
-			return auth.Credentials{}, fmt.Errorf("Session policy should not exceed 16 KiB characters")
+			return auth.Credentials{}, time.Time{}, fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
 	}
 
 	// found newly requested service account, to be same as
 	// parentUser, reject such operations.
 	if parentUser == opts.accessKey {
-		return auth.Credentials{}, errIAMActionNotAllowed
+		return auth.Credentials{}, time.Time{}, errIAMActionNotAllowed
 	}
 
 	m := make(map[string]interface{})
@@ -929,24 +922,24 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	} else {
 		accessKey, secretKey, err = auth.GenerateCredentials()
 		if err != nil {
-			return auth.Credentials{}, err
+			return auth.Credentials{}, time.Time{}, err
 		}
 	}
 	cred, err := auth.CreateNewCredentialsWithMetadata(accessKey, secretKey, m, secretKey)
 	if err != nil {
-		return auth.Credentials{}, err
+		return auth.Credentials{}, time.Time{}, err
 	}
 	cred.ParentUser = parentUser
 	cred.Groups = groups
 	cred.Status = string(auth.AccountOn)
 
-	err = sys.store.AddServiceAccount(ctx, cred)
+	updatedAt, err := sys.store.AddServiceAccount(ctx, cred)
 	if err != nil {
-		return auth.Credentials{}, err
+		return auth.Credentials{}, time.Time{}, err
 	}
 
 	sys.notifyForServiceAccount(ctx, cred.AccessKey)
-	return cred, nil
+	return cred, updatedAt, nil
 }
 
 type updateServiceAccountOpts struct {
@@ -956,18 +949,18 @@ type updateServiceAccountOpts struct {
 }
 
 // UpdateServiceAccount - edit a service account
-func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) error {
+func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
-	err := sys.store.UpdateServiceAccount(ctx, accessKey, opts)
+	updatedAt, err = sys.store.UpdateServiceAccount(ctx, accessKey, opts)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForServiceAccount(ctx, accessKey)
-	return nil
+	return updatedAt, nil
 }
 
 // ListServiceAccounts - lists all services accounts associated to a specific user
@@ -985,7 +978,7 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 }
 
 // ListTempAccounts - lists all services accounts associated to a specific user
-func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]UserIdentity, error) {
 	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
@@ -1002,32 +995,32 @@ func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]au
 func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
 	sa, embeddedPolicy, err := sys.getServiceAccount(ctx, accessKey)
 	if err != nil {
-		return sa, embeddedPolicy, err
+		return auth.Credentials{}, embeddedPolicy, err
 	}
 	// Hide secret & session keys
-	sa.SecretKey = ""
-	sa.SessionToken = ""
-	return sa, embeddedPolicy, nil
+	sa.Credentials.SecretKey = ""
+	sa.Credentials.SessionToken = ""
+	return sa.Credentials, embeddedPolicy, nil
 }
 
 // getServiceAccount - gets information about a service account
-func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
+func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (u UserIdentity, p *iampolicy.Policy, err error) {
 	if !sys.Initialized() {
-		return auth.Credentials{}, nil, errServerNotInitialized
+		return u, nil, errServerNotInitialized
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.IsServiceAccount() {
-		return auth.Credentials{}, nil, errNoSuchServiceAccount
+	if !ok || !sa.Credentials.IsServiceAccount() {
+		return u, nil, errNoSuchServiceAccount
 	}
 
 	var embeddedPolicy *iampolicy.Policy
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.Credentials.SessionToken, sa.Credentials.SecretKey)
 	if err != nil {
-		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		jwtClaims, err = auth.ExtractClaims(sa.Credentials.SessionToken, globalActiveCred.SecretKey)
 		if err != nil {
-			return auth.Credentials{}, nil, err
+			return u, nil, err
 		}
 	}
 	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
@@ -1035,11 +1028,11 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut
 	if ptok && spok && pt == embeddedPolicyType {
 		policyBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
-			return auth.Credentials{}, nil, err
+			return u, nil, err
 		}
 		embeddedPolicy, err = iampolicy.ParseConfig(bytes.NewReader(policyBytes))
 		if err != nil {
-			return auth.Credentials{}, nil, err
+			return u, nil, err
 		}
 	}
 
@@ -1057,13 +1050,13 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.IsServiceAccount() {
+	if !ok || !sa.Credentials.IsServiceAccount() {
 		return nil, errNoSuchServiceAccount
 	}
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.Credentials.SessionToken, sa.Credentials.SecretKey)
 	if err != nil {
-		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		jwtClaims, err = auth.ExtractClaims(sa.Credentials.SessionToken, globalActiveCred.SecretKey)
 		if err != nil {
 			return nil, err
 		}
@@ -1078,7 +1071,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 	}
 
 	sa, ok := sys.store.GetUser(accessKey)
-	if !ok || !sa.IsServiceAccount() {
+	if !ok || !sa.Credentials.IsServiceAccount() {
 		return nil
 	}
 
@@ -1100,30 +1093,30 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 
 // CreateUser - create new user credentials and policy, if user already exists
 // they shall be rewritten with new inputs.
-func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
+func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
 	if !auth.IsAccessKeyValid(accessKey) {
-		return auth.ErrInvalidAccessKeyLength
+		return updatedAt, auth.ErrInvalidAccessKeyLength
 	}
 
 	if !auth.IsSecretKeyValid(ureq.SecretKey) {
-		return auth.ErrInvalidSecretKeyLength
+		return updatedAt, auth.ErrInvalidSecretKeyLength
 	}
 
-	err := sys.store.AddUser(ctx, accessKey, ureq)
+	updatedAt, err = sys.store.AddUser(ctx, accessKey, ureq)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForUser(ctx, accessKey, false)
-	return nil
+	return updatedAt, nil
 }
 
 // SetUserSecretKey - sets user secret key
@@ -1298,9 +1291,9 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 }
 
 // GetUser - get user credentials
-func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Credentials, ok bool) {
+func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (u UserIdentity, ok bool) {
 	if !sys.Initialized() {
-		return cred, false
+		return u, false
 	}
 
 	fallback := false
@@ -1311,7 +1304,7 @@ func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Cre
 		fallback = true
 	}
 
-	cred, ok = sys.store.GetUser(accessKey)
+	u, ok = sys.store.GetUser(accessKey)
 	if !ok && !fallback {
 		// accessKey not found, also
 		// IAM store is not in fallback mode
@@ -1320,10 +1313,10 @@ func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Cre
 		// exists now. If it doesn't proceed to
 		// fail.
 		sys.store.LoadUser(ctx, accessKey)
-		cred, ok = sys.store.GetUser(accessKey)
+		u, ok = sys.store.GetUser(accessKey)
 	}
 
-	return cred, ok && cred.IsValid()
+	return u, ok && u.Credentials.IsValid()
 }
 
 // Notify all other MinIO peers to load group.
@@ -1340,61 +1333,61 @@ func (sys *IAMSys) notifyForGroup(ctx context.Context, group string) {
 
 // AddUsersToGroup - adds users to a group, creating the group if
 // needed. No error if user(s) already are in the group.
-func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []string) error {
+func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.AddUsersToGroup(ctx, group, members)
+	updatedAt, err = sys.store.AddUsersToGroup(ctx, group, members)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForGroup(ctx, group)
-	return nil
+	return updatedAt, nil
 }
 
 // RemoveUsersFromGroup - remove users from group. If no users are
 // given, and the group is empty, deletes the group as well.
-func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) error {
+func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.RemoveUsersFromGroup(ctx, group, members)
+	updatedAt, err = sys.store.RemoveUsersFromGroup(ctx, group, members)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForGroup(ctx, group)
-	return nil
+	return updatedAt, nil
 }
 
 // SetGroupStatus - enable/disabled a group
-func (sys *IAMSys) SetGroupStatus(ctx context.Context, group string, enabled bool) error {
+func (sys *IAMSys) SetGroupStatus(ctx context.Context, group string, enabled bool) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
+		return updatedAt, errIAMActionNotAllowed
 	}
 
-	err := sys.store.SetGroupStatus(ctx, group, enabled)
+	updatedAt, err = sys.store.SetGroupStatus(ctx, group, enabled)
 	if err != nil {
-		return err
+		return updatedAt, err
 	}
 
 	sys.notifyForGroup(ctx, group)
-	return nil
+	return updatedAt, nil
 }
 
 // GetGroupDescription - builds up group description
@@ -1421,9 +1414,9 @@ func (sys *IAMSys) ListGroups(ctx context.Context) (r []string, err error) {
 }
 
 // PolicyDBSet - sets a policy for a user or group in the PolicyDB.
-func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup bool) error {
+func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup bool) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
-		return errServerNotInitialized
+		return updatedAt, errServerNotInitialized
 	}
 
 	// Determine user-type based on IDP mode.
@@ -1432,9 +1425,9 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup
 		userType = stsUser
 	}
 
-	err := sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
+	updatedAt, err = sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
 	if err != nil {
-		return err
+		return
 	}
 
 	// Notify all other MinIO peers to reload policy
@@ -1447,7 +1440,7 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup
 		}
 	}
 
-	return nil
+	return updatedAt, nil
 }
 
 // PolicyDBGet - gets policy set on a user or group. If a list of groups is
@@ -1597,11 +1590,6 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 			return false
 		}
 		if len(policies) == 0 {
-			// TODO (deprecated in Dec 2021): Only need to handle
-			// behavior for STS credentials created in older
-			// releases. Otherwise, reject such cases, once older
-			// behavior is deprecated.
-
 			// If there is no parent policy mapping, we fall back to
 			// using policy claim from JWT.
 			policySet, ok := args.GetPolicies(iamPolicyClaimNameOpenID())

cmd/jwt.go

@@ -63,11 +63,11 @@ func authenticateJWTUsers(accessKey, secretKey string, expiry time.Duration) (st
 func authenticateJWTUsersWithCredentials(credentials auth.Credentials, expiresAt time.Time) (string, error) {
 	serverCred := globalActiveCred
 	if serverCred.AccessKey != credentials.AccessKey {
-		var ok bool
-		serverCred, ok = globalIAMSys.GetUser(context.TODO(), credentials.AccessKey)
+		u, ok := globalIAMSys.GetUser(context.TODO(), credentials.AccessKey)
 		if !ok {
 			return "", errInvalidAccessKeyID
 		}
+		serverCred = u.Credentials
 	}
 
 	if !serverCred.Equal(credentials) {
@@ -145,10 +145,11 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 		if claims.AccessKey == globalActiveCred.AccessKey {
 			return []byte(globalActiveCred.SecretKey), nil
 		}
-		cred, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
+		u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
 		if !ok {
 			return nil, errInvalidAccessKeyID
 		}
+		cred := u.Credentials
 		return []byte(cred.SecretKey), nil
 	}); err != nil {
 		return claims, nil, false, errAuthentication
@@ -157,11 +158,11 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 	var groups []string
 	if globalActiveCred.AccessKey != claims.AccessKey {
 		// Check if the access key is part of users credentials.
-		ucred, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
+		u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
 		if !ok {
 			return nil, nil, false, errInvalidAccessKeyID
 		}
-
+		ucred := u.Credentials
 		// get embedded claims
 		eclaims, s3Err := checkClaimsFromToken(req, ucred)
 		if s3Err != ErrNone {

cmd/last-minute.go

@@ -21,6 +21,8 @@ package cmd
 
 import (
 	"time"
+
+	"github.com/minio/madmin-go"
 )
 
 const (
@@ -75,29 +77,49 @@ func sizeTagToString(tag int) string {
 // AccElem holds information for calculating an average value
 type AccElem struct {
 	Total int64
+	Size  int64
 	N     int64
 }
 
 // Add a duration to a single element.
 func (a *AccElem) add(dur time.Duration) {
+	if dur < 0 {
+		dur = 0
+	}
 	a.Total += int64(dur)
 	a.N++
 }
 
+// Add a duration to a single element.
+func (a *AccElem) addSize(dur time.Duration, sz int64) {
+	if dur < 0 {
+		dur = 0
+	}
+	a.Total += int64(dur)
+	a.Size += sz
+	a.N++
+}
+
 // Merge b into a.
 func (a *AccElem) merge(b AccElem) {
 	a.N += b.N
 	a.Total += b.Total
+	a.Size += b.Size
 }
 
-// Avg converts total to average.
-func (a AccElem) avg() uint64 {
+// Avg returns average time spent.
+func (a AccElem) avg() time.Duration {
 	if a.N >= 1 && a.Total > 0 {
-		return uint64(a.Total / a.N)
+		return time.Duration(a.Total / a.N)
 	}
 	return 0
 }
 
+// asTimedAction returns the element as a madmin.TimedAction.
+func (a AccElem) asTimedAction() madmin.TimedAction {
+	return madmin.TimedAction{AccTime: uint64(a.Total), Count: uint64(a.N), Bytes: uint64(a.Size)}
+}
+
 // lastMinuteLatency keeps track of last minute latency.
 type lastMinuteLatency struct {
 	Totals  [60]AccElem
@@ -118,6 +140,7 @@ func (l lastMinuteLatency) merge(o lastMinuteLatency) (merged lastMinuteLatency)
 		merged.Totals[i] = AccElem{
 			Total: l.Totals[i].Total + o.Totals[i].Total,
 			N:     l.Totals[i].N + o.Totals[i].N,
+			Size:  l.Totals[i].Size + o.Totals[i].Size,
 		}
 	}
 	return merged
@@ -132,8 +155,17 @@ func (l *lastMinuteLatency) add(t time.Duration) {
 	l.LastSec = sec
 }
 
+// Add  a new duration data
+func (l *lastMinuteLatency) addSize(t time.Duration, sz int64) {
+	sec := time.Now().Unix()
+	l.forwardTo(sec)
+	winIdx := sec % 60
+	l.Totals[winIdx].addSize(t, sz)
+	l.LastSec = sec
+}
+
 // Merge all recorded latencies of last minute into one
-func (l *lastMinuteLatency) getAvgData() AccElem {
+func (l *lastMinuteLatency) getTotal() AccElem {
 	var res AccElem
 	sec := time.Now().Unix()
 	l.forwardTo(sec)
@@ -160,11 +192,11 @@ func (l *lastMinuteLatency) forwardTo(t int64) {
 	}
 }
 
-// LastMinuteLatencies keeps track of last minute latencies.
-type LastMinuteLatencies [sizeLastElemMarker]lastMinuteLatency
+// LastMinuteHistogram keeps track of last minute sizes added.
+type LastMinuteHistogram [sizeLastElemMarker]lastMinuteLatency
 
-// Merge safely merges two LastMinuteLatencies structures into one
-func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLatencies) {
+// Merge safely merges two LastMinuteHistogram structures into one
+func (l LastMinuteHistogram) Merge(o LastMinuteHistogram) (merged LastMinuteHistogram) {
 	for i := range l {
 		merged[i] = l[i].merge(o[i])
 	}
@@ -172,16 +204,16 @@ func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLate
 }
 
 // Add latency t from object with the specified size.
-func (l *LastMinuteLatencies) Add(size int64, t time.Duration) {
+func (l *LastMinuteHistogram) Add(size int64, t time.Duration) {
 	l[sizeToTag(size)].add(t)
 }
 
 // GetAvgData will return the average for each bucket from the last time minute.
 // The number of objects is also included.
-func (l *LastMinuteLatencies) GetAvgData() [sizeLastElemMarker]AccElem {
+func (l *LastMinuteHistogram) GetAvgData() [sizeLastElemMarker]AccElem {
 	var res [sizeLastElemMarker]AccElem
 	for i, elem := range l[:] {
-		res[i] = elem.getAvgData()
+		res[i] = elem.getTotal()
 	}
 	return res
 }

cmd/last-minute_gen.go

@@ -30,6 +30,12 @@ func (z *AccElem) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Total")
 				return
 			}
+		case "Size":
+			z.Size, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
 		case "N":
 			z.N, err = dc.ReadInt64()
 			if err != nil {
@@ -49,9 +55,9 @@ func (z *AccElem) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z AccElem) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
+	// map header, size 3
 	// write "Total"
-	err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+	err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 	if err != nil {
 		return
 	}
@@ -60,6 +66,16 @@ func (z AccElem) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Total")
 		return
 	}
+	// write "Size"
+	err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
 	// write "N"
 	err = en.Append(0xa1, 0x4e)
 	if err != nil {
@@ -76,10 +92,13 @@ func (z AccElem) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z AccElem) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
+	// map header, size 3
 	// string "Total"
-	o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+	o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 	o = msgp.AppendInt64(o, z.Total)
+	// string "Size"
+	o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.Size)
 	// string "N"
 	o = append(o, 0xa1, 0x4e)
 	o = msgp.AppendInt64(o, z.N)
@@ -110,6 +129,12 @@ func (z *AccElem) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Total")
 				return
 			}
+		case "Size":
+			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
 		case "N":
 			z.N, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
@@ -130,12 +155,12 @@ func (z *AccElem) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z AccElem) Msgsize() (s int) {
-	s = 1 + 6 + msgp.Int64Size + 2 + msgp.Int64Size
+	s = 1 + 6 + msgp.Int64Size + 5 + msgp.Int64Size + 2 + msgp.Int64Size
 	return
 }
 
 // DecodeMsg implements msgp.Decodable
-func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
+func (z *LastMinuteHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
 	var zb0001 uint32
 	zb0001, err = dc.ReadArrayHeader()
 	if err != nil {
@@ -195,6 +220,12 @@ func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
 								err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
 								return
 							}
+						case "Size":
+							z[za0001].Totals[za0002].Size, err = dc.ReadInt64()
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "Size")
+								return
+							}
 						case "N":
 							z[za0001].Totals[za0002].N, err = dc.ReadInt64()
 							if err != nil {
@@ -229,7 +260,7 @@ func (z *LastMinuteLatencies) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
+func (z *LastMinuteHistogram) EncodeMsg(en *msgp.Writer) (err error) {
 	err = en.WriteArrayHeader(uint32(sizeLastElemMarker))
 	if err != nil {
 		err = msgp.WrapError(err)
@@ -248,9 +279,9 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 		for za0002 := range z[za0001].Totals {
-			// map header, size 2
+			// map header, size 3
 			// write "Total"
-			err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+			err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 			if err != nil {
 				return
 			}
@@ -259,6 +290,16 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 				err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
 				return
 			}
+			// write "Size"
+			err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+			if err != nil {
+				return
+			}
+			err = en.WriteInt64(z[za0001].Totals[za0002].Size)
+			if err != nil {
+				err = msgp.WrapError(err, za0001, "Totals", za0002, "Size")
+				return
+			}
 			// write "N"
 			err = en.Append(0xa1, 0x4e)
 			if err != nil {
@@ -285,7 +326,7 @@ func (z *LastMinuteLatencies) EncodeMsg(en *msgp.Writer) (err error) {
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *LastMinuteHistogram) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	o = msgp.AppendArrayHeader(o, uint32(sizeLastElemMarker))
 	for za0001 := range z {
@@ -294,10 +335,13 @@ func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
 		o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
 		o = msgp.AppendArrayHeader(o, uint32(60))
 		for za0002 := range z[za0001].Totals {
-			// map header, size 2
+			// map header, size 3
 			// string "Total"
-			o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+			o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].Total)
+			// string "Size"
+			o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].Size)
 			// string "N"
 			o = append(o, 0xa1, 0x4e)
 			o = msgp.AppendInt64(o, z[za0001].Totals[za0002].N)
@@ -310,7 +354,7 @@ func (z *LastMinuteLatencies) MarshalMsg(b []byte) (o []byte, err error) {
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
+func (z *LastMinuteHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	var zb0001 uint32
 	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
@@ -370,6 +414,12 @@ func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
 								err = msgp.WrapError(err, za0001, "Totals", za0002, "Total")
 								return
 							}
+						case "Size":
+							z[za0001].Totals[za0002].Size, bts, err = msgp.ReadInt64Bytes(bts)
+							if err != nil {
+								err = msgp.WrapError(err, za0001, "Totals", za0002, "Size")
+								return
+							}
 						case "N":
 							z[za0001].Totals[za0002].N, bts, err = msgp.ReadInt64Bytes(bts)
 							if err != nil {
@@ -405,8 +455,8 @@ func (z *LastMinuteLatencies) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z *LastMinuteLatencies) Msgsize() (s int) {
-	s = msgp.ArrayHeaderSize + (sizeLastElemMarker * (16 + (60 * (9 + msgp.Int64Size + msgp.Int64Size)) + msgp.Int64Size))
+func (z *LastMinuteHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (sizeLastElemMarker * (16 + (60 * (14 + msgp.Int64Size + msgp.Int64Size + msgp.Int64Size)) + msgp.Int64Size))
 	return
 }
 
@@ -460,6 +510,12 @@ func (z *lastMinuteLatency) DecodeMsg(dc *msgp.Reader) (err error) {
 							err = msgp.WrapError(err, "Totals", za0001, "Total")
 							return
 						}
+					case "Size":
+						z.Totals[za0001].Size, err = dc.ReadInt64()
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001, "Size")
+							return
+						}
 					case "N":
 						z.Totals[za0001].N, err = dc.ReadInt64()
 						if err != nil {
@@ -506,9 +562,9 @@ func (z *lastMinuteLatency) EncodeMsg(en *msgp.Writer) (err error) {
 		return
 	}
 	for za0001 := range z.Totals {
-		// map header, size 2
+		// map header, size 3
 		// write "Total"
-		err = en.Append(0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+		err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 		if err != nil {
 			return
 		}
@@ -517,6 +573,16 @@ func (z *lastMinuteLatency) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "Totals", za0001, "Total")
 			return
 		}
+		// write "Size"
+		err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+		if err != nil {
+			return
+		}
+		err = en.WriteInt64(z.Totals[za0001].Size)
+		if err != nil {
+			err = msgp.WrapError(err, "Totals", za0001, "Size")
+			return
+		}
 		// write "N"
 		err = en.Append(0xa1, 0x4e)
 		if err != nil {
@@ -549,10 +615,13 @@ func (z *lastMinuteLatency) MarshalMsg(b []byte) (o []byte, err error) {
 	o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(60))
 	for za0001 := range z.Totals {
-		// map header, size 2
+		// map header, size 3
 		// string "Total"
-		o = append(o, 0x82, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
+		o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c)
 		o = msgp.AppendInt64(o, z.Totals[za0001].Total)
+		// string "Size"
+		o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+		o = msgp.AppendInt64(o, z.Totals[za0001].Size)
 		// string "N"
 		o = append(o, 0xa1, 0x4e)
 		o = msgp.AppendInt64(o, z.Totals[za0001].N)
@@ -613,6 +682,12 @@ func (z *lastMinuteLatency) UnmarshalMsg(bts []byte) (o []byte, err error) {
 							err = msgp.WrapError(err, "Totals", za0001, "Total")
 							return
 						}
+					case "Size":
+						z.Totals[za0001].Size, bts, err = msgp.ReadInt64Bytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Totals", za0001, "Size")
+							return
+						}
 					case "N":
 						z.Totals[za0001].N, bts, err = msgp.ReadInt64Bytes(bts)
 						if err != nil {
@@ -648,6 +723,6 @@ func (z *lastMinuteLatency) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *lastMinuteLatency) Msgsize() (s int) {
-	s = 1 + 7 + msgp.ArrayHeaderSize + (60 * (9 + msgp.Int64Size + msgp.Int64Size)) + 8 + msgp.Int64Size
+	s = 1 + 7 + msgp.ArrayHeaderSize + (60 * (14 + msgp.Int64Size + msgp.Int64Size + msgp.Int64Size)) + 8 + msgp.Int64Size
 	return
 }

cmd/last-minute_gen_test.go

@@ -122,8 +122,8 @@ func BenchmarkDecodeAccElem(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalLastMinuteLatencies(t *testing.T) {
-	v := LastMinuteLatencies{}
+func TestMarshalUnmarshalLastMinuteHistogram(t *testing.T) {
+	v := LastMinuteHistogram{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
@@ -145,8 +145,8 @@ func TestMarshalUnmarshalLastMinuteLatencies(t *testing.T) {
 	}
 }
 
-func BenchmarkMarshalMsgLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkMarshalMsgLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -154,8 +154,8 @@ func BenchmarkMarshalMsgLastMinuteLatencies(b *testing.B) {
 	}
 }
 
-func BenchmarkAppendMsgLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkAppendMsgLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
@@ -166,8 +166,8 @@ func BenchmarkAppendMsgLastMinuteLatencies(b *testing.B) {
 	}
 }
 
-func BenchmarkUnmarshalLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkUnmarshalLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
@@ -180,17 +180,17 @@ func BenchmarkUnmarshalLastMinuteLatencies(b *testing.B) {
 	}
 }
 
-func TestEncodeDecodeLastMinuteLatencies(t *testing.T) {
-	v := LastMinuteLatencies{}
+func TestEncodeDecodeLastMinuteHistogram(t *testing.T) {
+	v := LastMinuteHistogram{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 
 	m := v.Msgsize()
 	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeLastMinuteLatencies Msgsize() is inaccurate")
+		t.Log("WARNING: TestEncodeDecodeLastMinuteHistogram Msgsize() is inaccurate")
 	}
 
-	vn := LastMinuteLatencies{}
+	vn := LastMinuteHistogram{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
@@ -204,8 +204,8 @@ func TestEncodeDecodeLastMinuteLatencies(t *testing.T) {
 	}
 }
 
-func BenchmarkEncodeLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkEncodeLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
@@ -218,8 +218,8 @@ func BenchmarkEncodeLastMinuteLatencies(b *testing.B) {
 	en.Flush()
 }
 
-func BenchmarkDecodeLastMinuteLatencies(b *testing.B) {
-	v := LastMinuteLatencies{}
+func BenchmarkDecodeLastMinuteHistogram(b *testing.B) {
+	v := LastMinuteHistogram{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))

cmd/listen-notification-handlers.go

@@ -25,7 +25,8 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
-	policy "github.com/minio/pkg/bucket/policy"
+	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r *http.Request) {
@@ -100,13 +101,14 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 	pattern := event.NewPattern(prefix, suffix)
 
 	var eventNames []event.Name
+	var mask pubsub.Mask
 	for _, s := range values[peerRESTListenEvents] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-
+		mask.MergeMaskable(eventName)
 		eventNames = append(eventNames, eventName)
 	}
 
@@ -123,11 +125,11 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	listenCh := make(chan interface{}, 4000)
+	listenCh := make(chan pubsub.Maskable, 4000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
-	listenFn := func(evI interface{}) bool {
+	err := globalHTTPListen.Subscribe(mask, listenCh, ctx.Done(), func(evI pubsub.Maskable) bool {
 		ev, ok := evI.(event.Event)
 		if !ok {
 			return false
@@ -138,14 +140,11 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 			}
 		}
 		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
-	}
-
-	err := globalHTTPListen.Subscribe(listenCh, ctx.Done(), listenFn)
+	})
 	if err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
 		return
 	}
-
 	if bucketName != "" {
 		values.Set(peerRESTListenBucket, bucketName)
 	}
@@ -173,7 +172,10 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 					return
 				}
 			}
-			w.(http.Flusher).Flush()
+			if len(listenCh) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
+			}
 		case <-keepAliveTicker.C:
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return

cmd/main.go

@@ -19,10 +19,12 @@ package cmd
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
 	"sort"
+	"strings"
 
 	"github.com/minio/cli"
 	"github.com/minio/minio/internal/color"
@@ -163,11 +165,23 @@ func newApp(name string) *cli.App {
 	return app
 }
 
+func startupBanner(banner io.Writer) {
+	fmt.Fprintln(banner, color.Blue("Copyright:")+color.Bold(" 2015-%s MinIO, Inc.", CopyrightYear))
+	fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
+	fmt.Fprintln(banner, color.Blue("Version:")+color.Bold(" %s (%s %s/%s)", ReleaseTag, runtime.Version(), runtime.GOOS, runtime.GOARCH))
+}
+
+func versionBanner(c *cli.Context) io.Reader {
+	banner := &strings.Builder{}
+	fmt.Fprintln(banner, color.Bold("%s version %s (commit-id=%s)", c.App.Name, c.App.Version, CommitID))
+	fmt.Fprintln(banner, color.Blue("Runtime:")+color.Bold(" %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH))
+	fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
+	fmt.Fprintln(banner, color.Blue("Copyright:")+color.Bold(" 2015-%s MinIO, Inc.", CopyrightYear))
+	return strings.NewReader(banner.String())
+}
+
 func printMinIOVersion(c *cli.Context) {
-	fmt.Fprintln(c.App.Writer, color.Greenf("%s version %s (commit-id=%s)", c.App.Name, c.App.Version, CommitID))
-	fmt.Fprintln(c.App.Writer, color.Greenf("Runtime: %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH))
-	fmt.Fprintln(c.App.Writer, color.Greenf("Copyright (c) 2015-%s MinIO, Inc.", CopyrightYear))
-	fmt.Fprintln(c.App.Writer, color.Red("Licence AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
+	io.Copy(c.App.Writer, versionBanner(c))
 }
 
 // Main main for minio server.

cmd/metacache-entries.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"os"
 	"sort"
 	"strings"
@@ -330,7 +331,9 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 		// shallow decode.
 		xl, err := entry.xlmeta()
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			if !errors.Is(err, errFileNotFound) {
+				logger.LogIf(GlobalContext, err)
+			}
 			continue
 		}
 		objsValid++