fix(AuditLog): panic while st is nil (#13510)

Also show replica stats with an ARN in Prometheus output.

Dockerfile.release

@@ -18,7 +18,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=$PATH:/opt/bin
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
@@ -30,11 +31,12 @@ RUN \
      microdnf install curl ca-certificates shadow-utils util-linux iproute iputils --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /usr/bin/minio.minisig && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
      microdnf clean all && \
-     chmod +x /usr/bin/minio && \
+     chmod +x /opt/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
      /usr/bin/verify-minio.sh

Dockerfile.release.fips

@@ -18,7 +18,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=$PATH:/opt/bin
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
@@ -30,11 +31,12 @@ RUN \
      microdnf install curl ca-certificates shadow-utils util-linux iproute iputils --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /usr/bin/minio.minisig && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
      microdnf clean all && \
-     chmod +x /usr/bin/minio && \
+     chmod +x /opt/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
      /usr/bin/verify-minio.sh

Dockerfile.scratch

@@ -0,0 +1,5 @@
+FROM scratch
+
+COPY minio /minio
+
+CMD ["/minio"]

Makefile

@@ -10,33 +10,35 @@ TAG ?= "minio/minio:$(VERSION)"
 
 all: build
 
-checks:
+checks: ## check dependencies
 	@echo "Checking dependencies"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 
-getdeps:
+help: ## print this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.40.1
 	@which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3)
 	@which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer)
 
-crosscompile:
+crosscompile: ## cross compile minio
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
 verifiers: getdeps lint check-gen
 
-check-gen:
+check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
-lint:
+lint: ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
 
-# Builds minio, runs the verifiers then runs the tests.
 check: test
-test: verifiers build
+test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@GOGC=25 GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
@@ -44,20 +46,17 @@ test-race: verifiers build
 	@echo "Running unit tests under -race"
 	@(env bash $(PWD)/buildscripts/race.sh)
 
-# Verify minio binary
-verify:
+verify: ## verify minio various setups
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
-# Verify healing of disks with minio binary
-verify-healing:
+verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
-# Builds minio locally.
-build: checks
+build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
@@ -65,23 +64,22 @@ hotfix-vars:
 	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
 	$(eval TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)")
-hotfix: hotfix-vars install
+hotfix: hotfix-vars install ## builds minio binary with hotfix tags
 
-docker-hotfix: hotfix checks
+docker-hotfix: hotfix checks ## builds minio docker container with hotfix tags
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -t $(TAG) . -f Dockerfile.dev
 
-docker: build checks
+docker: build checks ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -t $(TAG) . -f Dockerfile.dev
 
-# Builds minio and installs it to $GOPATH/bin.
-install: build
+install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
-clean:
+clean: ## cleanup all generated assets
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@find . -name '*~' | xargs rm -fv

README.md

@@ -21,10 +21,8 @@ for more complete documentation.
 Run the following command to run the latest stable image of MinIO as a container using an ephemeral data volume:
 
 ```sh
-podman run \
-  -p 9000:9000 \
-  -p 9001:9001 \
-  minio/minio server /data --console-address ":9001"
+podman run -p 9000:9000 -p 9001:9001 \
+  quay.io/minio/minio server /data --console-address ":9001"
 ```
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded
@@ -259,5 +257,5 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 # License
-MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file. 
+MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
 MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).

buildscripts/verify-build.sh

@@ -20,6 +20,7 @@ export SECRET_KEY="minio123"
 export ENABLE_HTTPS=0
 export GO111MODULE=on
 export GOGC=25
+export ENABLE_ADMIN=1
 
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )

buildscripts/verify-healing.sh

@@ -1,7 +1,6 @@
-#!/bin/bash
+#!/bin/bash -e
 #
 
-set -e
 set -E
 set -o pipefail
 
@@ -14,8 +13,6 @@ WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
 
-export GOGC=25
-
 function start_minio_3_node() {
     export MINIO_ROOT_USER=minio
     export MINIO_ROOT_PASSWORD=minio123
@@ -88,22 +85,22 @@ function __init__()
 }
 
 function perform_test() {
-    start_minio_3_node 60
+    start_minio_3_node 120
 
     echo "Testing Distributed Erasure setup healing of drives"
     echo "Remove the contents of the disks belonging to '${1}' erasure set"
 
     rm -rf ${WORK_DIR}/${1}/*/
 
-    start_minio_3_node 60
+    start_minio_3_node 120
 
     rv=$(check_online)
     if [ "$rv" == "1" ]; then
-        pkill -9 minio
         for i in $(seq 1 3); do
             echo "server$i log:"
             cat "${WORK_DIR}/dist-minio-server$i.log"
         done
+        pkill -9 minio
         echo "FAILED"
         purge "$WORK_DIR"
         exit 1

cmd/admin-handlers-config-kv.go

@@ -117,7 +117,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
+	if err = validateConfig(cfg); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -248,7 +248,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
+	if err = validateConfig(cfg); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -360,7 +360,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
+	if err = validateConfig(cfg); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -417,6 +417,8 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 				off = !openid.Enabled(kv)
 			case config.IdentityLDAPSubSys:
 				off = !xldap.Enabled(kv)
+			case config.IdentityTLSSubSys:
+				off = !globalSTSTLSConfig.Enabled
 			}
 			if off {
 				s.WriteString(config.KvComment)

cmd/admin-handlers-site-replication.go

@@ -0,0 +1,315 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
+
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
+func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationAdd")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var sites []madmin.PeerSite
+	errCode := readJSONBody(ctx, r.Body, &sites, cred.SecretKey)
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	status, errInfo := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	if errInfo.Code != ErrNone {
+		logger.LogIf(ctx, errInfo)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(status)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SRInternalJoin - PUT /minio/admin/v3/site-replication/join
+//
+// used internally to tell current cluster to enable SR with
+// the provided peer clusters and service account.
+func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalJoin")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var joinArg madmin.SRInternalJoinReq
+	errCode := readJSONBody(ctx, r.Body, &joinArg, cred.SecretKey)
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	errInfo := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg)
+	if errInfo.Code != ErrNone {
+		logger.LogIf(ctx, errInfo)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+		return
+	}
+}
+
+// SRInternalBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
+func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalBucketOps")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	operation := madmin.BktOp(vars["operation"])
+
+	var err error
+	switch operation {
+	case madmin.MakeWithVersioningBktOp:
+		_, isLockEnabled := r.Form["lockEnabled"]
+		_, isVersioningEnabled := r.Form["versioningEnabled"]
+		opts := BucketOptions{
+			Location:          r.Form.Get("location"),
+			LockEnabled:       isLockEnabled,
+			VersioningEnabled: isVersioningEnabled,
+		}
+		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
+	case madmin.ConfigureReplBktOp:
+		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
+	case madmin.DeleteBucketBktOp:
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
+	case madmin.ForceDeleteBucketBktOp:
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+
+}
+
+// SRInternalReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
+func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var item madmin.SRIAMItem
+	errCode := readJSONBody(ctx, r.Body, &item, "")
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	var err error
+	switch item.Type {
+	case madmin.SRIAMItemPolicy:
+		var policy *iampolicy.Policy
+		if len(item.Policy) > 0 {
+			policy, err = iampolicy.ParseConfig(bytes.NewReader(item.Policy))
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+		}
+		err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+	case madmin.SRIAMItemSvcAcc:
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, *item.SvcAccChange)
+	case madmin.SRIAMItemPolicyMapping:
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, *item.PolicyMapping)
+	case madmin.SRIAMItemSTSAcc:
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, *item.STSCredential)
+
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+}
+
+// SRInternalReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var item madmin.SRBucketMeta
+	errCode := readJSONBody(ctx, r.Body, &item, "")
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	var err error
+	switch item.Type {
+	case madmin.SRBucketMetaTypePolicy:
+		var bktPolicy *policy.Policy
+		if len(item.Policy) > 0 {
+			bktPolicy, err = policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+		}
+		err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+	case madmin.SRBucketMetaTypeTags:
+		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
+	case madmin.SRBucketMetaTypeObjectLockConfig:
+		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
+	case madmin.SRBucketMetaTypeSSEConfig:
+		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
+
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+}
+
+// SiteReplicationDisable - PUT /minio/admin/v3/site-replication/disable
+func (a adminAPIHandlers) SiteReplicationDisable(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationDisable")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationDisableAction)
+	if objectAPI == nil {
+		return
+	}
+}
+
+// SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
+func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationInfo")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	if objectAPI == nil {
+		return
+	}
+
+	info, err := globalSiteReplicationSys.GetClusterInfo(ctx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(info); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	idpSettings := globalSiteReplicationSys.GetIDPSettings(ctx)
+	if err := json.NewEncoder(w).Encode(idpSettings); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+func readJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) APIErrorCode {
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		return ErrInvalidRequest
+	}
+
+	if encryptionKey != "" {
+		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return ErrInvalidRequest
+		}
+	}
+
+	err = json.Unmarshal(data, v)
+	if err != nil {
+		return ErrAdminConfigBadJSON
+	}
+
+	return ErrNone
+}

cmd/admin-handlers-users.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -216,7 +217,6 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	defer r.Body.Close()
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
@@ -242,10 +242,12 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Notify all other MinIO peers to load group.
-	for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
 		}
 	}
 }
@@ -334,10 +336,12 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Notify all other MinIO peers to reload user.
-	for _, nerr := range globalNotificationSys.LoadGroup(group) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadGroup(group) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
 		}
 	}
 }
@@ -369,10 +373,12 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Notify all other MinIO peers to reload user.
-	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
 		}
 	}
 }
@@ -477,10 +483,12 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Notify all other Minio peers to reload user
-	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
 		}
 	}
 }
@@ -522,13 +530,63 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		targetGroups []string
 	)
 
+	// If the request did not set a TargetUser, the service account is
+	// created for the request sender.
 	targetUser = createReq.TargetUser
+	if targetUser == "" {
+		targetUser = cred.AccessKey
+	}
 
-	// Need permission if we are creating a service acccount
-	// for a user <> to the request sender
-	if targetUser != "" && targetUser != cred.AccessKey {
+	opts := newServiceAccountOpts{
+		accessKey: createReq.AccessKey,
+		secretKey: createReq.SecretKey,
+		claims:    make(map[string]interface{}),
+	}
+
+	// Find the user for the request sender (as it may be sent via a service
+	// account or STS account):
+	requestorUser := cred.AccessKey
+	requestorParentUser := cred.AccessKey
+	requestorGroups := cred.Groups
+	requestorIsDerivedCredential := false
+	if cred.IsServiceAccount() || cred.IsTemp() {
+		requestorParentUser = cred.ParentUser
+		requestorIsDerivedCredential = true
+	}
+
+	// Check if we are creating svc account for request sender.
+	isSvcAccForRequestor := false
+	if targetUser == requestorUser || targetUser == requestorParentUser {
+		isSvcAccForRequestor = true
+	}
+
+	// If we are creating svc account for request sender, ensure
+	// that targetUser is a real user (i.e. not derived
+	// credentials).
+	if isSvcAccForRequestor {
+		if requestorIsDerivedCredential {
+			if requestorParentUser == "" {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
+					errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
+				return
+			}
+			targetUser = requestorParentUser
+		}
+		targetGroups = requestorGroups
+
+		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
+		// it is associated with the LDAP/OIDC user properly.
+		for k, v := range cred.Claims {
+			if k == expClaim {
+				continue
+			}
+			opts.claims[k] = v
+		}
+	} else {
+		// Need permission if we are creating a service acccount for a
+		// user <> to the request sender
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     cred.AccessKey,
+			AccountName:     requestorUser,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
@@ -537,36 +595,23 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
 		}
-	}
 
-	var ldapUsername string
-	if globalLDAPConfig.Enabled && targetUser != "" {
-		// If LDAP enabled, service accounts need
-		// to be created only for LDAP users.
-		var err error
-		ldapUsername = targetUser
-		targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		// targerUser is set to bindDN at this point in time.
-		// targetGroups is set to the groups at this point in time.
-	} else {
-		if cred.IsServiceAccount() || cred.IsTemp() {
-			if cred.ParentUser == "" {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
-					errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
+		// In case of LDAP we need to resolve the targetUser to a DN and
+		// query their groups:
+		if globalLDAPConfig.Enabled {
+			opts.claims[ldapUserN] = targetUser // simple username
+			targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
-			if targetUser == "" {
-				targetUser = cred.ParentUser
-			}
-		}
-		// targetGroups not yet set, so set this to cred.Groups
-		if len(targetGroups) == 0 {
-			targetGroups = cred.Groups
+			opts.claims[ldapUser] = targetUser // username DN
 		}
+
+		// NOTE: if not using LDAP, then internal IDP or open ID is
+		// being used - in the former, group info is enforced when
+		// generated credentials are used to make requests, and in the
+		// latter, a group notion is not supported.
 	}
 
 	var sp *iampolicy.Policy
@@ -578,14 +623,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	opts := newServiceAccountOpts{
-		accessKey:     createReq.AccessKey,
-		secretKey:     createReq.SecretKey,
-		sessionPolicy: sp,
-	}
-	if ldapUsername != "" {
-		opts.ldapUsername = ldapUsername
-	}
+	opts.sessionPolicy = sp
 	newCred, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -593,10 +631,40 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	// Notify all other Minio peers to reload user the service account
-	for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	// Call hook for cluster-replication.
+	//
+	// FIXME: This wont work in an OpenID situation as the parent credential
+	// may not be present on peer clusters to provide inherited policies.
+	// Also, we should not be replicating root user's service account - as
+	// they are not authenticated by a common external IDP, so we skip when
+	// opts.ldapUser == "".
+	if _, isLDAPAccount := opts.claims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Create: &madmin.SRSvcAccCreate{
+					Parent:        newCred.ParentUser,
+					AccessKey:     newCred.AccessKey,
+					SecretKey:     newCred.SecretKey,
+					Groups:        newCred.Groups,
+					Claims:        opts.claims,
+					SessionPolicy: createReq.Policy,
+					Status:        auth.AccountOn,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
 
@@ -704,13 +772,39 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 	}
 
 	// Notify all other Minio peers to reload user the service account
-	for _, nerr := range globalNotificationSys.LoadServiceAccount(accessKey) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadServiceAccount(accessKey) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
 		}
 	}
 
+	// Call site replication hook. Only LDAP accounts are supported for
+	// replication operations.
+	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Update: &madmin.SRSvcAccUpdate{
+					AccessKey:     accessKey,
+					SecretKey:     opts.secretKey,
+					Status:        opts.status,
+					SessionPolicy: updateReq.NewPolicy,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
 	writeSuccessNoContent(w)
 }
 
@@ -945,6 +1039,28 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
+	// Call site replication hook. Only LDAP accounts are supported for
+	// replication operations.
+	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, serviceAccount)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Delete: &madmin.SRSvcAccDelete{
+					AccessKey: serviceAccount,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -1006,7 +1122,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	var dataUsageInfo madmin.DataUsageInfo
+	var dataUsageInfo DataUsageInfo
 	var err error
 	if !globalIsGateway {
 		// Load the latest calculated data usage
@@ -1081,8 +1197,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		if rd || wr {
 			// Fetch the data usage of the current bucket
 			var size uint64
+			var objectsCount uint64
+			var objectsHist map[string]uint64
 			if !dataUsageInfo.LastUpdate.IsZero() {
 				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
+				objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
+				objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
 			}
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
@@ -1093,11 +1213,27 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 					logger.LogIf(ctx, err)
 				}
 			}
+
+			lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)
+			quota, _ := globalBucketQuotaSys.Get(bucket.Name)
+			rcfg, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name)
+			tcfg, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
+
 			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
-				Name:        bucket.Name,
-				Created:     bucket.Created,
-				Size:        size,
-				PrefixUsage: prefixUsage,
+				Name:                 bucket.Name,
+				Created:              bucket.Created,
+				Size:                 size,
+				Objects:              objectsCount,
+				ObjectSizesHistogram: objectsHist,
+				PrefixUsage:          prefixUsage,
+				Details: &madmin.BucketDetails{
+					Versioning:          globalBucketVersioningSys.Enabled(bucket.Name),
+					VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name),
+					Replication:         rcfg != nil,
+					Locking:             lcfg.LockEnabled,
+					Quota:               quota,
+					Tagging:             tcfg,
+				},
 				Access: madmin.AccountAccess{
 					Read:  rd,
 					Write: wr,
@@ -1236,6 +1372,16 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 			logger.LogIf(ctx, nerr.Err)
 		}
 	}
+
+	// Call cluster-replication policy creation hook to replicate policy deletion to
+	// other minio clusters.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemPolicy,
+		Name: policyName,
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // AddCannedPolicy - PUT /minio/admin/v3/add-canned-policy?name=<policy_name>
@@ -1264,7 +1410,13 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	iamPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	iamPolicy, err := iampolicy.ParseConfig(bytes.NewReader(iamPolicyBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1282,12 +1434,25 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}
 
 	// Notify all other MinIO peers to reload policy
-	for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
 		}
 	}
+
+	// Call cluster-replication policy creation hook to replicate policy to
+	// other minio clusters.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type:   madmin.SRIAMItemPolicy,
+		Name:   policyName,
+		Policy: iamPolicyBytes,
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // SetPolicyForUserOrGroup - PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
@@ -1324,10 +1489,24 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 	}
 
 	// Notify all other MinIO peers to reload policy
-	for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+	if !globalIAMSys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
 		}
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemPolicyMapping,
+		PolicyMapping: &madmin.SRPolicyMapping{
+			UserOrGroup: entityName,
+			IsGroup:     isGroup,
+			Policy:      policyName,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }

cmd/admin-handlers-users_test.go

@@ -0,0 +1,640 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go"
+	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/auth"
+)
+
+const (
+	testDefaultTimeout = 10 * time.Second
+)
+
+// API suite container for IAM
+type TestSuiteIAM struct {
+	TestSuiteCommon
+
+	endpoint string
+	adm      *madmin.AdminClient
+	client   *minio.Client
+}
+
+func newTestSuiteIAM(c TestSuiteCommon) *TestSuiteIAM {
+	return &TestSuiteIAM{TestSuiteCommon: c}
+}
+
+func (s *TestSuiteIAM) SetUpSuite(c *check) {
+	s.TestSuiteCommon.SetUpSuite(c)
+
+	var err error
+	// strip url scheme from endpoint
+	s.endpoint = strings.TrimPrefix(s.endPoint, "http://")
+	if s.secure {
+		s.endpoint = strings.TrimPrefix(s.endPoint, "https://")
+	}
+
+	s.adm, err = madmin.New(s.endpoint, s.accessKey, s.secretKey, s.secure)
+	if err != nil {
+		c.Fatalf("error creating admin client: %v", err)
+	}
+	// Set transport, so that TLS is handled correctly.
+	s.adm.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	s.client, err = minio.New(s.endpoint, &minio.Options{
+		Creds:     credentials.NewStaticV4(s.accessKey, s.secretKey, ""),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("error creating minio client: %v", err)
+	}
+}
+
+func (s *TestSuiteIAM) getUserClient(c *check, accessKey, secretKey, sessionToken string) *minio.Client {
+	client, err := minio.New(s.endpoint, &minio.Options{
+		Creds:     credentials.NewStaticV4(accessKey, secretKey, sessionToken),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("error creating user minio client: %s", err)
+	}
+	return client
+}
+
+func runAllIAMTests(suite *TestSuiteIAM, c *check) {
+	suite.SetUpSuite(c)
+	suite.TestUserCreate(c)
+	suite.TestPolicyCreate(c)
+	suite.TestGroupAddRemove(c)
+	suite.TestServiceAccountOps(c)
+	suite.TearDownSuite(c)
+}
+
+func TestIAMInternalIDPServerSuite(t *testing.T) {
+	testCases := []*TestSuiteIAM{
+		// Init and run test on FS backend with signature v4.
+		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4}),
+		// Init and run test on FS backend, with tls enabled.
+		newTestSuiteIAM(TestSuiteCommon{serverType: "FS", signer: signerV4, secure: true}),
+		// Init and run test on Erasure backend.
+		newTestSuiteIAM(TestSuiteCommon{serverType: "Erasure", signer: signerV4}),
+		// Init and run test on ErasureSet backend.
+		newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSet", signer: signerV4}),
+	}
+	for i, testCase := range testCases {
+		t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) {
+			runAllIAMTests(testCase, &check{t, testCase.serverType})
+		})
+	}
+}
+
+func (s *TestSuiteIAM) TestUserCreate(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	// 1. Create a user.
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	// 2. Check new user appears in listing
+	usersMap, err := s.adm.ListUsers(ctx)
+	if err != nil {
+		c.Fatalf("error listing: %v", err)
+	}
+	v, ok := usersMap[accessKey]
+	if !ok {
+		c.Fatalf("user not listed: %s", accessKey)
+	}
+	c.Assert(v.Status, madmin.AccountEnabled)
+
+	// 3. Associate policy and check that user can access
+	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	if err != nil {
+		c.Fatalf("unable to set policy: %v", err)
+	}
+	client := s.getUserClient(c, accessKey, secretKey, "")
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("user could not create bucket: %v", err)
+	}
+
+	// 4. Check that user can be disabled and verify it.
+	err = s.adm.SetUserStatus(ctx, accessKey, madmin.AccountDisabled)
+	if err != nil {
+		c.Fatalf("could not set user account to disabled")
+	}
+	usersMap, err = s.adm.ListUsers(ctx)
+	if err != nil {
+		c.Fatalf("error listing: %v", err)
+	}
+	v, ok = usersMap[accessKey]
+	if !ok {
+		c.Fatalf("user was not listed after disabling: %s", accessKey)
+	}
+	c.Assert(v.Status, madmin.AccountDisabled)
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err == nil {
+		c.Fatalf("user account was not disabled!")
+	}
+
+	// 5. Check that user can be deleted and verify it.
+	err = s.adm.RemoveUser(ctx, accessKey)
+	if err != nil {
+		c.Fatalf("user could not be deleted: %v", err)
+	}
+	usersMap, err = s.adm.ListUsers(ctx)
+	if err != nil {
+		c.Fatalf("error listing: %v", err)
+	}
+	_, ok = usersMap[accessKey]
+	if ok {
+		c.Fatalf("user not deleted: %s", accessKey)
+	}
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err == nil {
+		c.Fatalf("user account was not deleted!")
+	}
+}
+
+func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// 1. Create a policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	// 2. Verify that policy json is validated by server
+	invalidPolicyBytes := policyBytes[:len(policyBytes)-1]
+	err = s.adm.AddCannedPolicy(ctx, policy+"invalid", invalidPolicyBytes)
+	if err == nil {
+		c.Fatalf("invalid policy creation success")
+	}
+
+	// 3. Create a user, associate policy and verify access
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+	// 3.1 check that user does not have any access to the bucket
+	uClient := s.getUserClient(c, accessKey, secretKey, "")
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	// 3.2 associate policy to user
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+	// 3.3 check user has access to bucket
+	c.mustListObjects(ctx, uClient, bucket)
+	// 3.4 Check that user cannot exceed their permissions
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil {
+		c.Fatalf("bucket was deleted!")
+	}
+
+	// 4. Verify the policy appears in listing
+	ps, err := s.adm.ListCannedPolicies(ctx)
+	if err != nil {
+		c.Fatalf("policy list err: %v", err)
+	}
+	_, ok := ps[policy]
+	if !ok {
+		c.Fatalf("policy was missing!")
+	}
+
+	// 5. Check that policy can be deleted.
+	err = s.adm.RemoveCannedPolicy(ctx, policy)
+	if err != nil {
+		c.Fatalf("policy delete err: %v", err)
+	}
+}
+
+func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	// 1. Add user to a new group
+	group := "mygroup"
+	err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
+		Group:   group,
+		Members: []string{accessKey},
+	})
+	if err != nil {
+		c.Fatalf("Unable to add user to group: %v", err)
+	}
+
+	// 2. Check that user has no access
+	uClient := s.getUserClient(c, accessKey, secretKey, "")
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	// 3. Associate policy to group and check user got access.
+	err = s.adm.SetPolicy(ctx, policy, group, true)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+	// 3.1 check user has access to bucket
+	c.mustListObjects(ctx, uClient, bucket)
+	// 3.2 Check that user cannot exceed their permissions
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil {
+		c.Fatalf("bucket was deleted!")
+	}
+
+	// 4. List groups and members and verify
+	groups, err := s.adm.ListGroups(ctx)
+	if err != nil {
+		c.Fatalf("group list err: %v", err)
+	}
+	if !set.CreateStringSet(groups...).Contains(group) {
+		c.Fatalf("created group not present!")
+	}
+	groupInfo, err := s.adm.GetGroupDescription(ctx, group)
+	if err != nil {
+		c.Fatalf("group desc err: %v", err)
+	}
+	c.Assert(groupInfo.Name, group)
+	c.Assert(set.CreateStringSet(groupInfo.Members...), set.CreateStringSet(accessKey))
+	c.Assert(groupInfo.Policy, policy)
+	c.Assert(groupInfo.Status, string(madmin.GroupEnabled))
+
+	// 5. Disable/enable the group and verify that user access is revoked/restored.
+	err = s.adm.SetGroupStatus(ctx, group, madmin.GroupDisabled)
+	if err != nil {
+		c.Fatalf("group set status err: %v", err)
+	}
+	groupInfo, err = s.adm.GetGroupDescription(ctx, group)
+	if err != nil {
+		c.Fatalf("group desc err: %v", err)
+	}
+	c.Assert(groupInfo.Status, string(madmin.GroupDisabled))
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	err = s.adm.SetGroupStatus(ctx, group, madmin.GroupEnabled)
+	if err != nil {
+		c.Fatalf("group set status err: %v", err)
+	}
+	groupInfo, err = s.adm.GetGroupDescription(ctx, group)
+	if err != nil {
+		c.Fatalf("group desc err: %v", err)
+	}
+	c.Assert(groupInfo.Status, string(madmin.GroupEnabled))
+	c.mustListObjects(ctx, uClient, bucket)
+
+	// 6. Verify that group cannot be deleted with users.
+	err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
+		Group:    group,
+		IsRemove: true,
+	})
+	if err == nil {
+		c.Fatalf("group was removed!")
+	}
+	groupInfo, err = s.adm.GetGroupDescription(ctx, group)
+	if err != nil {
+		c.Fatalf("group desc err: %v", err)
+	}
+	c.Assert(groupInfo.Name, group)
+
+	// 7. Remove user from group and verify access is revoked.
+	err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
+		Group:    group,
+		Members:  []string{accessKey},
+		IsRemove: true,
+	})
+	if err != nil {
+		c.Fatalf("group update err: %v", err)
+	}
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	// 7.1 verify group still exists
+	groupInfo, err = s.adm.GetGroupDescription(ctx, group)
+	if err != nil {
+		c.Fatalf("group desc err: %v", err)
+	}
+	c.Assert(groupInfo.Name, group)
+	c.Assert(len(groupInfo.Members), 0)
+
+	// 8. Delete group and verify
+	err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
+		Group:    group,
+		IsRemove: true,
+	})
+	if err != nil {
+		c.Fatalf("group update err: %v", err)
+	}
+	groups, err = s.adm.ListGroups(ctx)
+	if err != nil {
+		c.Fatalf("group list err: %v", err)
+	}
+	if set.CreateStringSet(groups...).Contains(group) {
+		c.Fatalf("created group still present!")
+	}
+	groupInfo, err = s.adm.GetGroupDescription(ctx, group)
+	if err == nil {
+		c.Fatalf("group appears to exist")
+	}
+}
+
+func (s *TestSuiteIAM) TestServiceAccountOps(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Create policy, user and associate policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	// 1. Create a service account for the user
+	svcAK, svcSK := mustGenerateCredentials(c)
+	cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: accessKey,
+		AccessKey:  svcAK,
+		SecretKey:  svcSK,
+	})
+	if err != nil {
+		c.Fatalf("Unable to create svc acc: %v", err)
+	}
+	// 1.2 Check that svc account appears in listing
+	listResp, err := s.adm.ListServiceAccounts(ctx, accessKey)
+	if err != nil {
+		c.Fatalf("unable to list svc accounts: %v", err)
+	}
+	if !set.CreateStringSet(listResp.Accounts...).Contains(svcAK) {
+		c.Fatalf("created service account did not appear in listing!")
+	}
+	// 1.3 Check that svc account info can be queried
+	infoResp, err := s.adm.InfoServiceAccount(ctx, svcAK)
+	if err != nil {
+		c.Fatalf("unable to get svc acc info: %v", err)
+	}
+	c.Assert(infoResp.ParentUser, accessKey)
+	c.Assert(infoResp.AccountStatus, "on")
+	c.Assert(infoResp.ImpliedPolicy, true)
+
+	// 2. Check that svc account can access the bucket
+	{
+		svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+		c.mustListObjects(ctx, svcClient, bucket)
+	}
+
+	// 3. Check that svc account can restrict the policy, and that the
+	// session policy can be updated.
+	{
+		svcAK, svcSK := mustGenerateCredentials(c)
+		policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+		cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+			Policy:     policyBytes,
+			TargetUser: accessKey,
+			AccessKey:  svcAK,
+			SecretKey:  svcSK,
+		})
+		if err != nil {
+			c.Fatalf("Unable to create svc acc: %v", err)
+		}
+		svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+		c.mustNotListObjects(ctx, svcClient, bucket)
+
+		newPolicyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+		err = s.adm.UpdateServiceAccount(ctx, svcAK, madmin.UpdateServiceAccountReq{
+			NewPolicy: newPolicyBytes,
+		})
+		if err != nil {
+			c.Fatalf("unable to update session policy for svc acc: %v", err)
+		}
+		c.mustListObjects(ctx, svcClient, bucket)
+	}
+
+	// 4. Check that service account's secret key and account status can be
+	// updated.
+	{
+		svcAK, svcSK := mustGenerateCredentials(c)
+		cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+			TargetUser: accessKey,
+			AccessKey:  svcAK,
+			SecretKey:  svcSK,
+		})
+		if err != nil {
+			c.Fatalf("Unable to create svc acc: %v", err)
+		}
+		svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+		c.mustListObjects(ctx, svcClient, bucket)
+
+		_, svcSK2 := mustGenerateCredentials(c)
+		err = s.adm.UpdateServiceAccount(ctx, svcAK, madmin.UpdateServiceAccountReq{
+			NewSecretKey: svcSK2,
+		})
+		if err != nil {
+			c.Fatalf("unable to update secret key for svc acc: %v", err)
+		}
+		// old creds should not work:
+		c.mustNotListObjects(ctx, svcClient, bucket)
+		// new creds work:
+		svcClient2 := s.getUserClient(c, cr.AccessKey, svcSK2, "")
+		c.mustListObjects(ctx, svcClient2, bucket)
+
+		// update status to disabled
+		err = s.adm.UpdateServiceAccount(ctx, svcAK, madmin.UpdateServiceAccountReq{
+			NewStatus: "off",
+		})
+		if err != nil {
+			c.Fatalf("unable to update secret key for svc acc: %v", err)
+		}
+		c.mustNotListObjects(ctx, svcClient2, bucket)
+	}
+
+	// 5. Check that service account can be deleted.
+	{
+		svcAK, svcSK := mustGenerateCredentials(c)
+		cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+			TargetUser: accessKey,
+			AccessKey:  svcAK,
+			SecretKey:  svcSK,
+		})
+		if err != nil {
+			c.Fatalf("Unable to create svc acc: %v", err)
+		}
+		svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+		c.mustListObjects(ctx, svcClient, bucket)
+
+		err = s.adm.DeleteServiceAccount(ctx, svcAK)
+		if err != nil {
+			c.Fatalf("unable to delete svc acc: %v", err)
+		}
+		c.mustNotListObjects(ctx, svcClient, bucket)
+	}
+}
+
+func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bucket string) {
+	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
+	v, ok := <-res
+	if !ok || v.Err == nil {
+		c.Fatalf("user was able to list unexpectedly!")
+	}
+}
+
+func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) {
+	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
+	v, ok := <-res
+	if ok && v.Err != nil {
+		c.Fatalf("user was unable to list unexpectedly!")
+	}
+}
+
+func mustGenerateCredentials(c *check) (string, string) {
+	ak, sk, err := auth.GenerateCredentials()
+	if err != nil {
+		c.Fatalf("unable to generate credentials: %v", err)
+	}
+	return ak, sk
+}

cmd/admin-handlers.go

@@ -38,7 +38,7 @@ import (
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go"
@@ -149,9 +149,13 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
 		if nerr.Err != nil {
+			err := AdminError{
+				Code:       AdminUpdateApplyFailure,
+				Message:    nerr.Err.Error(),
+				StatusCode: http.StatusInternalServerError,
+			}
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-			err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", nerr.Err)
+			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -159,7 +163,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
 	if err != nil {
-		err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", err)
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -369,10 +373,10 @@ func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 		}
 		for k, v := range peerLock.Locks {
 			for _, lockReqInfo := range v {
-				if val, ok := entryMap[lockReqInfo.UID]; ok {
+				if val, ok := entryMap[lockReqInfo.Name]; ok {
 					val.ServerList = append(val.ServerList, peerLock.Addr)
 				} else {
-					entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
+					entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
 				}
 			}
 		}
@@ -925,6 +929,12 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
+	autotuneStr := r.Form.Get("autotune")
+
+	var autotune bool
+	if autotuneStr != "" {
+		autotune = true
+	}
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -941,14 +951,52 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		duration = time.Second * 10
 	}
 
-	results := globalNotificationSys.Speedtest(ctx, size, concurrent, duration)
+	throughputSize := size
+	iopsSize := size
 
-	if err := json.NewEncoder(w).Encode(results); err != nil {
+	if autotune {
+		iopsSize = 4 * humanize.KiByte
+	}
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	endBlankRepliesCh := make(chan error)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				endBlankRepliesCh <- nil
+				return
+			case <-keepAliveTicker.C:
+				// Write a blank entry to prevent client from disconnecting
+				if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil {
+					endBlankRepliesCh <- err
+					return
+				}
+				w.(http.Flusher).Flush()
+			case endBlankRepliesCh <- nil:
+				return
+			}
+		}
+	}()
+
+	result, err := speedTest(ctx, throughputSize, iopsSize, concurrent, duration, autotune)
+	if <-endBlankRepliesCh != nil {
+		return
+	}
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), true)
+	if err := json.NewEncoder(w).Encode(result); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true})
 
 	w.(http.Flusher).Flush()
 }
@@ -1567,6 +1615,22 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
+	getAndWriteSysConfig := func() {
+		if query.Get(string(madmin.HealthDataTypeSysConfig)) == "true" {
+			localSysConfig := madmin.GetSysConfig(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localSysConfig)
+			healthInfo.Sys.SysConfig = append(healthInfo.Sys.SysConfig, localSysConfig)
+			partialWrite(healthInfo)
+
+			peerSysConfig := globalNotificationSys.GetSysConfig(deadlinedCtx)
+			for _, sc := range peerSysConfig {
+				anonymizeAddr(&sc)
+				healthInfo.Sys.SysConfig = append(healthInfo.Sys.SysConfig, sc)
+			}
+			partialWrite(healthInfo)
+		}
+	}
+
 	getAndWriteSysServices := func() {
 		if query.Get(string(madmin.HealthDataTypeSysServices)) == "true" {
 			localSysServices := madmin.GetSysServices(deadlinedCtx, globalLocalNodeName)
@@ -1575,9 +1639,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			partialWrite(healthInfo)
 
 			peerSysServices := globalNotificationSys.GetSysServices(deadlinedCtx)
-			for _, sli := range peerSysServices {
-				anonymizeAddr(&sli)
-				healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, sli)
+			for _, ss := range peerSysServices {
+				anonymizeAddr(&ss)
+				healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, ss)
 			}
 			partialWrite(healthInfo)
 		}
@@ -1758,6 +1822,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		getAndWriteNetPerfInfo()
 		getAndWriteSysErrors()
 		getAndWriteSysServices()
+		getAndWriteSysConfig()
 
 		if query.Get("minioinfo") == "true" {
 			infoMessage := getServerInfo(ctx, r)
@@ -1782,6 +1847,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 					},
 				})
 			}
+
 			healthInfo.Minio.Info = madmin.MinioInfo{
 				Mode:         infoMessage.Mode,
 				Domain:       infoMessage.Domain,
@@ -1794,6 +1860,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				Services:     infoMessage.Services,
 				Backend:      infoMessage.Backend,
 				Servers:      servers,
+				TLS:          getTLSInfo(),
 			}
 			partialWrite(healthInfo)
 		}
@@ -1823,6 +1890,25 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 
 }
 
+func getTLSInfo() madmin.TLSInfo {
+	tlsInfo := madmin.TLSInfo{
+		TLSEnabled: globalIsTLS,
+		Certs:      []madmin.TLSCert{},
+	}
+
+	if globalIsTLS {
+		for _, c := range globalPublicCerts {
+			tlsInfo.Certs = append(tlsInfo.Certs, madmin.TLSCert{
+				PubKeyAlgo:    c.PublicKeyAlgorithm.String(),
+				SignatureAlgo: c.SignatureAlgorithm.String(),
+				NotBefore:     c.NotBefore,
+				NotAfter:      c.NotAfter,
+			})
+		}
+	}
+	return tlsInfo
+}
+
 // BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
 // ----------
 // Get bandwidth consumption information
@@ -2079,7 +2165,7 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 
 // getRawDataer provides an interface for getting raw FS files.
 type getRawDataer interface {
-	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
+	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
 }
 
 // InspectDataHandler - GET /minio/admin/v3/inspect-data
@@ -2112,6 +2198,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
+	file = strings.ReplaceAll(file, string(os.PathSeparator), "/")
+
+	// Reject attempts to traverse parent or absolute paths.
+	if strings.Contains(file, "..") || strings.Contains(volume, "..") {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
 
 	var key [32]byte
 	// MUST use crypto/rand
@@ -2149,15 +2242,23 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	zipWriter := zip.NewWriter(encw)
 	defer zipWriter.Close()
 
-	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
+	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, si StatInfo) error {
 		// Prefix host+disk
 		filename = path.Join(host, disk, filename)
+		if si.Dir {
+			filename += "/"
+			si.Size = 0
+		}
+		if si.Mode == 0 {
+			// Not, set it to default.
+			si.Mode = 0600
+		}
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
-			size:    size,
-			mode:    0600,
-			modTime: modtime,
-			isDir:   false,
+			size:    si.Size,
+			mode:    os.FileMode(si.Mode),
+			modTime: si.ModTime,
+			isDir:   si.Dir,
 			sys:     nil,
 		})
 		if zerr != nil {

cmd/admin-handlers_test.go

@@ -73,7 +73,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 
 	initAllSubsystems(ctx, objLayer)
 
-	globalIAMSys.InitStore(objLayer)
+	globalIAMSys.InitStore(objLayer, globalEtcdClient)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()

cmd/admin-heal-ops.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"sort"
@@ -396,9 +397,6 @@ type healSequence struct {
 	// bucket, and object on which heal seq. was initiated
 	bucket, object string
 
-	// A channel of entities (format, buckets, objects) to heal
-	sourceCh chan healSource
-
 	// A channel of entities with heal result
 	respCh chan healResult
 
@@ -648,11 +646,7 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	h.currentStatus.StartTime = UTCNow()
 	h.mutex.Unlock()
 
-	if h.sourceCh == nil {
-		go h.traverseAndHeal(objAPI)
-	} else {
-		go h.healFromSourceCh()
-	}
+	go h.traverseAndHeal(objAPI)
 
 	select {
 	case err, ok := <-h.traverseAndHealDoneCh:
@@ -696,45 +690,35 @@ func (h *healSequence) logHeal(healType madmin.HealItemType) {
 }
 
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
-	globalHealConfigMu.Lock()
-	opts := globalHealConfig
-	globalHealConfigMu.Unlock()
-
 	// Send heal request
 	task := healTask{
-		bucket:     source.bucket,
-		object:     source.object,
-		versionID:  source.versionID,
-		opts:       h.settings,
-		responseCh: h.respCh,
+		bucket:    source.bucket,
+		object:    source.object,
+		versionID: source.versionID,
+		opts:      h.settings,
+		respCh:    h.respCh,
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
 	}
-	if opts.Bitrot {
-		task.opts.ScanMode = madmin.HealDeepScan
-	}
-
-	// Wait and proceed if there are active requests
-	waitForLowHTTPReq(opts.IOCount, opts.Sleep)
+	task.opts.ScanMode = globalHealConfig.ScanMode()
 
 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++
 	h.lastHealActivity = UTCNow()
 	h.mutex.Unlock()
 
-	globalBackgroundHealRoutine.queueHealTask(task)
+	select {
+	case globalBackgroundHealRoutine.tasks <- task:
+	case <-h.ctx.Done():
+		return nil
+	}
 
 	select {
 	case res := <-h.respCh:
 		if !h.reportProgress {
-			// Object might have been deleted, by the time heal
-			// was attempted, we should ignore this object and
-			// return the error and not calculate this object
-			// as part of the metrics.
-			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
-				// Return the error so that caller can handle it.
-				return res.err
+			if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
+				return nil
 			}
 
 			h.mutex.Lock()
@@ -759,11 +743,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		}
 		res.result.Type = healType
 		if res.err != nil {
-			// Object might have been deleted, by the time heal
-			// was attempted, we should ignore this object and return success.
-			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
-				return nil
-			}
 			// Only report object error
 			if healType != madmin.HealItemObject {
 				return res.err
@@ -776,48 +755,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	}
 }
 
-func (h *healSequence) healItemsFromSourceCh() error {
-	for {
-		select {
-		case source, ok := <-h.sourceCh:
-			if !ok {
-				return nil
-			}
-
-			var itemType madmin.HealItemType
-			switch source.bucket {
-			case nopHeal:
-				continue
-			case SlashSeparator:
-				itemType = madmin.HealItemMetadata
-			default:
-				if source.object == "" {
-					itemType = madmin.HealItemBucket
-				} else {
-					itemType = madmin.HealItemObject
-				}
-			}
-
-			if err := h.queueHealTask(source, itemType); err != nil {
-				switch err.(type) {
-				case ObjectExistsAsDirectory:
-				case ObjectNotFound:
-				case VersionNotFound:
-				default:
-					logger.LogIf(h.ctx, fmt.Errorf("Heal attempt failed for %s: %w",
-						pathJoin(source.bucket, source.object), err))
-				}
-			}
-		case <-h.ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (h *healSequence) healFromSourceCh() {
-	h.healItemsFromSourceCh()
-}
-
 func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 	// Start healing the config prefix.
 	return h.healMinioSysMeta(objAPI, minioConfigPrefix)()
@@ -862,11 +799,6 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 				object:    object,
 				versionID: versionID,
 			}, madmin.HealItemBucketMetadata)
-			// Object might have been deleted, by the time heal
-			// was attempted we ignore this object an move on.
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				return nil
-			}
 			return err
 		})
 	}
@@ -915,9 +847,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly bool) error {
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
-		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-			return err
-		}
+		return err
 	}
 
 	if bucketsOnly {
@@ -931,9 +861,6 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 			oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
-					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-						return nil
-					}
 					return err
 				}
 			}
@@ -943,11 +870,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 	}
 
 	if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
-		// Object might have been deleted, by the time heal
-		// was attempted we ignore this object an move on.
-		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-			return errFnHealFromAPIErr(h.ctx, err)
-		}
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil
 }
@@ -963,5 +886,9 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		object:    object,
 		versionID: versionID,
 	}, madmin.HealItemObject)
+
+	// Wait and proceed if there are active requests
+	waitForLowHTTPReq()
+
 	return err
 }

cmd/admin-router.go

@@ -191,6 +191,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
+
+			// Tier stats
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
+
+			// Cluster Replication APIs
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalJoin)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateIAMItem)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateBucketItem)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalGetIDPSettings)))
 		}
 
 		if globalIsDistErasure {

cmd/api-datatypes.go

@@ -28,14 +28,10 @@ type DeletedObject struct {
 	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"`
 	ObjectName            string `xml:"Key,omitempty"`
 	VersionID             string `xml:"VersionId,omitempty"`
-
-	// MinIO extensions to support delete marker replication
-	// Replication status of DeleteMarker
-	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus,omitempty"`
 	// MTime of DeleteMarker on source that needs to be propagated to replica
-	DeleteMarkerMTime DeleteMarkerMTime `xml:"DeleteMarkerMTime,omitempty"`
-	// Status of versioned delete (of object or DeleteMarker)
-	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus,omitempty"`
+	DeleteMarkerMTime DeleteMarkerMTime `xml:"-"`
+	// MinIO extensions to support delete marker replication
+	ReplicationState ReplicationState `xml:"-"`
 }
 
 // DeleteMarkerMTime is an embedded type containing time.Time for XML marshal
@@ -60,8 +56,10 @@ type ObjectToDelete struct {
 	DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"`
 	// Status of versioned delete (of object or DeleteMarker)
 	VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus"`
-	// Version ID of delete marker
-	DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId"`
+	// VersionPurgeStatuses holds the internal
+	VersionPurgeStatuses string `xml:"VersionPurgeStatuses"`
+	// ReplicateDecisionStr stringified representation of replication decision
+	ReplicateDecisionStr string `xml:"-"`
 }
 
 // createBucketConfiguration container for bucket configuration request from client.

cmd/api-errors.go

@@ -20,9 +20,11 @@ package cmd
 import (
 	"context"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
@@ -265,6 +267,16 @@ const (
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
+
+	// Site-Replication errors
+	ErrSiteReplicationInvalidRequest
+	ErrSiteReplicationPeerResp
+	ErrSiteReplicationBackendIssue
+	ErrSiteReplicationServiceAccountError
+	ErrSiteReplicationBucketConfigError
+	ErrSiteReplicationBucketMetaError
+	ErrSiteReplicationIAMError
+
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
 	ErrAdminNoSuchQuotaConfiguration
@@ -442,7 +454,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		Description:    "Part number must be an integer between 1 and 10000, inclusive",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartNumberMarker: {
@@ -1267,6 +1279,43 @@ var errorCodes = errorCodeMap{
 		Description:    errObjectTampered.Error(),
 		HTTPStatusCode: http.StatusPartialContent,
 	},
+
+	ErrSiteReplicationInvalidRequest: {
+		Code:           "XMinioSiteReplicationInvalidRequest",
+		Description:    "Invalid site-replication request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSiteReplicationPeerResp: {
+		Code:           "XMinioSiteReplicationPeerResp",
+		Description:    "Error received when contacting a peer site",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBackendIssue: {
+		Code:           "XMinioSiteReplicationBackendIssue",
+		Description:    "Error when requesting object layer backend",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationServiceAccountError: {
+		Code:           "XMinioSiteReplicationServiceAccountError",
+		Description:    "Site replication related service account error",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBucketConfigError: {
+		Code:           "XMinioSiteReplicationBucketConfigError",
+		Description:    "Error while configuring replication on a bucket",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBucketMetaError: {
+		Code:           "XMinioSiteReplicationBucketMetaError",
+		Description:    "Error while replicating bucket metadata",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationIAMError: {
+		Code:           "XMinioSiteReplicationIAMError",
+		Description:    "Error while replicating an IAM item",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
@@ -1785,12 +1834,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
 	// This way downstream context.Canceled will still report ErrOperationTimedOut
-	select {
-	case <-ctx.Done():
+	if contextCanceled(ctx) {
 		if ctx.Err() == context.Canceled {
 			return ErrClientDisconnected
 		}
-	default:
 	}
 
 	switch err {
@@ -1965,8 +2012,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrReplicationConfigurationNotFoundError
 	case BucketRemoteDestinationNotFound:
 		apiErr = ErrRemoteDestinationNotFoundError
-	case BucketReplicationDestinationMissingLock:
-		apiErr = ErrReplicationDestinationMissingLock
 	case BucketRemoteTargetNotFound:
 		apiErr = ErrRemoteTargetNotFoundError
 	case BucketRemoteConnectionErr:
@@ -2177,10 +2222,30 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 			// Add more Gateway SDKs here if any in future.
 		default:
-			apiErr = APIError{
-				Code:           apiErr.Code,
-				Description:    fmt.Sprintf("%s: cause(%v)", apiErr.Description, err),
-				HTTPStatusCode: apiErr.HTTPStatusCode,
+			if errors.Is(err, errMalformedEncoding) {
+				apiErr = APIError{
+					Code:           "BadRequest",
+					Description:    err.Error(),
+					HTTPStatusCode: http.StatusBadRequest,
+				}
+			} else if errors.Is(err, errChunkTooBig) {
+				apiErr = APIError{
+					Code:           "BadRequest",
+					Description:    err.Error(),
+					HTTPStatusCode: http.StatusBadRequest,
+				}
+			} else if errors.Is(err, strconv.ErrRange) {
+				apiErr = APIError{
+					Code:           "BadRequest",
+					Description:    err.Error(),
+					HTTPStatusCode: http.StatusBadRequest,
+				}
+			} else {
+				apiErr = APIError{
+					Code:           apiErr.Code,
+					Description:    fmt.Sprintf("%s: cause(%v)", apiErr.Description, err),
+					HTTPStatusCode: apiErr.HTTPStatusCode,
+				}
 			}
 		}
 	}

cmd/api-response.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -38,7 +39,7 @@ const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
 	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
 	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
+	maxDeleteList     = 1000                       // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
 )
@@ -262,7 +263,7 @@ func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error
 	return e.EncodeElement(objectVersionWrapper(o), start)
 }
 
-// StringMap is a map[string]string.
+// StringMap is a map[string]string
 type StringMap map[string]string
 
 // MarshalXML - StringMap marshals into XML.
@@ -423,10 +424,10 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	}
 
 	for _, bucket := range buckets {
-		var listbucket = Bucket{}
-		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.UTC().Format(iso8601TimeFormat)
-		listbuckets = append(listbuckets, listbucket)
+		listbuckets = append(listbuckets, Bucket{
+			Name:         bucket.Name,
+			CreationDate: bucket.Created.UTC().Format(iso8601TimeFormat),
+		})
 	}
 
 	data.Owner = owner
@@ -570,6 +571,14 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		content.Owner = owner
 		if metadata {
 			content.UserMetadata = make(StringMap)
+			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+			case crypto.S3:
+				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+			case crypto.S3KMS:
+				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
+			case crypto.SSEC:
+				content.UserMetadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
+			}
 			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
 				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 					// Do not need to send any internal metadata

cmd/api-router.go

@@ -241,7 +241,7 @@ func registerAPIRouter(router *mux.Router) {
 			collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
 		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("completemutipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("completemultipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
 		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("newmultipartupload", maxClients(gz(httpTraceAll(api.NewMultipartUploadHandler))))).Queries("uploads", "")

cmd/auth-handler.go

@@ -138,7 +138,7 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	var owner bool
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
-		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		getRequestAuthType(r) == authTypeSigned {
 		// We only support admin credentials to access admin APIs.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {

cmd/auth-handler_test.go

@@ -366,7 +366,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	initAllSubsystems(context.Background(), objLayer)
 
-	globalIAMSys.InitStore(objLayer)
+	globalIAMSys.InitStore(objLayer, globalEtcdClient)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -457,7 +457,7 @@ func TestValidateAdminSignature(t *testing.T) {
 
 	initAllSubsystems(context.Background(), objLayer)
 
-	globalIAMSys.InitStore(objLayer)
+	globalIAMSys.InitStore(objLayer, globalEtcdClient)
 
 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {

cmd/background-heal-ops.go

@@ -19,10 +19,9 @@ package cmd
 
 import (
 	"context"
-	"time"
+	"runtime"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/logger"
 )
 
 // healTask represents what to heal along with options
@@ -35,7 +34,7 @@ type healTask struct {
 	versionID string
 	opts      madmin.HealOpts
 	// Healing response will be sent here
-	responseCh chan healResult
+	respCh chan healResult
 }
 
 // healResult represents a healing result with a possible error
@@ -46,54 +45,37 @@ type healResult struct {
 
 // healRoutine receives heal tasks, to heal buckets, objects and format.json
 type healRoutine struct {
-	tasks  chan healTask
-	doneCh chan struct{}
+	tasks   chan healTask
+	workers int
 }
 
-// Add a new task in the tasks queue
-func (h *healRoutine) queueHealTask(task healTask) {
-	h.tasks <- task
-}
-
-func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
-	// No need to wait run at full speed.
-	if maxIO <= 0 {
-		return
-	}
-
-	// At max 10 attempts to wait with 100 millisecond interval before proceeding
-	waitTick := 100 * time.Millisecond
-
+func systemIO() int {
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
-	maxIOFn := func() int {
-		return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
+	return int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
+}
+
+func waitForLowHTTPReq() {
+	var currentIO func() int
+	if httpServer := newHTTPServerFn(); httpServer != nil {
+		currentIO = httpServer.GetRequestCount
 	}
 
-	tmpMaxWait := maxWait
-	if httpServer := newHTTPServerFn(); httpServer != nil {
-		// Any requests in progress, delay the heal.
-		for httpServer.GetRequestCount() >= maxIOFn() {
-			if tmpMaxWait > 0 {
-				if tmpMaxWait < waitTick {
-					time.Sleep(tmpMaxWait)
-				} else {
-					time.Sleep(waitTick)
-				}
-				tmpMaxWait = tmpMaxWait - waitTick
-			}
-			if tmpMaxWait <= 0 {
-				if intDataUpdateTracker.debug {
-					logger.Info("waitForLowHTTPReq: waited max %s, resuming", maxWait)
-				}
-				break
-			}
-		}
+	globalHealConfig.Wait(currentIO, systemIO)
+}
+
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	// Run the background healer
+	globalBackgroundHealRoutine = newHealRoutine()
+	for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
+		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI)
 	}
+
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
 }
 
 // Wait for heal requests and process them
-func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
+func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 	for {
 		select {
 		case task, ok := <-h.tasks:
@@ -105,6 +87,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			var err error
 			switch task.bucket {
 			case nopHeal:
+				task.respCh <- healResult{err: errSkipFile}
 				continue
 			case SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
@@ -115,10 +98,8 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 					res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
 				}
 			}
-			task.responseCh <- healResult{result: res, err: err}
 
-		case <-h.doneCh:
-			return
+			task.respCh <- healResult{result: res, err: err}
 		case <-ctx.Done():
 			return
 		}
@@ -126,9 +107,13 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func newHealRoutine() *healRoutine {
+	workers := runtime.GOMAXPROCS(0) / 2
+	if workers == 0 {
+		workers = 4
+	}
 	return &healRoutine{
-		tasks:  make(chan healTask),
-		doneCh: make(chan struct{}),
+		tasks:   make(chan healTask),
+		workers: workers,
 	}
 
 }

cmd/background-newdisks-heal-ops.go

@@ -308,14 +308,6 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 
 }
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
-	go globalBackgroundHealRoutine.run(ctx, objAPI)
-
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
-}
-
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
@@ -337,10 +329,10 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 			healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints()
 			if len(healDisks) > 0 {
 				// Reformat disks
-				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
+				bgSeq.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
 
 				// Ensure that reformatting disks is finished
-				bgSeq.sourceCh <- healSource{bucket: nopHeal}
+				bgSeq.queueHealTask(healSource{bucket: nopHeal}, madmin.HealItemMetadata)
 
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))

cmd/bitrot-streaming.go

@@ -57,6 +57,10 @@ func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
 		b.closeWithErr(err)
 		return n, err
 	}
+	if n != len(p) {
+		err = io.ErrShortWrite
+		b.closeWithErr(err)
+	}
 	return n, err
 }
 

cmd/bitrot.go

@@ -27,8 +27,10 @@ import (
 	"io"
 
 	"github.com/minio/highwayhash"
-	"github.com/minio/minio/internal/logger"
 	"golang.org/x/crypto/blake2b"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/minio/minio/internal/logger"
 )
 
 // magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
@@ -172,8 +174,8 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
 		return errFileCorrupt
 	}
 
-	bufp := xlPoolSmall.Get().(*[]byte)
-	defer xlPoolSmall.Put(bufp)
+	bufp := xioutil.ODirectPoolSmall.Get().(*[]byte)
+	defer xioutil.ODirectPoolSmall.Put(bufp)
 
 	for left > 0 {
 		// Read expected hash...

cmd/bootstrap-peer-server.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"reflect"
 	"runtime"
 	"time"
 
@@ -32,6 +33,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
+	"github.com/minio/pkg/env"
 )
 
 const (
@@ -52,8 +54,8 @@ type bootstrapRESTServer struct{}
 // ServerSystemConfig - captures information about server configuration.
 type ServerSystemConfig struct {
 	MinioPlatform  string
-	MinioRuntime   string
 	MinioEndpoints EndpointServerPools
+	MinioEnv       map[string]string
 }
 
 // Diff - returns error on first difference found in two configs.
@@ -82,15 +84,35 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 					s2.MinioEndpoints[i].Endpoints[j])
 			}
 		}
-
+	}
+	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
+		return fmt.Errorf("Expected same MINIO_ environment variables and values")
 	}
 	return nil
 }
 
+var skipEnvs = map[string]struct{}{
+	"MINIO_OPTS":        {},
+	"MINIO_CERT_PASSWD": {},
+}
+
 func getServerSystemCfg() ServerSystemConfig {
+	envs := env.List("MINIO_")
+	envValues := make(map[string]string, len(envs))
+	for _, envK := range envs {
+		// skip certain environment variables as part
+		// of the whitelist and could be configured
+		// differently on each nodes, update skipEnvs()
+		// map if there are such environment values
+		if _, ok := skipEnvs[envK]; ok {
+			continue
+		}
+		envValues[envK] = env.Get(envK, "")
+	}
 	return ServerSystemConfig{
 		MinioPlatform:  fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
 		MinioEndpoints: globalEndpoints,
+		MinioEnv:       envValues,
 	}
 }
 

cmd/bucket-encryption-handlers.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -95,6 +97,20 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:      madmin.SRBucketMetaTypeSSEConfig,
+		Bucket:    bucket,
+		SSEConfig: &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	writeSuccessResponseHeadersOnly(w)
 }
 

cmd/bucket-encryption.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"io"
 
-	bucketsse "github.com/minio/minio/internal/bucket/encryption"
+	sse "github.com/minio/minio/internal/bucket/encryption"
 )
 
 // BucketSSEConfigSys - in-memory cache of bucket encryption config
@@ -33,7 +33,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 }
 
 // Get - gets bucket encryption config for the given bucket.
-func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
+func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerFn()
 		if objAPI == nil {
@@ -47,8 +47,8 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, e
 }
 
 // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.
-func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
-	encConfig, err := bucketsse.ParseBucketSSEConfig(r)
+func validateBucketSSEConfig(r io.Reader) (*sse.BucketSSEConfig, error) {
+	encConfig, err := sse.ParseBucketSSEConfig(r)
 	if err != nil {
 		return nil, err
 	}

cmd/bucket-handlers.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
@@ -37,8 +38,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/config/dns"
@@ -439,8 +442,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		deleteObjectsFn = api.CacheAPI().DeleteObjects
 	}
 
-	// Return Malformed XML as S3 spec if the list of objects is empty
-	if len(deleteObjects.Objects) == 0 {
+	// Return Malformed XML as S3 spec if the number of objects is empty
+	if len(deleteObjects.Objects) == 0 || len(deleteObjects.Objects) > maxDeleteList {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
 		return
 	}
@@ -452,9 +455,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	var (
-		hasLockEnabled, replicateSync bool
-		goi                           ObjectInfo
-		gerr                          error
+		hasLockEnabled bool
+		dsc            ReplicateDecision
+		goi            ObjectInfo
+		gerr           error
 	)
 	replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjects.Objects)
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
@@ -514,24 +518,18 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if replicateDeletes {
-			replicate, repsync := checkReplicateDelete(ctx, bucket, ObjectToDelete{
+			dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectName: object.ObjectName,
 				VersionID:  object.VersionID,
-			}, goi, gerr)
-			replicateSync = repsync
-			if replicate {
-				if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object.ObjectName); apiErrCode != ErrNone {
-					if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
-						writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
-						return
-					}
-					continue
-				}
+			}, goi, opts, gerr)
+			if dsc.ReplicateAny() {
 				if object.VersionID != "" {
 					object.VersionPurgeStatus = Pending
+					object.VersionPurgeStatuses = dsc.PendingStatus()
 				} else {
-					object.DeleteMarkerReplicationStatus = string(replication.Pending)
+					object.DeleteMarkerReplicationStatus = dsc.PendingStatus()
 				}
+				object.ReplicateDecisionStr = dsc.String()
 			}
 		}
 		if object.VersionID != "" && hasLockEnabled {
@@ -577,14 +575,15 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		objToDel := ObjectToDelete{
 			ObjectName:                    dObjects[i].ObjectName,
 			VersionID:                     dObjects[i].VersionID,
-			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus,
-			DeleteMarkerReplicationStatus: dObjects[i].DeleteMarkerReplicationStatus,
+			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus(),
+			VersionPurgeStatuses:          dObjects[i].ReplicationState.VersionPurgeStatusInternal,
+			DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal,
+			ReplicateDecisionStr:          dObjects[i].ReplicationState.ReplicateDecisionStr,
 		}
 		dindex := objectsToDelete[objToDel]
 		if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
 			if replicateDeletes {
-				dObjects[i].DeleteMarkerReplicationStatus = deleteList[i].DeleteMarkerReplicationStatus
-				dObjects[i].VersionPurgeStatus = deleteList[i].VersionPurgeStatus
+				dObjects[i].ReplicationState = deleteList[i].ReplicationState()
 			}
 			deletedObjects[dindex] = dObjects[i]
 			continue
@@ -617,12 +616,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if replicateDeletes {
-			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
+			if dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending {
 				dv := DeletedObjectReplicationInfo{
 					DeletedObject: dobj,
 					Bucket:        bucket,
 				}
-				scheduleReplicationDelete(ctx, dv, objectAPI, replicateSync)
+				scheduleReplicationDelete(ctx, dv, objectAPI)
 			}
 		}
 
@@ -730,7 +729,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(ctx, bucket, false)
+					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
@@ -773,6 +772,17 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
+	if _, ok := err.(BucketExists); ok {
+		// Though bucket exists locally, we send the site-replication
+		// hook to ensure all sites have this bucket. If the hook
+		// succeeds, the client will still receive a bucket exists
+		// message.
+		err2 := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+		if err2 != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -781,6 +791,13 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	// Load updated bucket metadata into memory.
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
+	// Call site replication hook
+	err = globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Make sure to add Location information here only for bucket
 	if cp := pathClean(r.URL.Path); cp != "" {
 		w.Header().Set(xhttp.Location, cp) // Clean any trailing slashes.
@@ -980,7 +997,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, globalAutoEncryption)
+	sseConfig.Apply(r.Header, sse.ApplyOptions{
+		AutoEncrypt: globalAutoEncryption,
+		Passthrough: globalIsGateway && globalGatewayName == S3BackendGateway,
+	})
 
 	// get gateway encryption options
 	var opts ObjectOptions
@@ -1240,7 +1260,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, forceDelete); err != nil {
+	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
 		apiErr := toAPIError(ctx, err)
 		if _, ok := err.(BucketNotEmpty); ok {
 			if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1258,6 +1278,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 
+	// Call site replication hook.
+	if err := globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessNoContent(w)
 
@@ -1323,6 +1349,20 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
+		Bucket:           bucket,
+		ObjectLockConfig: &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1384,6 +1424,12 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketTaggingAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -1408,6 +1454,20 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeTags,
+		Bucket: bucket,
+		Tags:   &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1476,6 +1536,14 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeTags,
+		Bucket: bucket,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1519,9 +1587,9 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	sameTarget, err := validateReplicationDestination(ctx, bucket, replicationConfig)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig)
+	if apiErr != noError {
+		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 	// Validate the received bucket replication config
@@ -1646,32 +1714,19 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	bucketStats := globalNotificationSys.GetClusterBucketStats(r.Context(), bucket)
-	bucketReplStats := BucketReplicationStats{}
-	// sum up metrics from each node in the cluster
-	for _, bucketStat := range bucketStats {
-		bucketReplStats.FailedCount += bucketStat.ReplicationStats.FailedCount
-		bucketReplStats.FailedSize += bucketStat.ReplicationStats.FailedSize
-		bucketReplStats.PendingCount += bucketStat.ReplicationStats.PendingCount
-		bucketReplStats.PendingSize += bucketStat.ReplicationStats.PendingSize
-		bucketReplStats.ReplicaSize += bucketStat.ReplicationStats.ReplicaSize
-		bucketReplStats.ReplicatedSize += bucketStat.ReplicationStats.ReplicatedSize
+	var usageInfo BucketUsageInfo
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err == nil && !dataUsageInfo.LastUpdate.IsZero() {
+		usageInfo = dataUsageInfo.BucketsUsage[bucket]
 	}
-	// add initial usage from the time of cluster up
-	usageStat := globalReplicationStats.GetInitialUsage(bucket)
-	bucketReplStats.FailedCount += usageStat.FailedCount
-	bucketReplStats.FailedSize += usageStat.FailedSize
-	bucketReplStats.PendingCount += usageStat.PendingCount
-	bucketReplStats.PendingSize += usageStat.PendingSize
-	bucketReplStats.ReplicaSize += usageStat.ReplicaSize
-	bucketReplStats.ReplicatedSize += usageStat.ReplicatedSize
 
-	if err := json.NewEncoder(w).Encode(&bucketReplStats); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+	bucketReplStats := getLatestReplicationStats(bucket, usageInfo)
+	jsonData, err := json.Marshal(bucketReplStats)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	w.(http.Flusher).Flush()
+	writeSuccessResponseJSON(w, jsonData)
 }
 
 // ResetBucketReplicationStateHandler - starts a replication reset for all objects in a bucket which
@@ -1680,12 +1735,16 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 // remote target is entirely lost,and previously replicated objects need to be re-synced.
 func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ResetBucketReplicationState")
-
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	durationStr := r.Form.Get("older-than")
+	durationStr := r.URL.Query().Get("older-than")
+	arn := r.URL.Query().Get("arn")
+	resetID := r.URL.Query().Get("reset-id")
+	if resetID == "" {
+		resetID = mustGetUUID()
+	}
 	var (
 		days time.Duration
 		err  error
@@ -1726,9 +1785,31 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoMatchingRuleError), r.URL)
 		return
 	}
-	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, config.RoleArn)
+	tgtArns := config.FilterTargetArns(
+		replication.ObjectOpts{
+			OpType:    replication.ResyncReplicationType,
+			TargetArn: arn})
+
+	if len(tgtArns) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("Remote target ARN %s missing/not eligible for replication resync", arn),
+		}), r.URL)
+		return
+	}
+
+	if len(tgtArns) > 1 && arn == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("ARN should be specified for replication reset"),
+		}), r.URL)
+		return
+	}
+	var rinfo ResyncTargetsInfo
+	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArns[0])
 	target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
-	target.ResetID = mustGetUUID()
+	target.ResetID = resetID
+	rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
 	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
 		switch err.(type) {
 		case BucketRemoteConnectionErr:
@@ -1752,7 +1833,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	data, err := json.Marshal(target.ResetID)
+	data, err := json.Marshal(rinfo)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/bucket-lifecycle.go

@@ -75,8 +75,9 @@ func NewLifecycleSys() *LifecycleSys {
 }
 
 type expiryTask struct {
-	objInfo       ObjectInfo
-	versionExpiry bool
+	objInfo        ObjectInfo
+	versionExpiry  bool
+	restoredObject bool
 }
 
 type expiryState struct {
@@ -89,13 +90,13 @@ func (es *expiryState) PendingTasks() int {
 	return len(es.expiryCh)
 }
 
-func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) {
+func (es *expiryState) queueExpiryTask(oi ObjectInfo, restoredObject bool, rmVersion bool) {
 	select {
 	case <-GlobalContext.Done():
 		es.once.Do(func() {
 			close(es.expiryCh)
 		})
-	case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion}:
+	case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion, restoredObject: restoredObject}:
 	default:
 	}
 }
@@ -114,7 +115,11 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()
 	go func() {
 		for t := range globalExpiryState.expiryCh {
-			applyExpiryRule(ctx, objectAPI, t.objInfo, false, t.versionExpiry)
+			if t.objInfo.TransitionedObject.Status != "" {
+				applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+			} else {
+				applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+			}
 		}
 	}()
 }
@@ -215,16 +220,31 @@ var errInvalidStorageClass = errors.New("invalid storage class")
 
 func validateTransitionTier(lc *lifecycle.Lifecycle) error {
 	for _, rule := range lc.Rules {
-		if rule.Transition.StorageClass == "" {
-			continue
+		if rule.Transition.StorageClass != "" {
+			if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
+				return errInvalidStorageClass
+			}
 		}
-		if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
-			return errInvalidStorageClass
+		if rule.NoncurrentVersionTransition.StorageClass != "" {
+			if valid := globalTierConfigMgr.IsTierValid(rule.NoncurrentVersionTransition.StorageClass); !valid {
+				return errInvalidStorageClass
+			}
 		}
 	}
 	return nil
 }
 
+// enqueueTransitionImmediate enqueues obj for transition if eligible.
+// This is to be called after a successful upload of an object (version).
+func enqueueTransitionImmediate(obj ObjectInfo) {
+	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
+		switch lc.ComputeAction(obj.ToLifecycleOpts()) {
+		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+			globalTransitionState.queueTransitionTask(obj)
+		}
+	}
+}
+
 // expireAction represents different actions to be performed on expiry of a
 // restored/transitioned object
 type expireAction int
@@ -247,6 +267,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
 	opts.VersionID = lcOpts.VersionID
+	opts.Expiration = ExpirationOptions{Expire: true}
 	switch action {
 	case expireObj:
 		// When an object is past expiry or when a transitioned object is being
@@ -327,9 +348,10 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 			Tier:   lc.TransitionTier(oi.ToLifecycleOpts()),
 			ETag:   oi.ETag,
 		},
-		VersionID: oi.VersionID,
-		Versioned: globalBucketVersioningSys.Enabled(oi.Bucket),
-		MTime:     oi.ModTime,
+		VersionID:        oi.VersionID,
+		Versioned:        globalBucketVersioningSys.Enabled(oi.Bucket),
+		VersionSuspended: globalBucketVersioningSys.Suspended(oi.Bucket),
+		MTime:            oi.ModTime,
 	}
 	return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
@@ -373,9 +395,9 @@ const (
 
 // Encryption specifies encryption setting on restored bucket
 type Encryption struct {
-	EncryptionType sse.SSEAlgorithm `xml:"EncryptionType"`
-	KMSContext     string           `xml:"KMSContext,omitempty"`
-	KMSKeyID       string           `xml:"KMSKeyId,omitempty"`
+	EncryptionType sse.Algorithm `xml:"EncryptionType"`
+	KMSContext     string        `xml:"KMSContext,omitempty"`
+	KMSKeyID       string        `xml:"KMSKeyId,omitempty"`
 }
 
 // MetadataEntry denotes name and value.
@@ -695,17 +717,16 @@ func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) {
 // ToLifecycleOpts returns lifecycle.ObjectOpts value for oi.
 func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
 	return lifecycle.ObjectOpts{
-		Name:                   oi.Name,
-		UserTags:               oi.UserTags,
-		VersionID:              oi.VersionID,
-		ModTime:                oi.ModTime,
-		IsLatest:               oi.IsLatest,
-		NumVersions:            oi.NumVersions,
-		DeleteMarker:           oi.DeleteMarker,
-		SuccessorModTime:       oi.SuccessorModTime,
-		RestoreOngoing:         oi.RestoreOngoing,
-		RestoreExpires:         oi.RestoreExpires,
-		TransitionStatus:       oi.TransitionedObject.Status,
-		RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
+		Name:             oi.Name,
+		UserTags:         oi.UserTags,
+		VersionID:        oi.VersionID,
+		ModTime:          oi.ModTime,
+		IsLatest:         oi.IsLatest,
+		NumVersions:      oi.NumVersions,
+		DeleteMarker:     oi.DeleteMarker,
+		SuccessorModTime: oi.SuccessorModTime,
+		RestoreOngoing:   oi.RestoreOngoing,
+		RestoreExpires:   oi.RestoreExpires,
+		TransitionStatus: oi.TransitionedObject.Status,
 	}
 }

cmd/bucket-listobjects-handlers.go

@@ -128,7 +128,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 10000)
+// This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
@@ -195,7 +195,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 10000)
+// This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
@@ -305,7 +305,7 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
 // --------------------------
-// This implementation of the GET operation returns some or all (up to 10000)
+// This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //

cmd/bucket-metadata-sys.go

@@ -80,46 +80,8 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		return errServerNotInitialized
 	}
 
-	if globalIsGateway {
-		// This code is needed only for gateway implementations.
-		switch configFile {
-		case bucketSSEConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.EncryptionConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketLifecycleConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.LifecycleConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketTaggingConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.TaggingConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketNotificationConfig:
-			if globalGatewayName == NASBackendGateway {
-				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
-				if err != nil {
-					return err
-				}
-				meta.NotificationConfigXML = configData
-				return meta.Save(GlobalContext, objAPI)
-			}
-		case bucketPolicyConfig:
+	if globalIsGateway && globalGatewayName != NASBackendGateway {
+		if configFile == bucketPolicyConfig {
 			if configData == nil {
 				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
 			}
@@ -138,7 +100,12 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 
 	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 	if err != nil {
-		return err
+		if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
+			// Only single drive mode needs this fallback.
+			meta = newBucketMetadata(bucket)
+		} else {
+			return err
+		}
 	}
 
 	switch configFile {
@@ -447,9 +414,9 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return errServerNotInitialized
 	}
 
-	// In gateway mode, we don't need to load the policies
-	// from the backend.
-	if globalIsGateway {
+	// In gateway mode, we don't need to load bucket metadata except
+	// NAS gateway backend.
+	if globalIsGateway && !objAPI.IsNotificationSupported() {
 		return nil
 	}
 
@@ -470,11 +437,20 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			})
 			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
 			if err != nil {
-				return err
+				if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
+					meta = newBucketMetadata(buckets[index].Name)
+				} else {
+					return err
+				}
 			}
 			sys.Lock()
 			sys.metadataMap[buckets[index].Name] = meta
 			sys.Unlock()
+
+			globalNotificationSys.set(buckets[index], meta) // set notification targets
+
+			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
+
 			return nil
 		}, index)
 	}

cmd/bucket-object-lock.go

@@ -93,7 +93,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 	if gerr != nil { // error from GetObjectInfo
 		switch gerr.(type) {
 		case MethodNotAllowed: // This happens usually for a delete marker
-			if oi.DeleteMarker {
+			if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 				// Delete marker should be present and valid.
 				return ErrNone
 			}

cmd/bucket-policy-handlers.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"io"
+	"io/ioutil"
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -76,7 +79,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
+	bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -99,6 +108,16 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
+	// Call site replication hook.
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypePolicy,
+		Bucket: bucket,
+		Policy: bucketPolicyBytes,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Success.
 	writeSuccessNoContent(w)
 }
@@ -134,6 +153,15 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Call site replication hook.
+	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypePolicy,
+		Bucket: bucket,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Success.
 	writeSuccessNoContent(w)
 }

cmd/bucket-quota.go

@@ -89,7 +89,10 @@ func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64)
 			return err
 		}
 
-		dui := v.(madmin.DataUsageInfo)
+		dui, ok := v.(DataUsageInfo)
+		if !ok {
+			return fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
+		}
 
 		bui, ok := dui.BucketsUsage[bucket]
 		if !ok {
@@ -116,7 +119,7 @@ func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
 
 // enforceFIFOQuota deletes objects in FIFO order until sufficient objects
 // have been deleted so as to bring bucket usage within quota.
-func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui madmin.BucketUsageInfo) {
+func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui BucketUsageInfo) {
 	// Check if the current bucket has quota restrictions, if not skip it
 	cfg, err := globalBucketQuotaSys.Get(bucket)
 	if err != nil {

cmd/bucket-replication-stats.go

@@ -21,15 +21,18 @@ import (
 	"context"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
 )
 
 func (b *BucketReplicationStats) hasReplicationUsage() bool {
-	return b.FailedSize > 0 ||
-		b.ReplicatedSize > 0 ||
-		b.ReplicaSize > 0 ||
-		b.FailedCount > 0
+	for _, s := range b.Stats {
+		if s.hasReplicationUsage() {
+			return true
+		}
+	}
+	return false
 }
 
 // ReplicationStats holds the global in-memory replication stats
@@ -37,6 +40,7 @@ type ReplicationStats struct {
 	Cache      map[string]*BucketReplicationStats
 	UsageCache map[string]*BucketReplicationStats
 	sync.RWMutex
+	ulock sync.RWMutex
 }
 
 // Delete deletes in-memory replication statistics for a bucket.
@@ -52,45 +56,65 @@ func (r *ReplicationStats) Delete(bucket string) {
 
 }
 
-// Update updates in-memory replication statistics with new values.
-func (r *ReplicationStats) Update(bucket string, n int64, status, prevStatus replication.StatusType, opType replication.Type) {
+// UpdateReplicaStat updates in-memory replica statistics with new values.
+func (r *ReplicationStats) UpdateReplicaStat(bucket string, n int64) {
 	if r == nil {
 		return
 	}
 
-	r.RLock()
-	b, ok := r.Cache[bucket]
+	r.Lock()
+	defer r.Unlock()
+	bs, ok := r.Cache[bucket]
 	if !ok {
-		b = &BucketReplicationStats{}
+		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+	}
+	atomic.AddInt64(&bs.ReplicaSize, n)
+	r.Cache[bucket] = bs
+}
+
+// Update updates in-memory replication statistics with new values.
+func (r *ReplicationStats) Update(bucket string, arn string, n int64, status, prevStatus replication.StatusType, opType replication.Type) {
+	if r == nil {
+		return
+	}
+	r.RLock()
+	bs, ok := r.Cache[bucket]
+	if !ok {
+		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+	}
+	b, ok := bs.Stats[arn]
+	if !ok {
+		b = &BucketReplicationStat{}
 	}
 	r.RUnlock()
 	switch status {
 	case replication.Completed:
 		switch prevStatus { // adjust counters based on previous state
 		case replication.Failed:
-			atomic.AddUint64(&b.FailedCount, ^uint64(0))
+			atomic.AddInt64(&b.FailedCount, -1)
 		}
 		if opType == replication.ObjectReplicationType {
-			atomic.AddUint64(&b.ReplicatedSize, uint64(n))
+			atomic.AddInt64(&b.ReplicatedSize, n)
 			switch prevStatus {
 			case replication.Failed:
-				atomic.AddUint64(&b.FailedSize, ^uint64(n-1))
+				atomic.AddInt64(&b.FailedSize, -1*n)
 			}
 		}
 	case replication.Failed:
 		if opType == replication.ObjectReplicationType {
 			if prevStatus == replication.Pending {
-				atomic.AddUint64(&b.FailedSize, uint64(n))
-				atomic.AddUint64(&b.FailedCount, 1)
+				atomic.AddInt64(&b.FailedSize, n)
+				atomic.AddInt64(&b.FailedCount, 1)
 			}
 		}
 	case replication.Replica:
 		if opType == replication.ObjectReplicationType {
-			atomic.AddUint64(&b.ReplicaSize, uint64(n))
+			atomic.AddInt64(&b.ReplicaSize, n)
 		}
 	}
 	r.Lock()
-	r.Cache[bucket] = b
+	bs.Stats[arn] = b
+	r.Cache[bucket] = bs
 	r.Unlock()
 }
 
@@ -99,26 +123,19 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 	if r == nil {
 		return BucketReplicationStats{}
 	}
-
-	r.RLock()
-	defer r.RUnlock()
-
+	r.ulock.RLock()
+	defer r.ulock.RUnlock()
 	st, ok := r.UsageCache[bucket]
-	if !ok {
-		return BucketReplicationStats{}
-	}
-	return BucketReplicationStats{
-		FailedSize:     atomic.LoadUint64(&st.FailedSize),
-		ReplicatedSize: atomic.LoadUint64(&st.ReplicatedSize),
-		ReplicaSize:    atomic.LoadUint64(&st.ReplicaSize),
-		FailedCount:    atomic.LoadUint64(&st.FailedCount),
+	if ok {
+		return st.Clone()
 	}
+	return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
 }
 
 // Get replication metrics for a bucket from this node since this node came up.
 func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 	if r == nil {
-		return BucketReplicationStats{}
+		return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
 	}
 
 	r.RLock()
@@ -126,45 +143,58 @@ func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 
 	st, ok := r.Cache[bucket]
 	if !ok {
-		return BucketReplicationStats{}
-	}
-
-	return BucketReplicationStats{
-		FailedSize:     atomic.LoadUint64(&st.FailedSize),
-		ReplicatedSize: atomic.LoadUint64(&st.ReplicatedSize),
-		ReplicaSize:    atomic.LoadUint64(&st.ReplicaSize),
-		FailedCount:    atomic.LoadUint64(&st.FailedCount),
+		return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
 	}
+	return st.Clone()
 }
 
 // NewReplicationStats initialize in-memory replication statistics
 func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *ReplicationStats {
-	st := &ReplicationStats{
+	return &ReplicationStats{
 		Cache:      make(map[string]*BucketReplicationStats),
 		UsageCache: make(map[string]*BucketReplicationStats),
 	}
-
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err != nil {
-		return st
-	}
-
-	// data usage has not captured any data yet.
-	if dataUsageInfo.LastUpdate.IsZero() {
-		return st
-	}
-
-	for bucket, usage := range dataUsageInfo.BucketsUsage {
-		b := &BucketReplicationStats{
-			FailedSize:     usage.ReplicationFailedSize,
-			ReplicatedSize: usage.ReplicatedSize,
-			ReplicaSize:    usage.ReplicaSize,
-			FailedCount:    usage.ReplicationFailedCount,
-		}
-		if b.hasReplicationUsage() {
-			st.UsageCache[bucket] = b
-		}
-	}
-
-	return st
+}
+
+// load replication metrics at cluster start from initial data usage
+func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
+	rTimer := time.NewTimer(time.Minute * 1)
+	defer rTimer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-rTimer.C:
+			dui, err := loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
+			if err != nil {
+				continue
+			}
+			// data usage has not captured any data yet.
+			if dui.LastUpdate.IsZero() {
+				continue
+			}
+			m := make(map[string]*BucketReplicationStats)
+			for bucket, usage := range dui.BucketsUsage {
+				b := &BucketReplicationStats{
+					Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
+				}
+				for arn, uinfo := range usage.ReplicationInfo {
+					b.Stats[arn] = &BucketReplicationStat{
+						FailedSize:     int64(uinfo.ReplicationFailedSize),
+						ReplicatedSize: int64(uinfo.ReplicatedSize),
+						ReplicaSize:    int64(uinfo.ReplicaSize),
+						FailedCount:    int64(uinfo.ReplicationFailedCount),
+					}
+				}
+				b.ReplicaSize += int64(usage.ReplicaSize)
+				if b.hasReplicationUsage() {
+					m[bucket] = b
+				}
+			}
+			r.ulock.Lock()
+			defer r.ulock.Unlock()
+			r.UsageCache = m
+			return
+		}
+	}
 }

cmd/bucket-replication-utils.go

@@ -0,0 +1,602 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/replication"
+)
+
+//go:generate msgp -file=$GOFILE
+
+// replicatedTargetInfo struct represents replication info on a target
+type replicatedTargetInfo struct {
+	Arn                   string
+	Size                  int64
+	ReplicationAction     replicationAction // full or metadata only
+	OpType                replication.Type  // whether incoming replication, existing object, healing etc..
+	ReplicationStatus     replication.StatusType
+	PrevReplicationStatus replication.StatusType
+	VersionPurgeStatus    VersionPurgeStatusType
+	ResyncTimestamp       string
+	ReplicationResynced   bool // true only if resync attempted for this target
+}
+
+// Empty returns true for a target if arn is empty
+func (rt replicatedTargetInfo) Empty() bool {
+	return rt.Arn == ""
+}
+
+type replicatedInfos struct {
+	ReplicationTimeStamp time.Time
+	Targets              []replicatedTargetInfo
+}
+
+func (ri replicatedInfos) CompletedSize() (sz int64) {
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		if t.ReplicationStatus == replication.Completed && t.PrevReplicationStatus != replication.Completed {
+			sz += t.Size
+		}
+	}
+	return sz
+}
+
+// ReplicationAttempted returns true if replication was attempted on any of the targets for the object version
+// queued
+func (ri replicatedInfos) ReplicationResynced() bool {
+	for _, t := range ri.Targets {
+		if t.Empty() || !t.ReplicationResynced {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+func (ri replicatedInfos) ReplicationStatusInternal() string {
+	b := new(bytes.Buffer)
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		fmt.Fprintf(b, "%s=%s;", t.Arn, t.ReplicationStatus.String())
+	}
+	return b.String()
+}
+
+func (ri replicatedInfos) ReplicationStatus() replication.StatusType {
+	if len(ri.Targets) == 0 {
+		return replication.StatusType("")
+	}
+	completed := 0
+	for _, v := range ri.Targets {
+		switch v.ReplicationStatus {
+		case replication.Failed:
+			return replication.Failed
+		case replication.Completed:
+			completed++
+		}
+	}
+	if completed == len(ri.Targets) {
+		return replication.Completed
+	}
+	return replication.Pending
+}
+
+func (ri replicatedInfos) VersionPurgeStatus() VersionPurgeStatusType {
+	if len(ri.Targets) == 0 {
+		return VersionPurgeStatusType("")
+	}
+	completed := 0
+	for _, v := range ri.Targets {
+		switch v.VersionPurgeStatus {
+		case Failed:
+			return Failed
+		case Complete:
+			completed++
+		}
+	}
+	if completed == len(ri.Targets) {
+		return Complete
+	}
+	return Pending
+}
+
+func (ri replicatedInfos) VersionPurgeStatusInternal() string {
+	b := new(bytes.Buffer)
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		if t.VersionPurgeStatus.Empty() {
+			continue
+		}
+		fmt.Fprintf(b, "%s=%s;", t.Arn, t.VersionPurgeStatus)
+	}
+	return b.String()
+}
+
+func (ri replicatedInfos) Action() replicationAction {
+	for _, t := range ri.Targets {
+		if t.Empty() {
+			continue
+		}
+		// rely on replication action from target that actually performed replication now.
+		if t.PrevReplicationStatus != replication.Completed {
+			return t.ReplicationAction
+		}
+	}
+	return replicateNone
+}
+
+var replStatusRegex = regexp.MustCompile(`([^=].*?)=([^,].*?);`)
+
+// TargetReplicationStatus - returns replication status of a target
+func (o *ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+	repStatMatches := replStatusRegex.FindAllStringSubmatch(o.ReplicationStatusInternal, -1)
+	for _, repStatMatch := range repStatMatches {
+		if len(repStatMatch) != 3 {
+			return
+		}
+		if repStatMatch[1] == arn {
+			return replication.StatusType(repStatMatch[2])
+		}
+	}
+	return
+}
+
+type replicateTargetDecision struct {
+	Replicate   bool   // Replicate to this target
+	Synchronous bool   // Synchronous replication configured.
+	Arn         string //ARN of replication target
+	ID          string
+}
+
+func (t *replicateTargetDecision) String() string {
+	return fmt.Sprintf("%t;%t;%s;%s", t.Replicate, t.Synchronous, t.Arn, t.ID)
+}
+
+func newReplicateTargetDecision(arn string, replicate bool, sync bool) replicateTargetDecision {
+	d := replicateTargetDecision{
+		Replicate:   replicate,
+		Synchronous: sync,
+		Arn:         arn,
+	}
+	return d
+}
+
+// ReplicateDecision represents replication decision for each target
+type ReplicateDecision struct {
+	targetsMap map[string]replicateTargetDecision
+}
+
+// ReplicateAny returns true if atleast one target qualifies for replication
+func (d *ReplicateDecision) ReplicateAny() bool {
+	for _, t := range d.targetsMap {
+		if t.Replicate {
+			return true
+		}
+	}
+	return false
+}
+
+// Synchronous returns true if atleast one target qualifies for synchronous replication
+func (d *ReplicateDecision) Synchronous() bool {
+	for _, t := range d.targetsMap {
+		if t.Synchronous {
+			return true
+		}
+	}
+	return false
+}
+
+func (d *ReplicateDecision) String() string {
+	b := new(bytes.Buffer)
+	for key, value := range d.targetsMap {
+		fmt.Fprintf(b, "%s=%s,", key, value.String())
+	}
+	return strings.TrimSuffix(b.String(), ",")
+}
+
+// Set updates ReplicateDecision with target's replication decision
+func (d *ReplicateDecision) Set(t replicateTargetDecision) {
+	if d.targetsMap == nil {
+		d.targetsMap = make(map[string]replicateTargetDecision)
+	}
+	d.targetsMap[t.Arn] = t
+}
+
+// PendingStatus returns a stringified representation of internal replication status with all targets marked as `PENDING`
+func (d *ReplicateDecision) PendingStatus() string {
+	b := new(bytes.Buffer)
+	for _, k := range d.targetsMap {
+		if k.Replicate {
+			fmt.Fprintf(b, "%s=%s;", k.Arn, replication.Pending.String())
+		}
+	}
+	return b.String()
+}
+
+// ResyncDecision is a struct representing a map with target's individual resync decisions
+type ResyncDecision struct {
+	targets map[string]ResyncTargetDecision
+}
+
+// Empty returns true if no targets with resync decision present
+func (r *ResyncDecision) Empty() bool {
+	return r.targets == nil
+}
+
+func (r *ResyncDecision) mustResync() bool {
+	for _, v := range r.targets {
+		if v.Replicate {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *ResyncDecision) mustResyncTarget(tgtArn string) bool {
+	if r.targets == nil {
+		return false
+	}
+	v, ok := r.targets[tgtArn]
+	if ok && v.Replicate {
+		return true
+	}
+	return false
+}
+
+// ResyncTargetDecision is struct that represents resync decision for this target
+type ResyncTargetDecision struct {
+	Replicate       bool
+	ResetID         string
+	ResetBeforeDate time.Time
+}
+
+var errInvalidReplicateDecisionFormat = fmt.Errorf("ReplicateDecision has invalid format")
+
+// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
+// ReplicateDecision struct
+func parseReplicateDecision(s string) (r ReplicateDecision, err error) {
+	r = ReplicateDecision{
+		targetsMap: make(map[string]replicateTargetDecision),
+	}
+	if len(s) == 0 {
+		return
+	}
+	pairs := strings.Split(s, ",")
+	for _, p := range pairs {
+		slc := strings.Split(p, "=")
+		if len(slc) != 2 {
+			return r, errInvalidReplicateDecisionFormat
+		}
+		tgtStr := strings.TrimPrefix(slc[1], "\"")
+		tgtStr = strings.TrimSuffix(tgtStr, "\"")
+		tgt := strings.Split(tgtStr, ";")
+		if len(tgt) != 4 {
+			return r, errInvalidReplicateDecisionFormat
+		}
+		var replicate, sync bool
+		var err error
+		replicate, err = strconv.ParseBool(tgt[0])
+		if err != nil {
+			return r, err
+		}
+		sync, err = strconv.ParseBool(tgt[1])
+		if err != nil {
+			return r, err
+		}
+		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3]}
+	}
+	return
+}
+
+// ReplicationState represents internal replication state
+type ReplicationState struct {
+	ReplicaTimeStamp          time.Time              // timestamp when last replica update was received
+	ReplicaStatus             replication.StatusType // replica statusstringis
+	DeleteMarker              bool                   // represents DeleteMarker replication state
+	ReplicationTimeStamp      time.Time              // timestamp when last replication activity happened
+	ReplicationStatusInternal string                 // stringified representation of all replication activity
+	// VersionPurgeStatusInternal is internally in the format "arn1=PENDING;arn2=COMMPLETED;"
+	VersionPurgeStatusInternal string                            // stringified representation of all version purge statuses
+	ReplicateDecisionStr       string                            // stringified representation of replication decision for each target
+	Targets                    map[string]replication.StatusType // map of ARN->replication status for ongoing replication activity
+	PurgeTargets               map[string]VersionPurgeStatusType // map of ARN->VersionPurgeStatus for all the targets
+	ResetStatusesMap           map[string]string                 // map of ARN-> stringified reset id and timestamp for all the targets
+}
+
+// Equal returns true if replication state is identical for version purge statuses and (replica)tion statuses.
+func (rs *ReplicationState) Equal(o ReplicationState) bool {
+	return rs.ReplicaStatus == o.ReplicaStatus &&
+		rs.ReplicaTimeStamp.Equal(o.ReplicaTimeStamp) &&
+		rs.ReplicationTimeStamp.Equal(o.ReplicationTimeStamp) &&
+		rs.ReplicationStatusInternal == o.ReplicationStatusInternal &&
+		rs.VersionPurgeStatusInternal == o.VersionPurgeStatusInternal
+}
+
+// CompositeReplicationStatus returns overall replication status for the object version being replicated.
+func (rs *ReplicationState) CompositeReplicationStatus() (st replication.StatusType) {
+	switch {
+	case rs.ReplicationStatusInternal != "":
+		switch replication.StatusType(rs.ReplicationStatusInternal) {
+		case replication.Pending, replication.Completed, replication.Failed, replication.Replica: // for backward compatibility
+			return replication.StatusType(rs.ReplicationStatusInternal)
+		default:
+			replStatus := getCompositeReplicationStatus(rs.Targets)
+			// return REPLICA status if replica received timestamp is later than replication timestamp
+			// provided object replication completed for all targets.
+			if !rs.ReplicaTimeStamp.Equal(timeSentinel) && replStatus == replication.Completed && rs.ReplicaTimeStamp.After(rs.ReplicationTimeStamp) {
+				return rs.ReplicaStatus
+			}
+			return replStatus
+		}
+	case !rs.ReplicaStatus.Empty():
+		return rs.ReplicaStatus
+	default:
+		return
+	}
+}
+
+// CompositeVersionPurgeStatus returns overall replication purge status for the permanent delete being replicated.
+func (rs *ReplicationState) CompositeVersionPurgeStatus() VersionPurgeStatusType {
+	switch VersionPurgeStatusType(rs.VersionPurgeStatusInternal) {
+	case Pending, Complete, Failed: // for backward compatibility
+		return VersionPurgeStatusType(rs.VersionPurgeStatusInternal)
+	default:
+		return getCompositeVersionPurgeStatus(rs.PurgeTargets)
+	}
+}
+
+// TargetState returns replicatedInfos struct initialized with the previous state of replication
+func (rs *ReplicationState) targetState(arn string) (r replicatedTargetInfo) {
+	return replicatedTargetInfo{
+		Arn:                   arn,
+		PrevReplicationStatus: rs.Targets[arn],
+		VersionPurgeStatus:    rs.PurgeTargets[arn],
+		ResyncTimestamp:       rs.ResetStatusesMap[arn],
+	}
+}
+
+// getReplicationState returns replication state using target replicated info for the targets
+func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID string) ReplicationState {
+	rs := ReplicationState{
+		ReplicateDecisionStr: prevState.ReplicateDecisionStr,
+		ResetStatusesMap:     prevState.ResetStatusesMap,
+		ReplicaTimeStamp:     prevState.ReplicaTimeStamp,
+		ReplicaStatus:        prevState.ReplicaStatus,
+	}
+	var replStatuses, vpurgeStatuses string
+	replStatuses = rinfos.ReplicationStatusInternal()
+	rs.Targets = replicationStatusesMap(replStatuses)
+	rs.ReplicationStatusInternal = replStatuses
+	rs.ReplicationTimeStamp = rinfos.ReplicationTimeStamp
+
+	vpurgeStatuses = rinfos.VersionPurgeStatusInternal()
+	rs.VersionPurgeStatusInternal = vpurgeStatuses
+	rs.PurgeTargets = versionPurgeStatusesMap(vpurgeStatuses)
+
+	for _, rinfo := range rinfos.Targets {
+		if rinfo.ResyncTimestamp != "" {
+			rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
+		}
+	}
+	return rs
+}
+
+// constructs a replication status map from string representation
+func replicationStatusesMap(s string) map[string]replication.StatusType {
+	targets := make(map[string]replication.StatusType)
+	repStatMatches := replStatusRegex.FindAllStringSubmatch(s, -1)
+	for _, repStatMatch := range repStatMatches {
+		if len(repStatMatch) != 3 {
+			continue
+		}
+		status := replication.StatusType(repStatMatch[2])
+		targets[repStatMatch[1]] = status
+	}
+	return targets
+}
+
+// constructs a version purge status map from string representation
+func versionPurgeStatusesMap(s string) map[string]VersionPurgeStatusType {
+	targets := make(map[string]VersionPurgeStatusType)
+	purgeStatusMatches := replStatusRegex.FindAllStringSubmatch(s, -1)
+	for _, purgeStatusMatch := range purgeStatusMatches {
+		if len(purgeStatusMatch) != 3 {
+			continue
+		}
+		targets[purgeStatusMatch[1]] = VersionPurgeStatusType(purgeStatusMatch[2])
+	}
+	return targets
+}
+
+// return the overall replication status for all the targets
+func getCompositeReplicationStatus(m map[string]replication.StatusType) replication.StatusType {
+	if len(m) == 0 {
+		return replication.StatusType("")
+	}
+	completed := 0
+	for _, v := range m {
+		switch v {
+		case replication.Failed:
+			return replication.Failed
+		case replication.Completed:
+			completed++
+		}
+	}
+	if completed == len(m) {
+		return replication.Completed
+	}
+	return replication.Pending
+}
+
+// return the overall version purge status for all the targets
+func getCompositeVersionPurgeStatus(m map[string]VersionPurgeStatusType) VersionPurgeStatusType {
+	if len(m) == 0 {
+		return VersionPurgeStatusType("")
+	}
+	completed := 0
+	for _, v := range m {
+		switch v {
+		case Failed:
+			return Failed
+		case Complete:
+			completed++
+		}
+	}
+	if completed == len(m) {
+		return Complete
+	}
+	return Pending
+}
+
+// getHealReplicateObjectInfo returns info needed by heal replication in ReplicateObjectInfo
+func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
+	oi := objInfo.Clone()
+	if rcfg.Config != nil && rcfg.Config.RoleArn != "" {
+		// For backward compatibility of objects pending/failed replication.
+		// Save replication related statuses in the new internal representation for
+		// compatible behavior.
+		if !oi.ReplicationStatus.Empty() {
+			oi.ReplicationStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.ReplicationStatus)
+		}
+		if !oi.VersionPurgeStatus.Empty() {
+			oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
+		}
+		for k, v := range oi.UserDefined {
+			switch {
+			case strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset):
+				delete(oi.UserDefined, k)
+				oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
+			}
+		}
+	}
+	var dsc ReplicateDecision
+	var tgtStatuses map[string]replication.StatusType
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
+			ObjectName: oi.Name,
+			VersionID:  oi.VersionID,
+		}, oi, ObjectOptions{}, nil)
+	} else {
+		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
+			UserDefined: oi.UserDefined,
+		}, replication.HealReplicationType, ObjectOptions{}))
+	}
+	tgtStatuses = replicationStatusesMap(oi.ReplicationStatusInternal)
+
+	existingObjResync := rcfg.Resync(GlobalContext, oi, &dsc, tgtStatuses)
+
+	return ReplicateObjectInfo{
+		ObjectInfo:        oi,
+		OpType:            replication.HealReplicationType,
+		Dsc:               dsc,
+		ExistingObjResync: existingObjResync,
+		TargetStatuses:    tgtStatuses,
+	}
+}
+
+// vID here represents the versionID client specified in request - need to distinguish between delete marker and delete marker deletion
+func (o *ObjectInfo) getReplicationState(dsc string, vID string, heal bool) ReplicationState {
+	rs := ReplicationState{
+		ReplicationStatusInternal:  o.ReplicationStatusInternal,
+		VersionPurgeStatusInternal: o.VersionPurgeStatusInternal,
+		ReplicateDecisionStr:       dsc,
+		Targets:                    make(map[string]replication.StatusType),
+		PurgeTargets:               make(map[string]VersionPurgeStatusType),
+		ResetStatusesMap:           make(map[string]string),
+	}
+	rs.Targets = replicationStatusesMap(o.ReplicationStatusInternal)
+	rs.PurgeTargets = versionPurgeStatusesMap(o.VersionPurgeStatusInternal)
+	for k, v := range o.UserDefined {
+		if strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset) {
+			arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset))
+			rs.ResetStatusesMap[arn] = v
+		}
+	}
+	return rs
+}
+
+// ReplicationState returns replication state using other internal replication metadata in ObjectToDelete
+func (o *ObjectToDelete) ReplicationState() ReplicationState {
+	r := ReplicationState{
+		ReplicationStatusInternal:  o.DeleteMarkerReplicationStatus,
+		VersionPurgeStatusInternal: o.VersionPurgeStatuses,
+		ReplicateDecisionStr:       o.ReplicateDecisionStr,
+	}
+
+	r.Targets = replicationStatusesMap(o.DeleteMarkerReplicationStatus)
+	r.PurgeTargets = versionPurgeStatusesMap(o.VersionPurgeStatuses)
+	return r
+}
+
+// VersionPurgeStatus returns a composite version purge status across targets
+func (d *DeletedObject) VersionPurgeStatus() VersionPurgeStatusType {
+	return d.ReplicationState.CompositeVersionPurgeStatus()
+}
+
+// DeleteMarkerReplicationStatus return composite replication status of delete marker across targets
+func (d *DeletedObject) DeleteMarkerReplicationStatus() replication.StatusType {
+	return d.ReplicationState.CompositeReplicationStatus()
+}
+
+// ResyncTargetsInfo holds a slice of targets with resync info per target
+type ResyncTargetsInfo struct {
+	Targets []ResyncTarget `json:"target,omitempty"`
+}
+
+// ResyncTarget is a struct representing the Target reset ID where target is identified by its Arn
+type ResyncTarget struct {
+	Arn     string `json:"arn"`
+	ResetID string `json:"resetid"`
+}
+
+// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication
+type VersionPurgeStatusType string
+
+const (
+	// Pending - versioned delete replication is pending.
+	Pending VersionPurgeStatusType = "PENDING"
+
+	// Complete - versioned delete replication is now complete, erase version on disk.
+	Complete VersionPurgeStatusType = "COMPLETE"
+
+	// Failed - versioned delete replication failed.
+	Failed VersionPurgeStatusType = "FAILED"
+)
+
+// Empty returns true if purge status was not set.
+func (v VersionPurgeStatusType) Empty() bool {
+	return string(v) == ""
+}
+
+// Pending returns true if the version is pending purge.
+func (v VersionPurgeStatusType) Pending() bool {
+	return v == Pending || v == Failed
+}

cmd/bucket-replication-utils_gen_test.go

@@ -0,0 +1,688 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
+	v := ReplicateDecision{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicateDecision(t *testing.T) {
+	v := ReplicateDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicateDecision Msgsize() is inaccurate")
+	}
+
+	vn := ReplicateDecision{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicateDecision(b *testing.B) {
+	v := ReplicateDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalReplicationState(t *testing.T) {
+	v := ReplicationState{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicationState(t *testing.T) {
+	v := ReplicationState{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicationState Msgsize() is inaccurate")
+	}
+
+	vn := ReplicationState{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicationState(b *testing.B) {
+	v := ReplicationState{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncDecision(t *testing.T) {
+	v := ResyncDecision{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncDecision(t *testing.T) {
+	v := ResyncDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncDecision Msgsize() is inaccurate")
+	}
+
+	vn := ResyncDecision{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncDecision(b *testing.B) {
+	v := ResyncDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncTarget(t *testing.T) {
+	v := ResyncTarget{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncTarget(t *testing.T) {
+	v := ResyncTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncTarget Msgsize() is inaccurate")
+	}
+
+	vn := ResyncTarget{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncTarget(b *testing.B) {
+	v := ResyncTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncTargetDecision(t *testing.T) {
+	v := ResyncTargetDecision{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncTargetDecision(t *testing.T) {
+	v := ResyncTargetDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncTargetDecision Msgsize() is inaccurate")
+	}
+
+	vn := ResyncTargetDecision{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncTargetDecision(b *testing.B) {
+	v := ResyncTargetDecision{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalResyncTargetsInfo(t *testing.T) {
+	v := ResyncTargetsInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeResyncTargetsInfo(t *testing.T) {
+	v := ResyncTargetsInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeResyncTargetsInfo Msgsize() is inaccurate")
+	}
+
+	vn := ResyncTargetsInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeResyncTargetsInfo(b *testing.B) {
+	v := ResyncTargetsInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-replication-utils_test.go

@@ -0,0 +1,247 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+
+	"github.com/minio/minio/internal/bucket/replication"
+)
+
+var replicatedInfosTests = []struct {
+	name                              string
+	tgtInfos                          []replicatedTargetInfo
+	expectedCompletedSize             int64
+	expectedReplicationStatusInternal string
+	expectedReplicationStatus         replication.StatusType
+	expectedOpType                    replication.Type
+	expectedAction                    replicationAction
+}{
+	{ //1. empty tgtInfos slice
+		name:                              "no replicated targets",
+		tgtInfos:                          []replicatedTargetInfo{},
+		expectedCompletedSize:             0,
+		expectedReplicationStatusInternal: "",
+		expectedReplicationStatus:         replication.StatusType(""),
+		expectedOpType:                    replication.UnsetReplicationType,
+		expectedAction:                    replicateNone,
+	},
+	{ //2. replication completed to single target
+		name: "replication completed to single target",
+		tgtInfos: []replicatedTargetInfo{
+			{
+				Arn:                   "arn1",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Completed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			},
+		},
+		expectedCompletedSize:             249,
+		expectedReplicationStatusInternal: "arn1=COMPLETED;",
+		expectedReplicationStatus:         replication.Completed,
+		expectedOpType:                    replication.ObjectReplicationType,
+		expectedAction:                    replicateAll,
+	},
+	{ //3. replication completed to single target; failed to another
+		name: "replication completed to single target",
+		tgtInfos: []replicatedTargetInfo{
+			{
+				Arn:                   "arn1",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Completed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			},
+			{
+				Arn:                   "arn2",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Failed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			}},
+		expectedCompletedSize:             249,
+		expectedReplicationStatusInternal: "arn1=COMPLETED;arn2=FAILED;",
+		expectedReplicationStatus:         replication.Failed,
+		expectedOpType:                    replication.ObjectReplicationType,
+		expectedAction:                    replicateAll,
+	},
+	{ //4. replication pending on one target; failed to another
+		name: "replication completed to single target",
+		tgtInfos: []replicatedTargetInfo{
+			{
+				Arn:                   "arn1",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Pending,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			},
+			{
+				Arn:                   "arn2",
+				Size:                  249,
+				PrevReplicationStatus: replication.Pending,
+				ReplicationStatus:     replication.Failed,
+				OpType:                replication.ObjectReplicationType,
+				ReplicationAction:     replicateAll,
+			}},
+		expectedCompletedSize:             0,
+		expectedReplicationStatusInternal: "arn1=PENDING;arn2=FAILED;",
+		expectedReplicationStatus:         replication.Failed,
+		expectedOpType:                    replication.ObjectReplicationType,
+		expectedAction:                    replicateAll,
+	},
+}
+
+func TestReplicatedInfos(t *testing.T) {
+	for i, test := range replicatedInfosTests {
+		rinfos := replicatedInfos{
+			Targets: test.tgtInfos,
+		}
+		if actualSize := rinfos.CompletedSize(); actualSize != test.expectedCompletedSize {
+			t.Errorf("Test%d (%s): Size  got %d , want %d", i+1, test.name, actualSize, test.expectedCompletedSize)
+		}
+		if repStatusStr := rinfos.ReplicationStatusInternal(); repStatusStr != test.expectedReplicationStatusInternal {
+			t.Errorf("Test%d (%s): Internal replication status  got %s , want %s", i+1, test.name, repStatusStr, test.expectedReplicationStatusInternal)
+		}
+		if repStatus := rinfos.ReplicationStatus(); repStatus != test.expectedReplicationStatus {
+			t.Errorf("Test%d (%s): ReplicationStatus  got %s , want %s", i+1, test.name, repStatus, test.expectedReplicationStatus)
+		}
+		if action := rinfos.Action(); action != test.expectedAction {
+			t.Errorf("Test%d (%s): Action  got %s , want %s", i+1, test.name, action, test.expectedAction)
+		}
+	}
+}
+
+var parseReplicationDecisionTest = []struct {
+	name   string
+	dsc    string
+	expDsc ReplicateDecision
+	expErr error
+}{
+	{ //1.
+		name: "empty string",
+		dsc:  "",
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{},
+		},
+		expErr: nil,
+	},
+
+	{ //2.
+		name:   "replicate decision for one target",
+		dsc:    "arn:minio:replication::id:bucket=true;false;arn:minio:replication::id:bucket;id",
+		expErr: nil,
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{
+				"arn:minio:replication::id:bucket": newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false),
+			},
+		},
+	},
+	{ //3.
+		name:   "replicate decision for multiple targets",
+		dsc:    "arn:minio:replication::id:bucket=true;false;arn:minio:replication::id:bucket;id,arn:minio:replication::id2:bucket=false;true;arn:minio:replication::id2:bucket;id2",
+		expErr: nil,
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{
+				"arn:minio:replication::id:bucket":  newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false),
+				"arn:minio:replication::id2:bucket": newReplicateTargetDecision("arn:minio:replication::id2:bucket", false, true),
+			},
+		},
+	},
+	{ //4.
+		name:   "invalid format replicate decision for one target",
+		dsc:    "arn:minio:replication::id:bucket:true;false;arn:minio:replication::id:bucket;id",
+		expErr: errInvalidReplicateDecisionFormat,
+		expDsc: ReplicateDecision{
+			targetsMap: map[string]replicateTargetDecision{
+				"arn:minio:replication::id:bucket": newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false),
+			},
+		},
+	},
+}
+
+func TestParseReplicateDecision(t *testing.T) {
+	for i, test := range parseReplicationDecisionTest {
+		//dsc, err := parseReplicateDecision(test.dsc)
+		dsc, err := parseReplicateDecision(test.expDsc.String())
+
+		if err != nil {
+			if test.expErr != err {
+				t.Errorf("Test%d (%s): Expected parse error got %t , want %t", i+1, test.name, err, test.expErr)
+			}
+			continue
+		}
+		if len(dsc.targetsMap) != len(test.expDsc.targetsMap) {
+			t.Errorf("Test%d (%s): Invalid number of entries in targetsMap  got %d , want %d", i+1, test.name, len(dsc.targetsMap), len(test.expDsc.targetsMap))
+		}
+		for arn, tdsc := range dsc.targetsMap {
+			expDsc, ok := test.expDsc.targetsMap[arn]
+			if !ok || expDsc != tdsc {
+				t.Errorf("Test%d (%s): Invalid  target replicate decision: got %+v, want %+v", i+1, test.name, tdsc, expDsc)
+			}
+		}
+	}
+}
+
+var replicationStateTest = []struct {
+	name      string
+	rs        ReplicationState
+	arn       string
+	expStatus replication.StatusType
+}{
+	{ //1. no replication status header
+		name:      "no replicated targets",
+		rs:        ReplicationState{},
+		expStatus: replication.StatusType(""),
+	},
+	{ //2. replication status for one target
+		name:      "replication status for one target",
+		rs:        ReplicationState{ReplicationStatusInternal: "arn1=PENDING;", Targets: map[string]replication.StatusType{"arn1": "PENDING"}},
+		expStatus: replication.Pending,
+	},
+	{ //3. replication status for one target - incorrect format
+		name:      "replication status for one target",
+		rs:        ReplicationState{ReplicationStatusInternal: "arn1=PENDING"},
+		expStatus: replication.StatusType(""),
+	},
+	{ //4. replication status for 3 targets, one of them failed
+		name: "replication status for 3 targets - one failed",
+		rs: ReplicationState{
+			ReplicationStatusInternal: "arn1=COMPLETED;arn2=COMPLETED;arn3=FAILED;",
+			Targets:                   map[string]replication.StatusType{"arn1": "COMPLETED", "arn2": "COMPLETED", "arn3": "FAILED"},
+		},
+		expStatus: replication.Failed,
+	},
+	{ //5. replication status for replica version
+		name:      "replication status for replica version",
+		rs:        ReplicationState{ReplicationStatusInternal: string(replication.Replica)},
+		expStatus: replication.Replica,
+	},
+}
+
+func TestCompositeReplicationStatus(t *testing.T) {
+	for i, test := range replicationStateTest {
+		if rstatus := test.rs.CompositeReplicationStatus(); rstatus != test.expStatus {
+			t.Errorf("Test%d (%s): Overall replication status  got %s , want %s", i+1, test.name, rstatus, test.expStatus)
+		}
+	}
+}

cmd/bucket-replication_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
@@ -50,6 +51,8 @@ var replicationConfigTests = []struct {
 	info         ObjectInfo
 	name         string
 	rcfg         replicationConfig
+	dsc          ReplicateDecision
+	tgtStatuses  map[string]replication.StatusType
 	expectedSync bool
 }{
 	{ //1. no replication config
@@ -84,8 +87,8 @@ var replicationConfigTests = []struct {
 func TestReplicationResync(t *testing.T) {
 	ctx := context.Background()
 	for i, test := range replicationConfigTests {
-		if sync := test.rcfg.Resync(ctx, test.info); sync != test.expectedSync {
-			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync, test.expectedSync)
+		if sync := test.rcfg.Resync(ctx, test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
 }
@@ -94,27 +97,37 @@ var start = UTCNow().AddDate(0, 0, -1)
 var replicationConfigTests2 = []struct {
 	info         ObjectInfo
 	name         string
-	replicate    bool
 	rcfg         replicationConfig
+	dsc          ReplicateDecision
+	tgtStatuses  map[string]replication.StatusType
 	expectedSync bool
 }{
 	{ // Cases 1-4: existing object replication enabled, versioning enabled, no reset - replication status varies
 		// 1: Pending replication
 		name: "existing object replication on object in Pending replication status",
 		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.Pending,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			ReplicationStatusInternal: "arn1:PENDING;",
+			ReplicationStatus:         replication.Pending,
+			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
 		expectedSync: true,
 	},
+
 	{ // 2. replication status Failed
 		name: "existing object replication on object in Failed replication status",
 		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.Failed,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			ReplicationStatusInternal: "arn1:FAILED",
+			ReplicationStatus:         replication.Failed,
+			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
 		expectedSync: true,
 	},
 	{ //3. replication status unset
@@ -123,87 +136,136 @@ var replicationConfigTests2 = []struct {
 			ReplicationStatus: replication.StatusType(""),
 			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
 		expectedSync: true,
 	},
 	{ //4. replication status Complete
 		name: "existing object replication on object in Completed replication status",
 		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.Completed,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			ReplicationStatusInternal: "arn1:COMPLETED",
+			ReplicationStatus:         replication.Completed,
+			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
 		},
-		replicate:    true,
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", false, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn: "arn1",
+		}}}},
 		expectedSync: false,
 	},
 	{ //5. existing object replication enabled, versioning enabled, replication status Pending & reset ID present
 		name: "existing object replication with reset in progress and object in Pending status",
 		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.Pending,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			ReplicationStatusInternal: "arn1:PENDING;",
+			ReplicationStatus:         replication.Pending,
+			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
 	},
 	{ //6. existing object replication enabled, versioning enabled, replication status Failed & reset ID present
 		name: "existing object replication with reset in progress and object in Failed status",
 		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.Failed,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			ReplicationStatusInternal: "arn1:FAILED;",
+			ReplicationStatus:         replication.Failed,
+			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
+		},
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
 	},
 	{ //7. existing object replication enabled, versioning enabled, replication status unset & reset ID present
 		name: "existing object replication with reset in progress and object never replicated before",
 		info: ObjectInfo{Size: 100,
 			ReplicationStatus: replication.StatusType(""),
 			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
 		},
-		replicate:    true,
+		dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
+
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
 	},
+
 	{ //8. existing object replication enabled, versioning enabled, replication status Complete & reset ID present
 		name: "existing object replication enabled - reset in progress for an object in Completed status",
 		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.Completed,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df8",
+			ReplicationStatusInternal: "arn1:COMPLETED;",
+			ReplicationStatus:         replication.Completed,
+			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df8",
+			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())},
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
 	},
 	{ //9. existing object replication enabled, versioning enabled, replication status Pending & reset ID different
 		name: "existing object replication enabled, newer reset in progress on object in Pending replication status",
 		info: ObjectInfo{Size: 100,
+			ReplicationStatusInternal: "arn1:PENDING;",
+
 			ReplicationStatus: replication.Pending,
 			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), "xyz")},
-			ModTime:           UTCNow().AddDate(0, 0, -1),
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().AddDate(0, 0, -1).Format(http.TimeFormat), "abc")},
+			ModTime:           UTCNow().AddDate(0, 0, -2),
 		},
-		replicate:    true,
 		expectedSync: true,
-		rcfg:         replicationConfig{ResetID: "abc", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: UTCNow(),
+		}}},
+		},
 	},
 	{ //10. existing object replication enabled, versioning enabled, replication status Complete & reset done
 		name: "reset done on object in Completed Status - ineligbile for re-replication",
 		info: ObjectInfo{Size: 100,
-			ReplicationStatus: replication.Completed,
-			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
-			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")},
+			ReplicationStatusInternal: "arn1:COMPLETED;",
+			ReplicationStatus:         replication.Completed,
+			VersionID:                 "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:               map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")},
 		},
-		replicate:    true,
 		expectedSync: false,
-		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+		dsc:          ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}},
+		rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{
+			Arn:             "arn1",
+			ResetID:         "xyz",
+			ResetBeforeDate: start,
+		}}},
+		},
 	},
 }
 
 func TestReplicationResyncwrapper(t *testing.T) {
 	for i, test := range replicationConfigTests2 {
-		if sync := test.rcfg.resync(test.info, test.replicate); sync != test.expectedSync {
-			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync, test.expectedSync)
+		if sync := test.rcfg.resync(test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
 }

cmd/bucket-stats.go

@@ -17,6 +17,10 @@
 
 package cmd
 
+import (
+	"sync/atomic"
+)
+
 //go:generate msgp -file $GOFILE
 
 // BucketStats bucket statistics
@@ -27,16 +31,75 @@ type BucketStats struct {
 // BucketReplicationStats represents inline replication statistics
 // such as pending, failed and completed bytes in total for a bucket
 type BucketReplicationStats struct {
+	Stats map[string]*BucketReplicationStat
 	// Pending size in bytes
-	PendingSize uint64 `json:"pendingReplicationSize"`
+	PendingSize int64 `json:"pendingReplicationSize"`
 	// Completed size in bytes
-	ReplicatedSize uint64 `json:"completedReplicationSize"`
+	ReplicatedSize int64 `json:"completedReplicationSize"`
 	// Total Replica size in bytes
-	ReplicaSize uint64 `json:"replicaSize"`
+	ReplicaSize int64 `json:"replicaSize"`
 	// Failed size in bytes
-	FailedSize uint64 `json:"failedReplicationSize"`
+	FailedSize int64 `json:"failedReplicationSize"`
 	// Total number of pending operations including metadata updates
-	PendingCount uint64 `json:"pendingReplicationCount"`
+	PendingCount int64 `json:"pendingReplicationCount"`
 	// Total number of failed operations including metadata updates
-	FailedCount uint64 `json:"failedReplicationCount"`
+	FailedCount int64 `json:"failedReplicationCount"`
+}
+
+// Empty returns true if there are no target stats
+func (brs *BucketReplicationStats) Empty() bool {
+	return len(brs.Stats) == 0 && brs.ReplicaSize == 0
+}
+
+// Clone creates a new BucketReplicationStats copy
+func (brs BucketReplicationStats) Clone() BucketReplicationStats {
+	c := BucketReplicationStats{
+		Stats: make(map[string]*BucketReplicationStat, len(brs.Stats)),
+	}
+	//this is called only by replicationStats cache and already holds a read lock before calling Clone()
+	for arn, st := range brs.Stats {
+		c.Stats[arn] = &BucketReplicationStat{
+			FailedSize:     atomic.LoadInt64(&st.FailedSize),
+			ReplicatedSize: atomic.LoadInt64(&st.ReplicatedSize),
+			ReplicaSize:    atomic.LoadInt64(&st.ReplicaSize),
+			FailedCount:    atomic.LoadInt64(&st.FailedCount),
+			PendingSize:    atomic.LoadInt64(&st.PendingSize),
+			PendingCount:   atomic.LoadInt64(&st.PendingCount),
+		}
+	}
+	// update total counts across targets
+	c.FailedSize = atomic.LoadInt64(&brs.FailedSize)
+	c.FailedCount = atomic.LoadInt64(&brs.FailedCount)
+	c.PendingCount = atomic.LoadInt64(&brs.PendingCount)
+	c.PendingSize = atomic.LoadInt64(&brs.PendingSize)
+	c.ReplicaSize = atomic.LoadInt64(&brs.ReplicaSize)
+	c.ReplicatedSize = atomic.LoadInt64(&brs.ReplicatedSize)
+	return c
+}
+
+// BucketReplicationStat represents inline replication statistics
+// such as pending, failed and completed bytes in total for a bucket
+// remote target
+type BucketReplicationStat struct {
+	// Pending size in bytes
+	PendingSize int64 `json:"pendingReplicationSize"`
+	// Completed size in bytes
+	ReplicatedSize int64 `json:"completedReplicationSize"`
+	// Total Replica size in bytes
+	ReplicaSize int64 `json:"replicaSize"`
+	// Failed size in bytes
+	FailedSize int64 `json:"failedReplicationSize"`
+	// Total number of pending operations including metadata updates
+	PendingCount int64 `json:"pendingReplicationCount"`
+	// Total number of failed operations including metadata updates
+	FailedCount int64 `json:"failedReplicationCount"`
+}
+
+func (bs *BucketReplicationStat) hasReplicationUsage() bool {
+	return bs.FailedSize > 0 ||
+		bs.ReplicatedSize > 0 ||
+		bs.ReplicaSize > 0 ||
+		bs.FailedCount > 0 ||
+		bs.PendingCount > 0 ||
+		bs.PendingSize > 0
 }

cmd/bucket-stats_gen.go

@@ -6,6 +6,234 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+// DecodeMsg implements msgp.Decodable
+func (z *BucketReplicationStat) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "PendingSize":
+			z.PendingSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "PendingSize")
+				return
+			}
+		case "ReplicatedSize":
+			z.ReplicatedSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicatedSize")
+				return
+			}
+		case "ReplicaSize":
+			z.ReplicaSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicaSize")
+				return
+			}
+		case "FailedSize":
+			z.FailedSize, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "FailedSize")
+				return
+			}
+		case "PendingCount":
+			z.PendingCount, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "PendingCount")
+				return
+			}
+		case "FailedCount":
+			z.FailedCount, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "FailedCount")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BucketReplicationStat) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 6
+	// write "PendingSize"
+	err = en.Append(0x86, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.PendingSize)
+	if err != nil {
+		err = msgp.WrapError(err, "PendingSize")
+		return
+	}
+	// write "ReplicatedSize"
+	err = en.Append(0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.ReplicatedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedSize")
+		return
+	}
+	// write "ReplicaSize"
+	err = en.Append(0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.ReplicaSize)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaSize")
+		return
+	}
+	// write "FailedSize"
+	err = en.Append(0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.FailedSize)
+	if err != nil {
+		err = msgp.WrapError(err, "FailedSize")
+		return
+	}
+	// write "PendingCount"
+	err = en.Append(0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.PendingCount)
+	if err != nil {
+		err = msgp.WrapError(err, "PendingCount")
+		return
+	}
+	// write "FailedCount"
+	err = en.Append(0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.FailedCount)
+	if err != nil {
+		err = msgp.WrapError(err, "FailedCount")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BucketReplicationStat) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 6
+	// string "PendingSize"
+	o = append(o, 0x86, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.PendingSize)
+	// string "ReplicatedSize"
+	o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.ReplicatedSize)
+	// string "ReplicaSize"
+	o = append(o, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.ReplicaSize)
+	// string "FailedSize"
+	o = append(o, 0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.FailedSize)
+	// string "PendingCount"
+	o = append(o, 0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	o = msgp.AppendInt64(o, z.PendingCount)
+	// string "FailedCount"
+	o = append(o, 0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74)
+	o = msgp.AppendInt64(o, z.FailedCount)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketReplicationStat) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "PendingSize":
+			z.PendingSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PendingSize")
+				return
+			}
+		case "ReplicatedSize":
+			z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicatedSize")
+				return
+			}
+		case "ReplicaSize":
+			z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicaSize")
+				return
+			}
+		case "FailedSize":
+			z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "FailedSize")
+				return
+			}
+		case "PendingCount":
+			z.PendingCount, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PendingCount")
+				return
+			}
+		case "FailedCount":
+			z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "FailedCount")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BucketReplicationStat) Msgsize() (s int) {
+	s = 1 + 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *BucketReplicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -24,38 +252,80 @@ func (z *BucketReplicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 			return
 		}
 		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Stats")
+				return
+			}
+			if z.Stats == nil {
+				z.Stats = make(map[string]*BucketReplicationStat, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 *BucketReplicationStat
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				if dc.IsNil() {
+					err = dc.ReadNil()
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+					za0002 = nil
+				} else {
+					if za0002 == nil {
+						za0002 = new(BucketReplicationStat)
+					}
+					err = za0002.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
 		case "PendingSize":
-			z.PendingSize, err = dc.ReadUint64()
+			z.PendingSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "PendingSize")
 				return
 			}
 		case "ReplicatedSize":
-			z.ReplicatedSize, err = dc.ReadUint64()
+			z.ReplicatedSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicatedSize")
 				return
 			}
 		case "ReplicaSize":
-			z.ReplicaSize, err = dc.ReadUint64()
+			z.ReplicaSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicaSize")
 				return
 			}
 		case "FailedSize":
-			z.FailedSize, err = dc.ReadUint64()
+			z.FailedSize, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "FailedSize")
 				return
 			}
 		case "PendingCount":
-			z.PendingCount, err = dc.ReadUint64()
+			z.PendingCount, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "PendingCount")
 				return
 			}
 		case "FailedCount":
-			z.FailedCount, err = dc.ReadUint64()
+			z.FailedCount, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "FailedCount")
 				return
@@ -73,13 +343,42 @@ func (z *BucketReplicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
-	// write "PendingSize"
-	err = en.Append(0x86, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	// map header, size 7
+	// write "Stats"
+	err = en.Append(0x87, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.PendingSize)
+	err = en.WriteMapHeader(uint32(len(z.Stats)))
+	if err != nil {
+		err = msgp.WrapError(err, "Stats")
+		return
+	}
+	for za0001, za0002 := range z.Stats {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Stats")
+			return
+		}
+		if za0002 == nil {
+			err = en.WriteNil()
+			if err != nil {
+				return
+			}
+		} else {
+			err = za0002.EncodeMsg(en)
+			if err != nil {
+				err = msgp.WrapError(err, "Stats", za0001)
+				return
+			}
+		}
+	}
+	// write "PendingSize"
+	err = en.Append(0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.PendingSize)
 	if err != nil {
 		err = msgp.WrapError(err, "PendingSize")
 		return
@@ -89,7 +388,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.ReplicatedSize)
+	err = en.WriteInt64(z.ReplicatedSize)
 	if err != nil {
 		err = msgp.WrapError(err, "ReplicatedSize")
 		return
@@ -99,7 +398,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.ReplicaSize)
+	err = en.WriteInt64(z.ReplicaSize)
 	if err != nil {
 		err = msgp.WrapError(err, "ReplicaSize")
 		return
@@ -109,7 +408,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.FailedSize)
+	err = en.WriteInt64(z.FailedSize)
 	if err != nil {
 		err = msgp.WrapError(err, "FailedSize")
 		return
@@ -119,7 +418,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.PendingCount)
+	err = en.WriteInt64(z.PendingCount)
 	if err != nil {
 		err = msgp.WrapError(err, "PendingCount")
 		return
@@ -129,7 +428,7 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint64(z.FailedCount)
+	err = en.WriteInt64(z.FailedCount)
 	if err != nil {
 		err = msgp.WrapError(err, "FailedCount")
 		return
@@ -140,25 +439,40 @@ func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketReplicationStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 7
+	// string "Stats"
+	o = append(o, 0x87, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Stats)))
+	for za0001, za0002 := range z.Stats {
+		o = msgp.AppendString(o, za0001)
+		if za0002 == nil {
+			o = msgp.AppendNil(o)
+		} else {
+			o, err = za0002.MarshalMsg(o)
+			if err != nil {
+				err = msgp.WrapError(err, "Stats", za0001)
+				return
+			}
+		}
+	}
 	// string "PendingSize"
-	o = append(o, 0x86, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.PendingSize)
+	o = append(o, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	o = msgp.AppendInt64(o, z.PendingSize)
 	// string "ReplicatedSize"
 	o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.ReplicatedSize)
+	o = msgp.AppendInt64(o, z.ReplicatedSize)
 	// string "ReplicaSize"
 	o = append(o, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.ReplicaSize)
+	o = msgp.AppendInt64(o, z.ReplicaSize)
 	// string "FailedSize"
 	o = append(o, 0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendUint64(o, z.FailedSize)
+	o = msgp.AppendInt64(o, z.FailedSize)
 	// string "PendingCount"
 	o = append(o, 0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74)
-	o = msgp.AppendUint64(o, z.PendingCount)
+	o = msgp.AppendInt64(o, z.PendingCount)
 	// string "FailedCount"
 	o = append(o, 0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74)
-	o = msgp.AppendUint64(o, z.FailedCount)
+	o = msgp.AppendInt64(o, z.FailedCount)
 	return
 }
 
@@ -180,38 +494,79 @@ func (z *BucketReplicationStats) UnmarshalMsg(bts []byte) (o []byte, err error)
 			return
 		}
 		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Stats")
+				return
+			}
+			if z.Stats == nil {
+				z.Stats = make(map[string]*BucketReplicationStat, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 *BucketReplicationStat
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				if msgp.IsNil(bts) {
+					bts, err = msgp.ReadNilBytes(bts)
+					if err != nil {
+						return
+					}
+					za0002 = nil
+				} else {
+					if za0002 == nil {
+						za0002 = new(BucketReplicationStat)
+					}
+					bts, err = za0002.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
 		case "PendingSize":
-			z.PendingSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.PendingSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "PendingSize")
 				return
 			}
 		case "ReplicatedSize":
-			z.ReplicatedSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicatedSize")
 				return
 			}
 		case "ReplicaSize":
-			z.ReplicaSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "ReplicaSize")
 				return
 			}
 		case "FailedSize":
-			z.FailedSize, bts, err = msgp.ReadUint64Bytes(bts)
+			z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "FailedSize")
 				return
 			}
 		case "PendingCount":
-			z.PendingCount, bts, err = msgp.ReadUint64Bytes(bts)
+			z.PendingCount, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "PendingCount")
 				return
 			}
 		case "FailedCount":
-			z.FailedCount, bts, err = msgp.ReadUint64Bytes(bts)
+			z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "FailedCount")
 				return
@@ -230,7 +585,19 @@ func (z *BucketReplicationStats) UnmarshalMsg(bts []byte) (o []byte, err error)
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketReplicationStats) Msgsize() (s int) {
-	s = 1 + 12 + msgp.Uint64Size + 15 + msgp.Uint64Size + 12 + msgp.Uint64Size + 11 + msgp.Uint64Size + 13 + msgp.Uint64Size + 12 + msgp.Uint64Size
+	s = 1 + 6 + msgp.MapHeaderSize
+	if z.Stats != nil {
+		for za0001, za0002 := range z.Stats {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001)
+			if za0002 == nil {
+				s += msgp.NilSize
+			} else {
+				s += za0002.Msgsize()
+			}
+		}
+	}
+	s += 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size
 	return
 }
 

cmd/bucket-stats_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshalBucketReplicationStat(t *testing.T) {
+	v := BucketReplicationStat{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBucketReplicationStat(t *testing.T) {
+	v := BucketReplicationStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBucketReplicationStat Msgsize() is inaccurate")
+	}
+
+	vn := BucketReplicationStat{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBucketReplicationStat(b *testing.B) {
+	v := BucketReplicationStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalBucketReplicationStats(t *testing.T) {
 	v := BucketReplicationStats{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-targets.go

@@ -19,8 +19,6 @@ package cmd
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"net/http"
 	"sync"
 	"time"
@@ -30,6 +28,7 @@ import (
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/kms"
@@ -37,7 +36,7 @@ import (
 )
 
 const (
-	defaultHealthCheckDuration = 100 * time.Second
+	defaultHealthCheckDuration = 30 * time.Second
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -93,6 +92,9 @@ func (sys *BucketTargetSys) Delete(bucket string) {
 		return
 	}
 	for _, t := range tgts {
+		if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
+			tgt.healthCancelFn()
+		}
 		delete(sys.arnRemotesMap, t.Arn)
 	}
 	delete(sys.targetsMap, bucket)
@@ -136,24 +138,28 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	defer sys.Unlock()
 
 	tgts := sys.targetsMap[bucket]
-
 	newtgts := make([]madmin.BucketTarget, len(tgts))
 	found := false
 	for idx, t := range tgts {
 		if t.Type == tgt.Type {
-			if t.Arn == tgt.Arn && !update {
-				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
+			if t.Arn == tgt.Arn {
+				if !update {
+					return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
+				}
+				newtgts[idx] = *tgt
+				found = true
+				continue
 			}
-			newtgts[idx] = *tgt
-			found = true
-			continue
 		}
 		newtgts[idx] = t
 	}
 	if !found && !update {
 		newtgts = append(newtgts, *tgt)
 	}
-
+	// cancel health check for previous target client to avoid leak.
+	if prevClnt, ok := sys.arnRemotesMap[tgt.Arn]; ok && prevClnt.healthCancelFn != nil {
+		prevClnt.healthCancelFn()
+	}
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
 	sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
@@ -194,12 +200,16 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	if arn.Type == madmin.ReplicationService {
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
-		if err == nil && rcfg.RoleArn == arnStr {
-			sys.RLock()
-			_, ok := sys.arnRemotesMap[arnStr]
-			sys.RUnlock()
-			if ok {
-				return BucketRemoteRemoveDisallowed{Bucket: bucket}
+		if err == nil {
+			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{}) {
+				if err == nil && (tgtArn == arnStr || rcfg.RoleArn == arnStr) {
+					sys.RLock()
+					_, ok := sys.arnRemotesMap[arnStr]
+					sys.RUnlock()
+					if ok {
+						return BucketRemoteRemoveDisallowed{Bucket: bucket}
+					}
+				}
 			}
 		}
 	}
@@ -224,6 +234,9 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		return BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	sys.targetsMap[bucket] = targets
+	if tgt, ok := sys.arnRemotesMap[arnStr]; ok && tgt.healthCancelFn != nil {
+		tgt.healthCancelFn()
+	}
 	delete(sys.arnRemotesMap, arnStr)
 	sys.updateBandwidthLimit(bucket, 0)
 	return nil
@@ -259,22 +272,6 @@ func NewBucketTargetSys() *BucketTargetSys {
 	}
 }
 
-// Init initializes the bucket targets subsystem for buckets which have targets configured.
-func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// In gateway mode, bucket targets is not supported.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load bucket targets once during boot in background.
-	go sys.load(ctx, buckets, objAPI)
-	return nil
-}
-
 // UpdateAllTargets updates target to reflect metadata updates
 func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
 	if sys == nil {
@@ -286,6 +283,9 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 		// remove target and arn association
 		if tgts, ok := sys.targetsMap[bucket]; ok {
 			for _, t := range tgts {
+				if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
+					tgt.healthCancelFn()
+				}
 				delete(sys.arnRemotesMap, t.Arn)
 			}
 		}
@@ -309,30 +309,26 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 }
 
 // create minio-go clients for buckets having remote targets
-func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
-	for _, bucket := range buckets {
-		cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket.Name)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-		if cfg == nil || cfg.Empty() {
-			continue
-		}
-		if len(cfg.Targets) > 0 {
-			sys.targetsMap[bucket.Name] = cfg.Targets
-		}
-		for _, tgt := range cfg.Targets {
-			tgtClient, err := sys.getRemoteTargetClient(&tgt)
-			if err != nil {
-				logger.LogIf(ctx, err)
-				continue
-			}
-			sys.arnRemotesMap[tgt.Arn] = tgtClient
-			sys.updateBandwidthLimit(bucket.Name, tgt.BandwidthLimit)
-		}
+func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
+	cfg := meta.bucketTargetConfig
+	if cfg == nil || cfg.Empty() {
+		return
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	if len(cfg.Targets) > 0 {
 		sys.targetsMap[bucket.Name] = cfg.Targets
 	}
+	for _, tgt := range cfg.Targets {
+		tgtClient, err := sys.getRemoteTargetClient(&tgt)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			continue
+		}
+		sys.arnRemotesMap[tgt.Arn] = tgtClient
+		sys.updateBandwidthLimit(bucket.Name, tgt.BandwidthLimit)
+	}
+	sys.targetsMap[bucket.Name] = cfg.Targets
 }
 
 // getRemoteTargetInstanceTransport contains a singleton roundtripper.
@@ -360,6 +356,10 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
 	}
+	cancelFn, err := api.HealthCheck(hcDuration)
+	if err != nil {
+		return nil, err
+	}
 	tc := &TargetClient{
 		Client:              api,
 		healthCheckDuration: hcDuration,
@@ -367,6 +367,9 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 		Bucket:              tcfg.TargetBucket,
 		StorageClass:        tcfg.StorageClass,
 		disableProxy:        tcfg.DisableProxy,
+		healthCancelFn:      cancelFn,
+		ARN:                 tcfg.Arn,
+		ResetID:             tcfg.ResetID,
 	}
 	return tc, nil
 }
@@ -390,14 +393,9 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 
 // generate ARN that is unique to this target type
 func generateARN(t *madmin.BucketTarget) string {
-	hash := sha256.New()
-	hash.Write([]byte(t.Type))
-	hash.Write([]byte(t.Region))
-	hash.Write([]byte(t.TargetBucket))
-	hashSum := hex.EncodeToString(hash.Sum(nil))
 	arn := madmin.ARN{
 		Type:   t.Type,
-		ID:     hashSum,
+		ID:     mustGetUUID(),
 		Region: t.Region,
 		Bucket: t.TargetBucket,
 	}
@@ -445,4 +443,7 @@ type TargetClient struct {
 	replicateSync       bool
 	StorageClass        string // storage class on remote
 	disableProxy        bool
+	healthCancelFn      context.CancelFunc // cancellation function for client healthcheck
+	ARN                 string             //ARN to uniquely identify remote target
+	ResetID             string
 }

cmd/common-main.go

@@ -51,7 +51,6 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/handlers"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -59,6 +58,7 @@ import (
 	"github.com/minio/pkg/ellipses"
 	"github.com/minio/pkg/env"
 	xnet "github.com/minio/pkg/net"
+	"github.com/rs/dnscache"
 )
 
 // serverDebugLog will enable debug printing
@@ -71,17 +71,34 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
-		// 30 seconds matches the orchestrator DNS TTLs, have
-		// a 5 second timeout to lookup from DNS servers.
-		globalDNSCache = xhttp.NewDNSCache(30*time.Second, 5*time.Second, logger.LogOnceIf)
-	} else {
-		// On bare-metals DNS do not change often, so it is
-		// safe to assume a higher timeout upto 10 minutes.
-		globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf)
-	}
 	initGlobalContext()
 
+	options := dnscache.ResolverRefreshOptions{
+		ClearUnused:      true,
+		PersistOnFailure: false,
+	}
+
+	// Call to refresh will refresh names in cache. If you pass true, it will also
+	// remove cached names not looked up since the last call to Refresh. It is a good idea
+	// to call this method on a regular interval.
+	go func() {
+		var t *time.Ticker
+		if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
+			t = time.NewTicker(1 * time.Minute)
+		} else {
+			t = time.NewTicker(10 * time.Minute)
+		}
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				globalDNSCache.RefreshWithOptions(options)
+			case <-GlobalContext.Done():
+				return
+			}
+		}
+	}()
+
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
 		RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
@@ -138,12 +155,15 @@ func minioConfigToConsoleFeatures() {
 	}
 	// if IDP is enabled, set IDP environment variables
 	if globalOpenIDConfig.URL != nil {
-		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.DiscoveryDoc.Issuer)
+		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.URL.String())
 		os.Setenv("CONSOLE_IDP_CLIENT_ID", globalOpenIDConfig.ClientID)
 		os.Setenv("CONSOLE_IDP_SECRET", globalOpenIDConfig.ClientSecret)
 		os.Setenv("CONSOLE_IDP_HMAC_SALT", globalDeploymentID)
 		os.Setenv("CONSOLE_IDP_HMAC_PASSPHRASE", globalOpenIDConfig.ClientID)
 		os.Setenv("CONSOLE_IDP_SCOPES", strings.Join(globalOpenIDConfig.DiscoveryDoc.ScopesSupported, ","))
+		if globalOpenIDConfig.ClaimUserinfo {
+			os.Setenv("CONSOLE_IDP_USERINFO", "on")
+		}
 		if globalOpenIDConfig.RedirectURI != "" {
 			os.Setenv("CONSOLE_IDP_CALLBACK", globalOpenIDConfig.RedirectURI)
 		} else {
@@ -152,8 +172,8 @@ func minioConfigToConsoleFeatures() {
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalServerRegion)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
-	if globalSubnetLicense != "" {
-		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetLicense)
+	if globalSubnetConfig.License != "" {
+		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
 	}
 }
 
@@ -179,16 +199,16 @@ func initConsoleServer() (*restapi.Server, error) {
 		return nil, err
 	}
 
-	noLog := func(string, ...interface{}) {
-		// nothing to log
-	}
-
-	// Initialize MinIO loggers
-	restapi.LogInfo = noLog
-	restapi.LogError = noLog
-
 	api := operations.NewConsoleAPI(swaggerSpec)
-	api.Logger = noLog
+
+	if !serverDebugLog {
+		// Disable console logging if server debug log is not enabled
+		noLog := func(string, ...interface{}) {}
+
+		restapi.LogInfo = noLog
+		restapi.LogError = noLog
+		api.Logger = noLog
+	}
 
 	server := restapi.NewServer(api)
 	// register all APIs
@@ -595,12 +615,6 @@ func handleCommonEnvVars() {
 		}
 		GlobalKMS = KMS
 	}
-
-	if tiers := env.Get("_MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY", ""); tiers != "" {
-		globalDebugRemoteTiersImmediately = strings.Split(tiers, ",")
-	}
-
-	globalSubnetLicense = env.Get(config.EnvMinIOSubnetLicense, "")
 }
 
 func logStartupMessage(msg string) {

cmd/config-common.go

@@ -57,7 +57,9 @@ type objectDeleter interface {
 }
 
 func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error {
-	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{})
+	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{
+		DeletePrefix: true,
+	})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound
 	}

cmd/config-current.go

@@ -34,10 +34,12 @@ import (
 	"github.com/minio/minio/internal/config/heal"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
 	"github.com/minio/minio/internal/config/scanner"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
@@ -54,6 +56,7 @@ func initHelp() {
 		config.CompressionSubSys:    compress.DefaultKVS,
 		config.IdentityLDAPSubSys:   xldap.DefaultKVS,
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
+		config.IdentityTLSSubSys:    xtls.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
@@ -63,6 +66,7 @@ func initHelp() {
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
 		config.HealSubSys:           heal.DefaultKVS,
 		config.ScannerSubSys:        scanner.DefaultKVS,
+		config.SubnetSubSys:         subnet.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -98,6 +102,10 @@ func initHelp() {
 			Key:         config.IdentityLDAPSubSys,
 			Description: "enable LDAP SSO support",
 		},
+		config.HelpKV{
+			Key:         config.IdentityTLSSubSys,
+			Description: "enable X.509 TLS certificate SSO support",
+		},
 		config.HelpKV{
 			Key:         config.PolicyOPASubSys,
 			Description: "[DEPRECATED] enable external OPA for policy enforcement",
@@ -179,6 +187,12 @@ func initHelp() {
 			Description:     "publish bucket notifications to Redis datastores",
 			MultipleTargets: true,
 		},
+		config.HelpKV{
+			Key:         config.SubnetSubSys,
+			Type:        "string",
+			Description: "set subnet config for the cluster e.g. license token",
+			Optional:    true,
+		},
 	}
 
 	if globalIsErasure {
@@ -202,6 +216,7 @@ func initHelp() {
 		config.ScannerSubSys:        scanner.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
+		config.IdentityTLSSubSys:    xtls.Help,
 		config.PolicyOPASubSys:      opa.Help,
 		config.LoggerWebhookSubSys:  logger.Help,
 		config.AuditWebhookSubSys:   logger.HelpWebhook,
@@ -216,6 +231,7 @@ func initHelp() {
 		config.NotifyRedisSubSys:    notify.HelpRedis,
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
+		config.SubnetSubSys:         subnet.HelpLicense,
 	}
 
 	config.RegisterHelpSubSys(helpMap)
@@ -227,7 +243,9 @@ var (
 	globalServerConfigMu sync.RWMutex
 )
 
-func validateConfig(s config.Config, setDriveCounts []int) error {
+func validateConfig(s config.Config) error {
+	objAPI := newObjectLayerFn()
+
 	// We must have a global lock for this so nobody else modifies env while we do.
 	defer env.LockSetEnv()()
 
@@ -250,7 +268,10 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 	}
 
 	if globalIsErasure {
-		for _, setDriveCount := range setDriveCounts {
+		if objAPI == nil {
+			return errServerNotInitialized
+		}
+		for _, setDriveCount := range objAPI.SetDriveCounts() {
 			if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
 				return err
 			}
@@ -265,7 +286,7 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 	if err != nil {
 		return err
 	}
-	objAPI := newObjectLayerFn()
+
 	if objAPI != nil {
 		if compCfg.Enabled && !objAPI.IsCompressionSupported() {
 			return fmt.Errorf("Backend does not support compression")
@@ -312,6 +333,12 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 			conn.Close()
 		}
 	}
+	{
+		_, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
+		if err != nil {
+			return err
+		}
+	}
 
 	if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
@@ -325,7 +352,7 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 	return notify.TestNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs())
 }
 
-func lookupConfigs(s config.Config, setDriveCounts []int) {
+func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	ctx := GlobalContext
 
 	var err error
@@ -337,7 +364,15 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		}
 	}
 
-	if dnsURL, dnsUser, dnsPass, ok := env.LookupEnv(config.EnvDNSWebhook); ok {
+	dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
+	if err != nil {
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+		}
+	}
+	if err == nil && dnsURL != "" {
 		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
@@ -412,14 +447,13 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	globalAPIConfig.init(apiConfig, setDriveCounts)
-
 	// Initialize remote instance transport once.
 	getRemoteInstanceTransportOnce.Do(func() {
 		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
 	})
 
-	if globalIsErasure {
+	if globalIsErasure && objAPI != nil {
+		setDriveCounts := objAPI.SetDriveCounts()
 		for i, setDriveCount := range setDriveCounts {
 			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
 			if err != nil {
@@ -457,6 +491,15 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
 	}
 
+	globalSTSTLSConfig, err = xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
+	}
+
+	if globalSTSTLSConfig.InsecureSkipVerify {
+		logger.Info("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify)
+	}
+
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
 		NewGatewayHTTPTransport(), xhttp.DrainBody)
 	if err != nil {
@@ -478,6 +521,11 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
 	}
 
+	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
+	}
+
 	// Load logger targets based on user's configuration
 	loggerUserAgent := getUserAgent(getMinioMode())
 
@@ -531,16 +579,18 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 	}
 
 	// Apply dynamic config values
-	logger.LogIf(ctx, applyDynamicConfig(ctx, newObjectLayerFn(), s))
+	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize dynamic configuration")
+		} else {
+			logger.LogIf(ctx, err)
+		}
+	}
 }
 
 // applyDynamicConfig will apply dynamic config values.
 // Dynamic systems should be in config.SubSystemsDynamic as well.
 func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {
-	if objAPI == nil {
-		return nil
-	}
-
 	// Read all dynamic configs.
 	// API
 	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
@@ -555,8 +605,10 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config
 	}
 
 	// Validate if the object layer supports compression.
-	if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
-		return fmt.Errorf("Backend does not support compression")
+	if objAPI != nil {
+		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+			return fmt.Errorf("Backend does not support compression")
+		}
 	}
 
 	// Heal
@@ -573,15 +625,17 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config
 
 	// Apply configurations.
 	// We should not fail after this.
-	globalAPIConfig.init(apiConfig, objAPI.SetDriveCounts())
+	var setDriveCounts []int
+	if objAPI != nil {
+		setDriveCounts = objAPI.SetDriveCounts()
+	}
+	globalAPIConfig.init(apiConfig, setDriveCounts)
 
 	globalCompressConfigMu.Lock()
 	globalCompressConfig = cmpCfg
 	globalCompressConfigMu.Unlock()
 
-	globalHealConfigMu.Lock()
-	globalHealConfig = healCfg
-	globalHealConfigMu.Unlock()
+	globalHealConfig.Update(healCfg)
 
 	// update dynamic scanner values.
 	scannerCycle.Update(scannerCfg.Cycle)
@@ -706,7 +760,7 @@ func loadConfig(objAPI ObjectLayer) error {
 	}
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg, objAPI.SetDriveCounts())
+	lookupConfigs(srvCfg, objAPI)
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/config-migrate.go

@@ -2754,7 +2754,6 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 	}
 
 	xldap.SetIdentityLDAP(newCfg, cfg.LDAPServerConfig)
-	openid.SetIdentityOpenID(newCfg, cfg.OpenID)
 	opa.SetPolicyOPAConfig(newCfg, cfg.Policy.OPA)
 	cache.SetCacheConfig(newCfg, cfg.Cache)
 	compress.SetCompressionConfig(newCfg, cfg.Compression)

cmd/config.go

@@ -96,7 +96,9 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 
 func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
 	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
-	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{})
+	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{
+		DeletePrefix: true,
+	})
 	return err
 }
 
@@ -155,7 +157,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	data, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			lookupConfigs(srvCfg, objAPI.SetDriveCounts())
+			lookupConfigs(srvCfg, objAPI)
 			return srvCfg, nil
 		}
 		return nil, err
@@ -166,7 +168,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 			minioMetaBucket: path.Join(minioMetaBucket, configFile),
 		})
 		if err != nil {
-			lookupConfigs(srvCfg, objAPI.SetDriveCounts())
+			lookupConfigs(srvCfg, objAPI)
 			return nil, err
 		}
 	}

cmd/data-scanner.go

@@ -29,6 +29,7 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/bits-and-blooms/bloom/v3"
@@ -57,8 +58,7 @@ const (
 )
 
 var (
-	globalHealConfig   heal.Config
-	globalHealConfigMu sync.Mutex
+	globalHealConfig heal.Config
 
 	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 	// Sleeper values are updated when config is loaded.
@@ -143,11 +143,11 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 			}
 
 			// Wait before starting next cycle and wait on startup.
-			results := make(chan madmin.DataUsageInfo, 1)
+			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results)
+			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle))
 			logger.LogIf(ctx, err)
 			if err == nil {
 				// Store new cycle...
@@ -195,6 +195,22 @@ type folderScanner struct {
 	lastUpdate time.Time
 }
 
+type scannerStats struct {
+	// All fields must be accessed atomically and aligned.
+
+	accTotalObjects  uint64
+	accTotalVersions uint64
+	accFolders       uint64
+	bucketsStarted   uint64
+	bucketsFinished  uint64
+	ilmChecks        uint64
+
+	// actions records actions performed.
+	actions [lifecycle.ActionCount]uint64
+}
+
+var globalScannerStats scannerStats
+
 // Cache structure and compaction:
 //
 // A cache structure will be kept with a tree of usages.
@@ -242,6 +258,10 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 
 	logPrefix := color.Green("data-usage: ")
 	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
+	atomic.AddUint64(&globalScannerStats.bucketsStarted, 1)
+	defer func() {
+		atomic.AddUint64(&globalScannerStats.bucketsFinished, 1)
+	}()
 	if intDataUpdateTracker.debug {
 		defer func() {
 			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
@@ -320,7 +340,7 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		console.Debugf(logPrefix+"Finished scanner, %v entries (%+v) %s \n", len(s.newCache.Cache), *s.newCache.sizeRecursive(s.newCache.Info.Name), logSuffix)
 	}
 	s.newCache.Info.LastUpdate = UTCNow()
-	s.newCache.Info.NextCycle++
+	s.newCache.Info.NextCycle = cache.Info.NextCycle
 	return s.newCache, nil
 }
 
@@ -350,6 +370,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
+	atomic.AddUint64(&globalScannerStats.accFolders, 1)
 
 	for {
 		select {
@@ -406,11 +427,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
 			// Parse
 			entName = pathClean(path.Join(folder.name, entName))
-			if entName == "" {
+			if entName == "" || entName == folder.name {
 				if f.dataUsageScannerDebug {
-					console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
+					console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName)
 				}
-				return errDoneForNow
+				return nil
 			}
 			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
 			if bucket == "" {
@@ -436,7 +457,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			if typ&os.ModeDir != 0 {
 				h := hashPath(entName)
 				_, exists := f.oldCache.Cache[h.Key()]
-
+				if h == thisHash {
+					return nil
+				}
 				this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
 				delete(abandonedChildren, h.Key()) // h.Key() already accounted for.
 				if exists {
@@ -469,11 +492,14 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			item.heal = item.heal && f.healObjectSelect > 0
 
 			sz, err := f.getSize(item)
-			if err == errSkipFile {
+			if err != nil {
 				wait() // wait to proceed to next entry.
-
+				if err != errSkipFile && f.dataUsageScannerDebug {
+					console.Debugf(scannerLogPrefix+" getSize \"%v/%v\" returned err: %v\n", bucket, item.objectPath(), err)
+				}
 				return nil
 			}
+
 			// successfully read means we have a valid object.
 			foundObjects = true
 			// Remove filename i.e is the meta file to construct object name
@@ -638,10 +664,16 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
 			}
 
+			if bucket != resolver.bucket {
+				// Bucket might be missing as well with abandoned children.
+				// make sure it is created first otherwise healing won't proceed
+				// for objects.
+				_, _ = objAPI.HealBucket(ctx, bucket, madmin.HealOpts{})
+			}
+
 			resolver.bucket = bucket
 
 			foundObjs := false
-			dangling := false
 			ctx, cancel := context.WithCancel(ctx)
 
 			err := listPathRaw(ctx, listPathRawOptions{
@@ -663,18 +695,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
 					}
 
-					// agreed value less than expected quorum
-					dangling = nAgreed < resolver.objQuorum || nAgreed < resolver.dirQuorum
-
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
-						for _, err := range errs {
-							if err != nil {
-								return
-							}
-						}
-
-						// If no errors, queue it for healing.
+						// check if we can get one entry atleast
+						// proceed to heal nonetheless, since
+						// this object might be dangling.
 						entry, _ = entries.firstFound()
 					}
 
@@ -698,9 +723,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							object:    entry.name,
 							versionID: "",
 						}, madmin.HealItemObject)
-						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogIf(ctx, err)
-						}
+						logger.LogIf(ctx, err)
 						foundObjs = foundObjs || err == nil
 						return
 					}
@@ -715,9 +738,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							object:    fiv.Name,
 							versionID: ver.VersionID,
 						}, madmin.HealItemObject)
-						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogIf(ctx, err)
-						}
+						logger.LogIf(ctx, err)
 						foundObjs = foundObjs || err == nil
 					}
 				},
@@ -734,30 +755,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
 			}
 
-			// If we found one or more disks with this folder, delete it.
-			if err == nil && dangling {
-				if f.dataUsageScannerDebug {
-					console.Debugf(healObjectsPrefix+" deleting dangling directory %s\n", prefix)
-				}
-
-				// wait on timer per object.
-				wait := scannerSleeper.Timer(ctx)
-
-				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
-					Recursive: true,
-					Remove:    healDeleteDangling,
-				}, func(bucket, object, versionID string) error {
-					// Wait for each heal as per scanner frequency.
-					wait()
-					wait = scannerSleeper.Timer(ctx)
-					return bgSeq.queueHealTask(healSource{
-						bucket:    bucket,
-						object:    object,
-						versionID: versionID,
-					}, madmin.HealItemObject)
-				})
-			}
-
 			// Add unless healing returned an error.
 			if foundObjs {
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
@@ -832,12 +829,23 @@ type scannerItem struct {
 }
 
 type sizeSummary struct {
-	totalSize      int64
-	versions       uint64
+	totalSize       int64
+	versions        uint64
+	replicatedSize  int64
+	pendingSize     int64
+	failedSize      int64
+	replicaSize     int64
+	pendingCount    uint64
+	failedCount     uint64
+	replTargetStats map[string]replTargetSizeSummary
+	tiers           map[string]tierStats
+}
+
+// replTargetSizeSummary holds summary of replication stats by target
+type replTargetSizeSummary struct {
 	replicatedSize int64
 	pendingSize    int64
 	failedSize     int64
-	replicaSize    int64
 	pendingCount   uint64
 	failedCount    uint64
 }
@@ -856,30 +864,21 @@ func (i *scannerItem) transformMetaDir() {
 	i.objectName = split[len(split)-1]
 }
 
-// actionMeta contains information used to apply actions.
-type actionMeta struct {
-	oi         ObjectInfo
-	bitRotScan bool // indicates if bitrot check was requested.
-}
-
 var applyActionsLogPrefix = color.Green("applyActions:")
 
-func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, meta actionMeta) (size int64) {
+func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi ObjectInfo) (size int64) {
 	if i.debug {
-		if meta.oi.VersionID != "" {
-			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), meta.oi.VersionID)
+		if oi.VersionID != "" {
+			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), oi.VersionID)
 		} else {
 			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
 		}
 	}
-	healOpts := madmin.HealOpts{Remove: healDeleteDangling}
-	if meta.bitRotScan {
-		healOpts.ScanMode = madmin.HealDeepScan
-	}
-	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, healOpts)
-	if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-		return 0
+	healOpts := madmin.HealOpts{
+		Remove:   healDeleteDangling,
+		ScanMode: globalHealConfig.ScanMode(),
 	}
+	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
 	if err != nil && !errors.Is(err, NotImplemented{}) {
 		logger.LogIf(ctx, err)
 		return 0
@@ -887,8 +886,8 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, meta acti
 	return res.ObjectSize
 }
 
-func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta actionMeta) (applied bool, size int64) {
-	size, err := meta.oi.GetActualSize()
+func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (applied bool, size int64) {
+	size, err := oi.GetActualSize()
 	if i.debug {
 		logger.LogIf(ctx, err)
 	}
@@ -900,21 +899,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta ac
 		return false, size
 	}
 
-	versionID := meta.oi.VersionID
+	atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
+	versionID := oi.VersionID
 	action := i.lifeCycle.ComputeAction(
 		lifecycle.ObjectOpts{
-			Name:                   i.objectPath(),
-			UserTags:               meta.oi.UserTags,
-			ModTime:                meta.oi.ModTime,
-			VersionID:              meta.oi.VersionID,
-			DeleteMarker:           meta.oi.DeleteMarker,
-			IsLatest:               meta.oi.IsLatest,
-			NumVersions:            meta.oi.NumVersions,
-			SuccessorModTime:       meta.oi.SuccessorModTime,
-			RestoreOngoing:         meta.oi.RestoreOngoing,
-			RestoreExpires:         meta.oi.RestoreExpires,
-			TransitionStatus:       meta.oi.TransitionedObject.Status,
-			RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
+			Name:             i.objectPath(),
+			UserTags:         oi.UserTags,
+			ModTime:          oi.ModTime,
+			VersionID:        oi.VersionID,
+			DeleteMarker:     oi.DeleteMarker,
+			IsLatest:         oi.IsLatest,
+			NumVersions:      oi.NumVersions,
+			SuccessorModTime: oi.SuccessorModTime,
+			RestoreOngoing:   oi.RestoreOngoing,
+			RestoreExpires:   oi.RestoreExpires,
+			TransitionStatus: oi.TransitionedObject.Status,
 		})
 	if i.debug {
 		if versionID != "" {
@@ -923,60 +922,23 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta ac
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
 		}
 	}
+	atomic.AddUint64(&globalScannerStats.actions[action], 1)
+
 	switch action {
-	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		return applyLifecycleAction(action, oi), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		return applyLifecycleAction(action, oi), size
 	default:
 		// No action.
-		if i.debug {
-			console.Debugf(applyActionsLogPrefix+" object not expirable: %q\n", i.objectPath())
-		}
 		return false, size
 	}
-
-	obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{
-		VersionID: versionID,
-	})
-	if err != nil {
-		switch err.(type) {
-		case MethodNotAllowed: // This happens usually for a delete marker
-			if !obj.DeleteMarker { // if this is not a delete marker log and return
-				// Do nothing - heal in the future.
-				logger.LogIf(ctx, err)
-				return false, size
-			}
-		case ObjectNotFound, VersionNotFound:
-			// object not found or version not found return 0
-			return false, 0
-		default:
-			// All other errors proceed.
-			logger.LogIf(ctx, err)
-			return false, size
-		}
-	}
-
-	action = evalActionFromLifecycle(ctx, *i.lifeCycle, obj, i.debug)
-	if action != lifecycle.NoneAction {
-		applied = applyLifecycleAction(ctx, action, o, obj)
-	}
-
-	if applied {
-		switch action {
-		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			return true, size
-		}
-		// For all other lifecycle actions that remove data
-		return true, 0
-	}
-
-	return false, size
 }
 
 // applyTierObjSweep removes remote object pending deletion and the free-version
 // tracking this information.
-func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta actionMeta) {
-	if !meta.oi.TransitionedObject.FreeVersion {
+func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+	if !oi.TransitionedObject.FreeVersion {
 		// nothing to be done
 		return
 	}
@@ -989,18 +951,18 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta
 		return err
 	}
 	// Remove the remote object
-	err := deleteObjectFromRemoteTier(ctx, meta.oi.TransitionedObject.Name, meta.oi.TransitionedObject.VersionID, meta.oi.TransitionedObject.Tier)
+	err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
 
 	// Remove this free version
-	opts := ObjectOptions{}
-	opts.VersionID = meta.oi.VersionID
-	_, err = o.DeleteObject(ctx, meta.oi.Bucket, meta.oi.Name, opts)
+	_, err = o.DeleteObject(ctx, oi.Bucket, oi.Name, ObjectOptions{
+		VersionID: oi.VersionID,
+	})
 	if err == nil {
-		auditLogLifecycle(ctx, meta.oi, ILMFreeVersionDelete)
+		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete)
 	}
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
@@ -1012,19 +974,19 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta
 // The resulting size on disk will always be returned.
 // The metadata will be compared to consensus on the object layer before any changes are applied.
 // If no metadata is supplied, -1 is returned if no action is taken.
-func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, meta actionMeta, sizeS *sizeSummary) int64 {
-	i.applyTierObjSweep(ctx, o, meta)
+func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) int64 {
+	i.applyTierObjSweep(ctx, o, oi)
 
-	applied, size := i.applyLifecycle(ctx, o, meta)
+	applied, size := i.applyLifecycle(ctx, o, oi)
 	// For instance, an applied lifecycle means we remove/transitioned an object
 	// from the current deployment, which means we don't have to call healing
 	// routine even if we are asked to do via heal flag.
 	if !applied {
 		if i.heal {
-			size = i.applyHealing(ctx, o, meta)
+			size = i.applyHealing(ctx, o, oi)
 		}
 		// replicate only if lifecycle rules are not applied.
-		i.healReplication(ctx, o, meta.oi.Clone(), sizeS)
+		i.healReplication(ctx, o, oi.Clone(), sizeS)
 	}
 	return size
 }
@@ -1063,17 +1025,9 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, obj Ob
 	return action
 }
 
-func applyTransitionAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) bool {
-	srcOpts := ObjectOptions{}
-	if obj.TransitionedObject.Status == "" {
-		srcOpts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
-		srcOpts.VersionID = obj.VersionID
-		// mark transition as pending
-		obj.UserDefined[ReservedMetadataPrefixLower+TransitionStatus] = lifecycle.TransitionPending
-		obj.metadataOnly = true // Perform only metadata updates.
-		if obj.DeleteMarker {
-			return false
-		}
+func applyTransitionRule(obj ObjectInfo) bool {
+	if obj.DeleteMarker {
+		return false
 	}
 	globalTransitionState.queueTransitionTask(obj)
 	return true
@@ -1097,7 +1051,9 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 }
 
 func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
-	opts := ObjectOptions{}
+	opts := ObjectOptions{
+		Expiration: ExpirationOptions{Expire: true},
+	}
 
 	if applyOnVersion {
 		opts.VersionID = obj.VersionID
@@ -1136,22 +1092,20 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 }
 
 // Apply object, object version, restored object or restored object version action on the given object
-func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
-	if obj.TransitionedObject.Status != "" {
-		return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject)
-	}
-	return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj, applyOnVersion)
+func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
+	globalExpiryState.queueExpiryTask(obj, restoredObject, applyOnVersion)
+	return true
 }
 
 // Perform actions (removal or transitioning of objects), return true the action is successfully performed
-func applyLifecycleAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) (success bool) {
+func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo) (success bool) {
 	switch action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-		success = applyExpiryRule(ctx, objLayer, obj, false, action == lifecycle.DeleteVersionAction)
+		success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
 	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		success = applyExpiryRule(ctx, objLayer, obj, true, action == lifecycle.DeleteRestoredVersionAction)
+		success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		success = applyTransitionAction(ctx, action, objLayer, obj)
+		success = applyTransitionRule(obj)
 	}
 	return
 }
@@ -1163,74 +1117,94 @@ func (i *scannerItem) objectPath() string {
 
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
-	existingObjResync := i.replication.Resync(ctx, oi)
+	roi := getHealReplicateObjectInfo(oi, i.replication)
+
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		// heal delete marker replication failure or versioned delete replication failure
 		if oi.ReplicationStatus == replication.Pending ||
 			oi.ReplicationStatus == replication.Failed ||
 			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
+			i.healReplicationDeletes(ctx, o, roi)
 			return
 		}
 		// if replication status is Complete on DeleteMarker and existing object resync required
-		if existingObjResync && oi.ReplicationStatus == replication.Completed {
-			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
+		if roi.ExistingObjResync.mustResync() && (oi.ReplicationStatus == replication.Completed || oi.ReplicationStatus.Empty()) {
+			i.healReplicationDeletes(ctx, o, roi)
 			return
 		}
 		return
 	}
-	roi := ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType}
-	if existingObjResync {
+	if roi.ExistingObjResync.mustResync() {
 		roi.OpType = replication.ExistingObjectReplicationType
-		roi.ResetID = i.replication.ResetID
 	}
+
+	if roi.TargetStatuses != nil {
+		if sizeS.replTargetStats == nil {
+			sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
+		}
+		for arn, tgtStatus := range roi.TargetStatuses {
+			tgtSizeS, ok := sizeS.replTargetStats[arn]
+			if !ok {
+				tgtSizeS = replTargetSizeSummary{}
+			}
+			switch tgtStatus {
+			case replication.Pending:
+				tgtSizeS.pendingCount++
+				tgtSizeS.pendingSize += oi.Size
+				sizeS.pendingCount++
+				sizeS.pendingSize += oi.Size
+			case replication.Failed:
+				tgtSizeS.failedSize += oi.Size
+				tgtSizeS.failedCount++
+				sizeS.failedSize += oi.Size
+				sizeS.failedCount++
+			case replication.Completed, "COMPLETE":
+				tgtSizeS.replicatedSize += oi.Size
+				sizeS.replicatedSize += oi.Size
+			}
+			sizeS.replTargetStats[arn] = tgtSizeS
+		}
+	}
+
 	switch oi.ReplicationStatus {
-	case replication.Pending:
-		sizeS.pendingCount++
-		sizeS.pendingSize += oi.Size
+	case replication.Pending, replication.Failed:
 		globalReplicationPool.queueReplicaTask(roi)
 		return
-	case replication.Failed:
-		sizeS.failedSize += oi.Size
-		sizeS.failedCount++
-		globalReplicationPool.queueReplicaTask(roi)
-		return
-	case replication.Completed, "COMPLETE":
-		sizeS.replicatedSize += oi.Size
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
-	if existingObjResync {
+	if roi.ExistingObjResync.mustResync() {
 		globalReplicationPool.queueReplicaTask(roi)
 	}
 }
 
 // healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo, existingObject bool) {
+func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, roi ReplicateObjectInfo) {
 	// handle soft delete and permanent delete failures here.
-	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
+	if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
 		versionID := ""
 		dmVersionID := ""
-		if oi.VersionPurgeStatus.Empty() {
-			dmVersionID = oi.VersionID
+		if roi.VersionPurgeStatus.Empty() {
+			dmVersionID = roi.VersionID
 		} else {
-			versionID = oi.VersionID
+			versionID = roi.VersionID
 		}
+
 		doi := DeletedObjectReplicationInfo{
 			DeletedObject: DeletedObject{
-				ObjectName:                    oi.Name,
-				DeleteMarkerVersionID:         dmVersionID,
-				VersionID:                     versionID,
-				DeleteMarkerReplicationStatus: string(oi.ReplicationStatus),
-				DeleteMarkerMTime:             DeleteMarkerMTime{oi.ModTime},
-				DeleteMarker:                  oi.DeleteMarker,
-				VersionPurgeStatus:            oi.VersionPurgeStatus,
+				ObjectName:            roi.Name,
+				DeleteMarkerVersionID: dmVersionID,
+				VersionID:             versionID,
+				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+				DeleteMarker:          roi.DeleteMarker,
 			},
-			Bucket: oi.Bucket,
+			Bucket: roi.Bucket,
 		}
-		if existingObject {
+		if roi.ExistingObjResync.mustResync() {
 			doi.OpType = replication.ExistingObjectReplicationType
-			doi.ResetID = i.replication.ResetID
+			queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
+			return
 		}
 		globalReplicationPool.queueReplicaDeleteTask(doi)
 	}

cmd/data-update-tracker.go

@@ -46,7 +46,7 @@ const (
 	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 6
+	dataUpdateTrackerVersion      = 7
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -397,7 +397,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2, 3, 4, 5:
+	case 1, 2, 3, 4, 5, 6:
 		if intDataUpdateTracker.debug {
 			console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		}

cmd/data-usage-cache.go

@@ -46,20 +46,74 @@ type dataUsageHash string
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
 
-//msgp:tuple dataUsageEntry
 type dataUsageEntry struct {
-	Children dataUsageHashMap
+	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
-	Size             int64
-	Objects          uint64
-	Versions         uint64 // Versions that are not delete markers.
-	ObjSizes         sizeHistogram
-	ReplicationStats *replicationStats
-	Compacted        bool
+	Size             int64                `msg:"sz"`
+	Objects          uint64               `msg:"os"`
+	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
+	ObjSizes         sizeHistogram        `msg:"szs"`
+	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
+	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
+	Compacted        bool                 `msg:"c"`
 }
 
-//msgp:tuple replicationStats
-type replicationStats struct {
+// allTierStats is a collection of per-tier stats across all configured remote
+// tiers.
+type allTierStats struct {
+	Tiers map[string]tierStats `msg:"ts"`
+}
+
+func newAllTierStats() *allTierStats {
+	return &allTierStats{
+		Tiers: make(map[string]tierStats),
+	}
+}
+
+func (ats *allTierStats) addSizes(sz sizeSummary) {
+	for tier, st := range sz.tiers {
+		ats.Tiers[tier] = ats.Tiers[tier].add(st)
+	}
+}
+
+func (ats *allTierStats) merge(other *allTierStats) {
+	for tier, st := range other.Tiers {
+		ats.Tiers[tier] = ats.Tiers[tier].add(st)
+	}
+}
+
+func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats {
+	if ats == nil {
+		return stats
+	}
+
+	// Update stats for tiers as they become available.
+	for tier, st := range ats.Tiers {
+		stats[tier] = madmin.TierStats{
+			TotalSize:   st.TotalSize,
+			NumVersions: st.NumVersions,
+			NumObjects:  st.NumObjects,
+		}
+	}
+	return stats
+}
+
+// tierStats holds per-tier stats of a remote tier.
+type tierStats struct {
+	TotalSize   uint64 `msg:"ts"`
+	NumVersions int    `msg:"nv"`
+	NumObjects  int    `msg:"no"`
+}
+
+func (ts tierStats) add(u tierStats) tierStats {
+	ts.TotalSize += u.TotalSize
+	ts.NumVersions += u.NumVersions
+	ts.NumObjects += u.NumObjects
+	return ts
+}
+
+//msgp:tuple replicationStatsV1
+type replicationStatsV1 struct {
 	PendingSize          uint64
 	ReplicatedSize       uint64
 	FailedSize           uint64
@@ -72,8 +126,44 @@ type replicationStats struct {
 	AfterThresholdCount  uint64
 }
 
-//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4
-//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4
+func (rsv1 replicationStatsV1) Empty() bool {
+	return rsv1.ReplicatedSize == 0 &&
+		rsv1.FailedSize == 0 &&
+		rsv1.FailedCount == 0
+}
+
+//msgp:tuple replicationStats
+type replicationStats struct {
+	PendingSize          uint64
+	ReplicatedSize       uint64
+	FailedSize           uint64
+	FailedCount          uint64
+	PendingCount         uint64
+	MissedThresholdSize  uint64
+	AfterThresholdSize   uint64
+	MissedThresholdCount uint64
+	AfterThresholdCount  uint64
+}
+
+func (rs replicationStats) Empty() bool {
+	return rs.ReplicatedSize == 0 &&
+		rs.FailedSize == 0 &&
+		rs.FailedCount == 0
+}
+
+type replicationAllStats struct {
+	Targets     map[string]replicationStats `msg:"t,omitempty"`
+	ReplicaSize uint64                      `msg:"r,omitempty"`
+}
+
+//msgp:tuple replicationAllStatsV1
+type replicationAllStatsV1 struct {
+	Targets     map[string]replicationStats
+	ReplicaSize uint64 `msg:"ReplicaSize,omitempty"`
+}
+
+//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
+//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
 
 //msgp:tuple dataUsageEntryV2
 type dataUsageEntryV2 struct {
@@ -104,7 +194,31 @@ type dataUsageEntryV4 struct {
 	Size             int64
 	Objects          uint64
 	ObjSizes         sizeHistogram
-	ReplicationStats replicationStats
+	ReplicationStats replicationStatsV1
+}
+
+//msgp:tuple dataUsageEntryV5
+type dataUsageEntryV5 struct {
+	Children dataUsageHashMap
+	// These fields do no include any children.
+	Size             int64
+	Objects          uint64
+	Versions         uint64 // Versions that are not delete markers.
+	ObjSizes         sizeHistogram
+	ReplicationStats *replicationStatsV1
+	Compacted        bool
+}
+
+//msgp:tuple dataUsageEntryV6
+type dataUsageEntryV6 struct {
+	Children dataUsageHashMap
+	// These fields do no include any children.
+	Size             int64
+	Objects          uint64
+	Versions         uint64 // Versions that are not delete markers.
+	ObjSizes         sizeHistogram
+	ReplicationStats *replicationAllStatsV1
+	Compacted        bool
 }
 
 // dataUsageCache contains a cache of data usage entries latest version.
@@ -114,8 +228,8 @@ type dataUsageCache struct {
 	Disks []string
 }
 
-//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4
-//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4
+//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
+//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
 
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
@@ -124,20 +238,34 @@ type dataUsageCacheV2 struct {
 	Cache map[string]dataUsageEntryV2
 }
 
-// dataUsageCache contains a cache of data usage entries version 3.
+// dataUsageCacheV3 contains a cache of data usage entries version 3.
 type dataUsageCacheV3 struct {
 	Info  dataUsageCacheInfo
 	Disks []string
 	Cache map[string]dataUsageEntryV3
 }
 
-// dataUsageCache contains a cache of data usage entries version 4.
+// dataUsageCacheV4 contains a cache of data usage entries version 4.
 type dataUsageCacheV4 struct {
 	Info  dataUsageCacheInfo
 	Disks []string
 	Cache map[string]dataUsageEntryV4
 }
 
+// dataUsageCacheV5 contains a cache of data usage entries version 5.
+type dataUsageCacheV5 struct {
+	Info  dataUsageCacheInfo
+	Disks []string
+	Cache map[string]dataUsageEntryV5
+}
+
+// dataUsageCacheV6 contains a cache of data usage entries version 6.
+type dataUsageCacheV6 struct {
+	Info  dataUsageCacheInfo
+	Disks []string
+	Cache map[string]dataUsageEntryV6
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -170,17 +298,34 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Versions += summary.versions
 	e.ObjSizes.add(summary.totalSize)
 
-	if summary.replicaSize > 0 || summary.pendingSize > 0 || summary.replicatedSize > 0 ||
-		summary.failedCount > 0 || summary.pendingCount > 0 || summary.failedSize > 0 {
-		if e.ReplicationStats == nil {
-			e.ReplicationStats = &replicationStats{}
+	if e.ReplicationStats == nil {
+		e.ReplicationStats = &replicationAllStats{
+			Targets: make(map[string]replicationStats),
 		}
-		e.ReplicationStats.ReplicatedSize += uint64(summary.replicatedSize)
-		e.ReplicationStats.FailedSize += uint64(summary.failedSize)
-		e.ReplicationStats.PendingSize += uint64(summary.pendingSize)
-		e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
-		e.ReplicationStats.PendingCount += summary.pendingCount
-		e.ReplicationStats.FailedCount += summary.failedCount
+	} else if e.ReplicationStats.Targets == nil {
+		e.ReplicationStats.Targets = make(map[string]replicationStats)
+	}
+	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
+
+	if summary.replTargetStats != nil {
+		for arn, st := range summary.replTargetStats {
+			tgtStat, ok := e.ReplicationStats.Targets[arn]
+			if !ok {
+				tgtStat = replicationStats{}
+			}
+			tgtStat.PendingSize += uint64(st.pendingSize)
+			tgtStat.FailedSize += uint64(st.failedSize)
+			tgtStat.ReplicatedSize += uint64(st.replicatedSize)
+			tgtStat.FailedCount += st.failedCount
+			tgtStat.PendingCount += st.pendingCount
+			e.ReplicationStats.Targets[arn] = tgtStat
+		}
+	}
+	if summary.tiers != nil {
+		if e.AllTierStats == nil {
+			e.AllTierStats = newAllTierStats()
+		}
+		e.AllTierStats.addSizes(summary)
 	}
 }
 
@@ -189,24 +334,35 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Objects += other.Objects
 	e.Versions += other.Versions
 	e.Size += other.Size
-	ors := other.ReplicationStats
-	empty := replicationStats{}
-	if ors != nil && *ors != empty {
+	if other.ReplicationStats != nil {
 		if e.ReplicationStats == nil {
-			e.ReplicationStats = &replicationStats{}
+			e.ReplicationStats = &replicationAllStats{Targets: make(map[string]replicationStats)}
+		} else if e.ReplicationStats.Targets == nil {
+			e.ReplicationStats.Targets = make(map[string]replicationStats)
 		}
-		e.ReplicationStats.PendingSize += other.ReplicationStats.PendingSize
-		e.ReplicationStats.FailedSize += other.ReplicationStats.FailedSize
-		e.ReplicationStats.ReplicatedSize += other.ReplicationStats.ReplicatedSize
 		e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize
-		e.ReplicationStats.PendingCount += other.ReplicationStats.PendingCount
-		e.ReplicationStats.FailedCount += other.ReplicationStats.FailedCount
-
+		for arn, stat := range other.ReplicationStats.Targets {
+			st := e.ReplicationStats.Targets[arn]
+			e.ReplicationStats.Targets[arn] = replicationStats{
+				PendingSize:    stat.PendingSize + st.PendingSize,
+				FailedSize:     stat.FailedSize + st.FailedSize,
+				ReplicatedSize: stat.ReplicatedSize + st.ReplicatedSize,
+				PendingCount:   stat.PendingCount + st.PendingCount,
+				FailedCount:    stat.FailedCount + st.FailedCount,
+			}
+		}
 	}
 
 	for i, v := range other.ObjSizes[:] {
 		e.ObjSizes[i] += v
 	}
+
+	if other.AllTierStats != nil {
+		if e.AllTierStats == nil {
+			e.AllTierStats = newAllTierStats()
+		}
+		e.AllTierStats.merge(other.AllTierStats)
+	}
 }
 
 // mod returns true if the hash mod cycles == cycle.
@@ -238,6 +394,29 @@ func (e *dataUsageEntry) removeChild(hash dataUsageHash) {
 	}
 }
 
+// Create a clone of the entry.
+func (e dataUsageEntry) clone() dataUsageEntry {
+	// We operate on a copy from the receiver.
+	if e.Children != nil {
+		ch := make(dataUsageHashMap, len(e.Children))
+		for k, v := range e.Children {
+			ch[k] = v
+		}
+		e.Children = ch
+	}
+	if e.ReplicationStats != nil {
+		// Copy to new struct
+		r := *e.ReplicationStats
+		e.ReplicationStats = &r
+	}
+	if e.AllTierStats != nil {
+		ats := newAllTierStats()
+		ats.merge(e.AllTierStats)
+		e.AllTierStats = ats
+	}
+	return e
+}
+
 // find a path in the cache.
 // Returns nil if not found.
 func (d *dataUsageCache) find(path string) *dataUsageEntry {
@@ -343,27 +522,20 @@ func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
 
 // dui converts the flattened version of the path to madmin.DataUsageInfo.
 // As a side effect d will be flattened, use a clone if this is not ok.
-func (d *dataUsageCache) dui(path string, buckets []BucketInfo) madmin.DataUsageInfo {
+func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	e := d.find(path)
 	if e == nil {
 		// No entry found, return empty.
-		return madmin.DataUsageInfo{}
+		return DataUsageInfo{}
 	}
 	flat := d.flatten(*e)
-	dui := madmin.DataUsageInfo{
+	dui := DataUsageInfo{
 		LastUpdate:        d.Info.LastUpdate,
 		ObjectsTotalCount: flat.Objects,
 		ObjectsTotalSize:  uint64(flat.Size),
 		BucketsCount:      uint64(len(e.Children)),
 		BucketsUsage:      d.bucketsUsageInfo(buckets),
-	}
-	if flat.ReplicationStats != nil {
-		dui.ReplicationPendingSize = flat.ReplicationStats.PendingSize
-		dui.ReplicatedSize = flat.ReplicationStats.ReplicatedSize
-		dui.ReplicationFailedSize = flat.ReplicationStats.FailedSize
-		dui.ReplicationPendingCount = flat.ReplicationStats.PendingCount
-		dui.ReplicationFailedCount = flat.ReplicationStats.FailedCount
-		dui.ReplicaSize = flat.ReplicationStats.ReplicaSize
+		TierStats:         d.tiersUsageInfo(buckets),
 	}
 	return dui
 }
@@ -580,28 +752,53 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
 	return res
 }
 
-// bucketsUsageInfo returns the buckets usage info as a map, with
-// key as bucket name
-func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]madmin.BucketUsageInfo {
-	var dst = make(map[string]madmin.BucketUsageInfo, len(buckets))
+func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
+	dst := newAllTierStats()
 	for _, bucket := range buckets {
 		e := d.find(bucket.Name)
 		if e == nil {
 			continue
 		}
 		flat := d.flatten(*e)
-		bui := madmin.BucketUsageInfo{
+		if flat.AllTierStats == nil {
+			continue
+		}
+		dst.merge(flat.AllTierStats)
+	}
+	if len(dst.Tiers) == 0 {
+		return nil
+	}
+	return dst
+}
+
+// bucketsUsageInfo returns the buckets usage info as a map, with
+// key as bucket name
+func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]BucketUsageInfo {
+	var dst = make(map[string]BucketUsageInfo, len(buckets))
+	for _, bucket := range buckets {
+		e := d.find(bucket.Name)
+		if e == nil {
+			continue
+		}
+		flat := d.flatten(*e)
+		bui := BucketUsageInfo{
 			Size:                 uint64(flat.Size),
 			ObjectsCount:         flat.Objects,
 			ObjectSizesHistogram: flat.ObjSizes.toMap(),
 		}
 		if flat.ReplicationStats != nil {
-			bui.ReplicationPendingSize = flat.ReplicationStats.PendingSize
-			bui.ReplicatedSize = flat.ReplicationStats.ReplicatedSize
-			bui.ReplicationFailedSize = flat.ReplicationStats.FailedSize
-			bui.ReplicationPendingCount = flat.ReplicationStats.PendingCount
-			bui.ReplicationFailedCount = flat.ReplicationStats.FailedCount
 			bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
+			bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
+			for arn, stat := range flat.ReplicationStats.Targets {
+				bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
+					ReplicationPendingSize:  stat.PendingSize,
+					ReplicatedSize:          stat.ReplicatedSize,
+					ReplicationFailedSize:   stat.FailedSize,
+					ReplicationPendingCount: stat.PendingCount,
+					ReplicationFailedCount:  stat.FailedCount,
+				}
+
+			}
 		}
 		dst[bucket.Name] = bui
 	}
@@ -610,24 +807,29 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]madmi
 
 // bucketUsageInfo returns the buckets usage info.
 // If not found all values returned are zero values.
-func (d *dataUsageCache) bucketUsageInfo(bucket string) madmin.BucketUsageInfo {
+func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
 	e := d.find(bucket)
 	if e == nil {
-		return madmin.BucketUsageInfo{}
+		return BucketUsageInfo{}
 	}
 	flat := d.flatten(*e)
-	bui := madmin.BucketUsageInfo{
+	bui := BucketUsageInfo{
 		Size:                 uint64(flat.Size),
 		ObjectsCount:         flat.Objects,
 		ObjectSizesHistogram: flat.ObjSizes.toMap(),
 	}
 	if flat.ReplicationStats != nil {
-		bui.ReplicationPendingSize = flat.ReplicationStats.PendingSize
-		bui.ReplicatedSize = flat.ReplicationStats.ReplicatedSize
-		bui.ReplicationFailedSize = flat.ReplicationStats.FailedSize
-		bui.ReplicationPendingCount = flat.ReplicationStats.PendingCount
-		bui.ReplicationFailedCount = flat.ReplicationStats.FailedCount
 		bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
+		bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
+		for arn, stat := range flat.ReplicationStats.Targets {
+			bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
+				ReplicationPendingSize:  stat.PendingSize,
+				ReplicatedSize:          stat.ReplicatedSize,
+				ReplicationFailedSize:   stat.FailedSize,
+				ReplicationPendingCount: stat.PendingCount,
+				ReplicationFailedCount:  stat.FailedCount,
+			}
+		}
 	}
 	return bui
 }
@@ -672,7 +874,7 @@ func (d *dataUsageCache) clone() dataUsageCache {
 		Cache: make(map[string]dataUsageEntry, len(d.Cache)),
 	}
 	for k, v := range d.Cache {
-		clone.Cache[k] = v
+		clone.Cache[k] = v.clone()
 	}
 	return clone
 }
@@ -726,6 +928,7 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 		case ObjectNotFound:
 		case BucketNotFound:
 		case InsufficientReadQuorum:
+		case StorageErr:
 		default:
 			return toObjectErr(err, dataUsageBucket, name)
 		}
@@ -771,7 +974,9 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
 const (
-	dataUsageCacheVerCurrent = 5
+	dataUsageCacheVerCurrent = 7
+	dataUsageCacheVerV6      = 6
+	dataUsageCacheVerV5      = 5
 	dataUsageCacheVerV4      = 4
 	dataUsageCacheVerV3      = 3
 	dataUsageCacheVerV2      = 2
@@ -866,9 +1071,17 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				Children: v.Children,
 			}
 			if v.ReplicatedSize > 0 || v.ReplicaSize > 0 || v.ReplicationFailedSize > 0 || v.ReplicationPendingSize > 0 {
-				due.ReplicationStats = &replicationStats{
+				due.ReplicationStats = &replicationAllStats{
+					Targets: make(map[string]replicationStats),
+				}
+				cfg, err := getReplicationConfig(GlobalContext, d.Info.Name)
+				if err != nil {
+					return err
+				}
+				due.ReplicationStats.ReplicaSize = v.ReplicaSize
+
+				due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
 					ReplicatedSize: v.ReplicatedSize,
-					ReplicaSize:    v.ReplicaSize,
 					FailedSize:     v.ReplicationFailedSize,
 					PendingSize:    v.ReplicationPendingSize,
 				}
@@ -899,9 +1112,24 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 				ObjSizes: v.ObjSizes,
 				Children: v.Children,
 			}
-			empty := replicationStats{}
+			empty := replicationStatsV1{}
+
 			if v.ReplicationStats != empty {
-				due.ReplicationStats = &v.ReplicationStats
+				due.ReplicationStats = &replicationAllStats{
+					Targets: make(map[string]replicationStats),
+				}
+				cfg, err := getReplicationConfig(GlobalContext, d.Info.Name)
+				if err != nil {
+					return err
+				}
+				due.ReplicationStats.Targets[cfg.RoleArn] = replicationStats{
+					ReplicatedSize: v.ReplicationStats.ReplicatedSize,
+					FailedSize:     v.ReplicationStats.FailedSize,
+					FailedCount:    v.ReplicationStats.FailedCount,
+					PendingSize:    v.ReplicationStats.PendingSize,
+					PendingCount:   v.ReplicationStats.PendingCount,
+				}
+				due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
 			}
 			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
 
@@ -909,15 +1137,107 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		}
 
 		// Populate compacted value and remove unneeded replica stats.
-		empty := replicationStats{}
 		for k, e := range d.Cache {
-			if e.ReplicationStats != nil && *e.ReplicationStats == empty {
+			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
 				e.ReplicationStats = nil
 			}
-
 			d.Cache[k] = e
 		}
 		return nil
+	case dataUsageCacheVerV5:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
+		dold := &dataUsageCacheV5{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Disks = dold.Disks
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		var arn string
+		for k, v := range dold.Cache {
+			due := dataUsageEntry{
+				Size:     v.Size,
+				Objects:  v.Objects,
+				ObjSizes: v.ObjSizes,
+				Children: v.Children,
+			}
+			if v.ReplicationStats != nil && !v.ReplicationStats.Empty() {
+				if arn == "" {
+					cfg, err := getReplicationConfig(GlobalContext, d.Info.Name)
+					if err != nil {
+						return err
+					}
+					d.Info.replication = replicationConfig{Config: cfg}
+					arn = d.Info.replication.Config.RoleArn
+				}
+
+				due.ReplicationStats = &replicationAllStats{
+					Targets: make(map[string]replicationStats),
+				}
+
+				if arn != "" {
+					due.ReplicationStats.Targets[arn] = replicationStats{
+						ReplicatedSize: v.ReplicationStats.ReplicatedSize,
+						FailedSize:     v.ReplicationStats.FailedSize,
+						FailedCount:    v.ReplicationStats.FailedCount,
+						PendingSize:    v.ReplicationStats.PendingSize,
+						PendingCount:   v.ReplicationStats.PendingCount,
+					}
+					due.ReplicationStats.ReplicaSize = v.ReplicationStats.ReplicaSize
+				}
+			}
+			due.Compacted = len(due.Children) == 0 && k != d.Info.Name
+
+			d.Cache[k] = due
+		}
+
+		// Populate compacted value and remove unneeded replica stats.
+		for k, e := range d.Cache {
+			if e.ReplicationStats != nil && len(e.ReplicationStats.Targets) == 0 {
+				e.ReplicationStats = nil
+			}
+			d.Cache[k] = e
+		}
+		return nil
+	case dataUsageCacheVerV6:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
+		dold := &dataUsageCacheV6{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Disks = dold.Disks
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		for k, v := range dold.Cache {
+			var replicationStats *replicationAllStats
+			if v.ReplicationStats != nil {
+				replicationStats = &replicationAllStats{
+					Targets:     v.ReplicationStats.Targets,
+					ReplicaSize: v.ReplicationStats.ReplicaSize,
+				}
+			}
+			due := dataUsageEntry{
+				Children:         v.Children,
+				Size:             v.Size,
+				Objects:          v.Objects,
+				Versions:         v.Versions,
+				ObjSizes:         v.ObjSizes,
+				ReplicationStats: replicationStats,
+				Compacted:        v.Compacted,
+			}
+			d.Cache[k] = due
+		}
+		return nil
 	case dataUsageCacheVerCurrent:
 		// Zstd compressed.
 		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))

cmd/data-usage-cache_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshalallTierStats(t *testing.T) {
+	v := allTierStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgallTierStats(b *testing.B) {
+	v := allTierStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgallTierStats(b *testing.B) {
+	v := allTierStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalallTierStats(b *testing.B) {
+	v := allTierStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeallTierStats(t *testing.T) {
+	v := allTierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeallTierStats Msgsize() is inaccurate")
+	}
+
+	vn := allTierStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeallTierStats(b *testing.B) {
+	v := allTierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeallTierStats(b *testing.B) {
+	v := allTierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaldataUsageCache(t *testing.T) {
 	v := dataUsageCache{}
 	bts, err := v.MarshalMsg(nil)
@@ -348,6 +461,232 @@ func BenchmarkDecodedataUsageEntry(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalreplicationAllStats(t *testing.T) {
+	v := replicationAllStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgreplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgreplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalreplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodereplicationAllStats(t *testing.T) {
+	v := replicationAllStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodereplicationAllStats Msgsize() is inaccurate")
+	}
+
+	vn := replicationAllStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodereplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodereplicationAllStats(b *testing.B) {
+	v := replicationAllStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalreplicationAllStatsV1(t *testing.T) {
+	v := replicationAllStatsV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgreplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgreplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalreplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodereplicationAllStatsV1(t *testing.T) {
+	v := replicationAllStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodereplicationAllStatsV1 Msgsize() is inaccurate")
+	}
+
+	vn := replicationAllStatsV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodereplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodereplicationAllStatsV1(b *testing.B) {
+	v := replicationAllStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalreplicationStats(t *testing.T) {
 	v := replicationStats{}
 	bts, err := v.MarshalMsg(nil)
@@ -461,6 +800,119 @@ func BenchmarkDecodereplicationStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalreplicationStatsV1(t *testing.T) {
+	v := replicationStatsV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgreplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgreplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalreplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodereplicationStatsV1(t *testing.T) {
+	v := replicationStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodereplicationStatsV1 Msgsize() is inaccurate")
+	}
+
+	vn := replicationStatsV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodereplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodereplicationStatsV1(b *testing.B) {
+	v := replicationStatsV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
 	v := sizeHistogram{}
 	bts, err := v.MarshalMsg(nil)
@@ -573,3 +1025,116 @@ func BenchmarkDecodesizeHistogram(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshaltierStats(t *testing.T) {
+	v := tierStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgtierStats(b *testing.B) {
+	v := tierStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgtierStats(b *testing.B) {
+	v := tierStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaltierStats(b *testing.B) {
+	v := tierStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodetierStats(t *testing.T) {
+	v := tierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodetierStats Msgsize() is inaccurate")
+	}
+
+	vn := tierStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodetierStats(b *testing.B) {
+	v := tierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodetierStats(b *testing.B) {
+	v := tierStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage-utils.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"sort"
+	"time"
+
+	"github.com/minio/madmin-go"
+)
+
+// BucketTargetUsageInfo - bucket target usage info provides
+// - replicated size for all objects sent to this target
+// - replica size for all objects received from this target
+// - replication pending size for all objects pending replication to this target
+// - replication failed size for all objects failed replication to this target
+// - replica pending count
+// - replica failed count
+type BucketTargetUsageInfo struct {
+	ReplicationPendingSize  uint64 `json:"objectsPendingReplicationTotalSize"`
+	ReplicationFailedSize   uint64 `json:"objectsFailedReplicationTotalSize"`
+	ReplicatedSize          uint64 `json:"objectsReplicatedTotalSize"`
+	ReplicaSize             uint64 `json:"objectReplicaTotalSize"`
+	ReplicationPendingCount uint64 `json:"objectsPendingReplicationCount"`
+	ReplicationFailedCount  uint64 `json:"objectsFailedReplicationCount"`
+}
+
+// BucketUsageInfo - bucket usage info provides
+// - total size of the bucket
+// - total objects in a bucket
+// - object size histogram per bucket
+type BucketUsageInfo struct {
+	Size uint64 `json:"size"`
+	// Following five fields suffixed with V1 are here for backward compatibility
+	// Total Size for objects that have not yet been replicated
+	ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
+	// Total size for objects that have witness one or more failures and will be retried
+	ReplicationFailedSizeV1 uint64 `json:"objectsFailedReplicationTotalSize"`
+	// Total size for objects that have been replicated to destination
+	ReplicatedSizeV1 uint64 `json:"objectsReplicatedTotalSize"`
+	// Total number of objects pending replication
+	ReplicationPendingCountV1 uint64 `json:"objectsPendingReplicationCount"`
+	// Total number of objects that failed replication
+	ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
+
+	ObjectsCount         uint64                           `json:"objectsCount"`
+	ObjectSizesHistogram map[string]uint64                `json:"objectsSizesHistogram"`
+	ReplicaSize          uint64                           `json:"objectReplicaTotalSize"`
+	ReplicationInfo      map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+}
+
+// DataUsageInfo represents data usage stats of the underlying Object API
+type DataUsageInfo struct {
+	// LastUpdate is the timestamp of when the data usage info was last updated.
+	// This does not indicate a full scan.
+	LastUpdate time.Time `json:"lastUpdate"`
+
+	// Objects total count across all buckets
+	ObjectsTotalCount uint64 `json:"objectsCount"`
+
+	// Objects total size across all buckets
+	ObjectsTotalSize uint64                           `json:"objectsTotalSize"`
+	ReplicationInfo  map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+	// Total number of buckets in this cluster
+	BucketsCount uint64 `json:"bucketsCount"`
+
+	// Buckets usage info provides following information across all buckets
+	// - total size of the bucket
+	// - total objects in a bucket
+	// - object size histogram per bucket
+	BucketsUsage map[string]BucketUsageInfo `json:"bucketsUsageInfo"`
+	// Deprecated kept here for backward compatibility reasons.
+	BucketSizes map[string]uint64 `json:"bucketsSizes"`
+
+	// TierStats contains per-tier stats of all configured remote tiers
+	TierStats *allTierStats `json:"tierStats,omitempty"`
+}
+
+func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
+	if globalTierConfigMgr.Empty() {
+		return nil
+	}
+
+	ts := make(map[string]madmin.TierStats)
+	// Add configured remote tiers
+	for tier := range globalTierConfigMgr.Tiers {
+		ts[tier] = madmin.TierStats{}
+	}
+	// Add STANDARD (hot-tier)
+	ts[minioHotTier] = madmin.TierStats{}
+
+	ts = dui.TierStats.adminStats(ts)
+	infos := make([]madmin.TierInfo, 0, len(ts))
+	for tier, st := range ts {
+		var tierType string
+		if tier == minioHotTier {
+			tierType = "internal"
+		} else {
+			tierType = globalTierConfigMgr.Tiers[tier].Type.String()
+		}
+		infos = append(infos, madmin.TierInfo{
+			Name:  tier,
+			Type:  tierType,
+			Stats: st,
+		})
+	}
+
+	sort.Slice(infos, func(i, j int) bool {
+		if infos[i].Type == "internal" {
+			return true
+		}
+		if infos[j].Type == "internal" {
+			return false
+		}
+		return infos[i].Name < infos[j].Name
+	})
+	return infos
+}

cmd/data-usage.go

@@ -24,7 +24,6 @@ import (
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 )
@@ -39,7 +38,7 @@ const (
 )
 
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan madmin.DataUsageInfo) {
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
 	for dataUsageInfo := range dui {
 		var json = jsoniter.ConfigCompatibleWithStandardLibrary
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
@@ -84,7 +83,8 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 				}
 
 				for id, usageInfo := range cache.flattenChildrens(*root) {
-					prefix := strings.TrimPrefix(id, bucket+slashSeparator)
+					prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator))
+					// decodeDirObject to avoid any __XL_DIR__ objects
 					m[prefix] += uint64(usageInfo.Size)
 				}
 			}
@@ -94,27 +94,26 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 	return m, nil
 }
 
-func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (madmin.DataUsageInfo, error) {
+func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
-			return madmin.DataUsageInfo{}, nil
+			return DataUsageInfo{}, nil
 		}
-		return madmin.DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
+		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
 	}
 	defer r.Close()
 
-	var dataUsageInfo madmin.DataUsageInfo
+	var dataUsageInfo DataUsageInfo
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.NewDecoder(r).Decode(&dataUsageInfo); err != nil {
-		return madmin.DataUsageInfo{}, err
+		return DataUsageInfo{}, err
 	}
-
 	// For forward compatibility reasons, we need to add this code.
 	if len(dataUsageInfo.BucketsUsage) == 0 {
-		dataUsageInfo.BucketsUsage = make(map[string]madmin.BucketUsageInfo, len(dataUsageInfo.BucketSizes))
+		dataUsageInfo.BucketsUsage = make(map[string]BucketUsageInfo, len(dataUsageInfo.BucketSizes))
 		for bucket, size := range dataUsageInfo.BucketSizes {
-			dataUsageInfo.BucketsUsage[bucket] = madmin.BucketUsageInfo{Size: size}
+			dataUsageInfo.BucketsUsage[bucket] = BucketUsageInfo{Size: size}
 		}
 	}
 
@@ -125,6 +124,23 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (madmin.D
 			dataUsageInfo.BucketSizes[bucket] = bui.Size
 		}
 	}
-
+	// For forward compatibility reasons, we need to add this code.
+	for bucket, bui := range dataUsageInfo.BucketsUsage {
+		if bui.ReplicatedSizeV1 > 0 || bui.ReplicationFailedCountV1 > 0 ||
+			bui.ReplicationFailedSizeV1 > 0 || bui.ReplicationPendingCountV1 > 0 {
+			dataUsageInfo.ReplicationInfo = make(map[string]BucketTargetUsageInfo)
+			cfg, err := getReplicationConfig(GlobalContext, bucket)
+			if err != nil {
+				return DataUsageInfo{}, err
+			}
+			dataUsageInfo.ReplicationInfo[cfg.RoleArn] = BucketTargetUsageInfo{
+				ReplicationFailedSize:   bui.ReplicationFailedSizeV1,
+				ReplicationFailedCount:  bui.ReplicationFailedCountV1,
+				ReplicatedSize:          bui.ReplicatedSizeV1,
+				ReplicationPendingCount: bui.ReplicationPendingCountV1,
+				ReplicationPendingSize:  bui.ReplicationPendingSizeV1,
+			}
+		}
+	}
 	return dataUsageInfo, nil
 }

cmd/data-usage_test.go

@@ -179,6 +179,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
 		got, err = scanDataFolder(context.Background(), base, got, getSize)
+		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -423,6 +424,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
 		got, err = scanDataFolder(context.Background(), base, got, getSize)
+		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -578,17 +580,16 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		t.Fatal(err)
 	}
 	e := want.find("abucket/dir2")
-	e.ReplicationStats = &replicationStats{
-		PendingSize:          1,
-		ReplicatedSize:       2,
-		FailedSize:           3,
-		ReplicaSize:          4,
-		FailedCount:          5,
-		PendingCount:         6,
-		MissedThresholdSize:  7,
-		AfterThresholdSize:   8,
-		MissedThresholdCount: 9,
-		AfterThresholdCount:  10,
+	e.ReplicationStats = &replicationAllStats{
+		Targets: map[string]replicationStats{
+			"arn": {
+				PendingSize:    1,
+				ReplicatedSize: 2,
+				FailedSize:     3,
+				FailedCount:    5,
+				PendingCount:   6,
+			},
+		},
 	}
 	want.replace("abucket/dir2", "", *e)
 	var buf bytes.Buffer

cmd/disk-cache-backend.go

@@ -938,6 +938,11 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		rs = nil
 	}
 
+	if objInfo.IsCompressed() {
+		// Cache isn't compressed.
+		delete(objInfo.UserDefined, ReservedMetadataPrefix+"compression")
+	}
+
 	// For a directory, we need to send an reader that returns no bytes.
 	if HasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
@@ -945,7 +950,6 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		gr, gerr := NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts)
 		return gr, numHits, gerr
 	}
-
 	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts)
 	if nErr != nil {
 		return nil, numHits, nErr

cmd/encryption-v1.go

@@ -68,14 +68,15 @@ const (
 
 )
 
-// isEncryptedMultipart returns true if the current object is
+// isMultipart returns true if the current object is
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
-func (o *ObjectInfo) isEncryptedMultipart() bool {
+func (o *ObjectInfo) isMultipart() bool {
 	if len(o.Parts) == 0 {
 		return false
 	}
-	if !crypto.IsMultiPart(o.UserDefined) {
+	_, encrypted := crypto.IsEncrypted(o.UserDefined)
+	if encrypted && !crypto.IsMultiPart(o.UserDefined) {
 		return false
 	}
 	for _, part := range o.Parts {
@@ -427,7 +428,7 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint3
 
 	bucket, object := oi.Bucket, oi.Name
 	// Single part case
-	if !oi.isEncryptedMultipart() {
+	if !oi.isMultipart() {
 		var reader io.Reader
 		var err error
 		if copySource {
@@ -589,7 +590,7 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	if !o.isEncryptedMultipart() {
+	if !o.isMultipart() {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
 			err = errObjectTampered // assign correct error type
@@ -732,7 +733,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	// Assemble slice of (decrypted) part sizes in `sizes`
 	var sizes []int64
 	var decObjSize int64 // decrypted total object size
-	if o.isEncryptedMultipart() {
+	if o.isMultipart() {
 		sizes = make([]int64, len(o.Parts))
 		for i, part := range o.Parts {
 			var partSize uint64

cmd/endpoint_test.go

@@ -128,6 +128,12 @@ func TestNewEndpoints(t *testing.T) {
 }
 
 func TestCreateEndpoints(t *testing.T) {
+	tempGlobalMinioPort := globalMinioPort
+	defer func() {
+		globalMinioPort = tempGlobalMinioPort
+	}()
+	globalMinioPort = "9000"
+
 	// Filter ipList by IPs those do not start with '127.'.
 	nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {
 		return !net.ParseIP(ip).IsLoopback()
@@ -333,6 +339,12 @@ func TestCreateEndpoints(t *testing.T) {
 // So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
 // is considered a remote service from localhost:9000 perspective.
 func TestGetLocalPeer(t *testing.T) {
+	tempGlobalMinioPort := globalMinioPort
+	defer func() {
+		globalMinioPort = tempGlobalMinioPort
+	}()
+	globalMinioPort = "9000"
+
 	testCases := []struct {
 		endpointArgs   []string
 		expectedResult string

cmd/erasure-bucket.go

@@ -134,32 +134,8 @@ func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi B
 	return bucketInfo, nil
 }
 
-// Dangling buckets should be handled appropriately, in this following situation
-// we actually have quorum error to be `nil` but we have some disks where
-// the bucket delete returned `errVolumeNotEmpty` but this is not correct
-// can only happen if there are dangling objects in a bucket. Under such
-// a situation we simply attempt a full delete of the bucket including
-// the dangling objects. All of this happens under a lock and there
-// is no way a user can create buckets and sneak in objects into namespace,
-// so it is safer to do.
-func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs []error, bucket string) {
-	for index, err := range dErrs {
-		if err == errVolumeNotEmpty {
-			// Attempt to delete bucket again.
-			if derr := storageDisks[index].DeleteVol(ctx, bucket, false); derr == errVolumeNotEmpty {
-				_ = storageDisks[index].Delete(ctx, bucket, "", true)
-
-				_ = storageDisks[index].DeleteVol(ctx, bucket, false)
-
-				// Cleanup all the previously incomplete multiparts.
-				_ = storageDisks[index].Delete(ctx, minioMetaMultipartBucket, bucket, true)
-			}
-		}
-	}
-}
-
 // DeleteBucket - deletes a bucket.
-func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	// Collect if all disks report volume not found.
 	defer NSUpdated(bucket, slashSeparator)
 
@@ -171,13 +147,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 		index := index
 		g.Go(func() error {
 			if storageDisks[index] != nil {
-				if err := storageDisks[index].DeleteVol(ctx, bucket, forceDelete); err != nil {
-					return err
-				}
-				if err := storageDisks[index].Delete(ctx, minioMetaMultipartBucket, bucket, true); err != errFileNotFound {
-					return err
-				}
-				return nil
+				return storageDisks[index].DeleteVol(ctx, bucket, opts.Force)
 			}
 			return errDiskNotFound
 		}, index)
@@ -186,7 +156,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 	// Wait for all the delete vols to finish.
 	dErrs := g.Wait()
 
-	if forceDelete {
+	if opts.Force {
 		for _, err := range dErrs {
 			if err != nil {
 				undoDeleteBucket(storageDisks, bucket)
@@ -199,20 +169,20 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 
 	writeQuorum := getWriteQuorum(len(storageDisks))
 	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum)
-	if err == errErasureWriteQuorum {
+	if err == errErasureWriteQuorum && !opts.NoRecreate {
 		undoDeleteBucket(storageDisks, bucket)
 	}
 	if err != nil {
 		return toObjectErr(err, bucket)
 	}
 
-	// If we reduce quorum to nil, means we have deleted buckets properly
-	// on some servers in quorum, we should look for volumeNotEmpty errors
-	// and delete those buckets as well.
-	//
-	// let this call succeed, even if client cancels the context
-	// this is to ensure that we don't leave any stale content
-	deleteDanglingBucket(context.Background(), storageDisks, dErrs, bucket)
+	// At this point we have `err == nil` but some errors might be `errVolumeNotEmpty`
+	// we should proceed to attempt a force delete of such buckets.
+	for index, err := range dErrs {
+		if err == errVolumeNotEmpty && storageDisks[index] != nil {
+			storageDisks[index].RenameFile(ctx, bucket, "", minioMetaTmpDeletedBucket, mustGetUUID())
+		}
+	}
 
 	return nil
 }

cmd/erasure-common.go

@@ -22,16 +22,6 @@ import (
 	"sync"
 )
 
-func (er erasureObjects) getLocalDisks() (localDisks []StorageAPI) {
-	disks := er.getDisks()
-	for _, disk := range disks {
-		if disk != nil && disk.IsLocal() {
-			localDisks = append(localDisks, disk)
-		}
-	}
-	return localDisks
-}
-
 func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	// Based on the random shuffling return back randomized disks.

cmd/erasure-healing-common.go

@@ -40,7 +40,11 @@ func commonTime(modTimes []time.Time, dataDirs []string) (modTime time.Time, dat
 	}
 
 	for _, dataDir := range dataDirs {
-		if dataDir == "" {
+		if dataDir == errorDir {
+			continue
+		}
+		if dataDir == delMarkerDir {
+			dataDirOccurenceMap[delMarkerDir]++
 			continue
 		}
 		dataDirOccurenceMap[dataDir]++
@@ -97,6 +101,11 @@ func listObjectModtimes(partsMetadata []FileInfo, errs []error) (modTimes []time
 	return modTimes
 }
 
+const (
+	errorDir     = "error-dir"
+	delMarkerDir = ""
+)
+
 // Notes:
 // There are 5 possible states a disk could be in,
 // 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
@@ -130,6 +139,7 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error)
 	dataDirs := make([]string, len(partsMetadata))
 	for idx, fi := range partsMetadata {
 		if errs[idx] != nil {
+			dataDirs[idx] = errorDir
 			continue
 		}
 		dataDirs[idx] = fi.DataDir
@@ -151,9 +161,9 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error)
 }
 
 // Returns the latest updated FileInfo files and error in case of failure.
-func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []error) (FileInfo, error) {
+func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []error, quorum int) (FileInfo, error) {
 	// There should be atleast half correct entries, if not return failure
-	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, len(partsMetadata)/2); reducedErr != nil {
+	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, quorum); reducedErr != nil {
 		return FileInfo{}, reducedErr
 	}
 
@@ -182,7 +192,8 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 			count++
 		}
 	}
-	if count < len(partsMetadata)/2 {
+
+	if count < quorum {
 		return FileInfo{}, errErasureReadQuorum
 	}
 

cmd/erasure-healing-common_test.go

@@ -190,7 +190,7 @@ func TestListOnlineDisks(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, errs, getReadQuorum(len(disks)))
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -365,7 +365,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, errs, getReadQuorum(len(disks)))
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -421,7 +421,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 
 			}
 			partsMetadata, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			_, err = getLatestFileInfo(ctx, partsMetadata, errs)
+			_, err = getLatestFileInfo(ctx, partsMetadata, errs, len(disks)/2)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}

cmd/erasure-healing.go

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"io"
 	"sync"
-	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
@@ -54,7 +53,7 @@ func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts mad
 }
 
 // Heal bucket - create buckets on disks where it does not exist.
-func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []string, bucket string, writeQuorum int,
+func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []Endpoint, bucket string, writeQuorum int,
 	opts madmin.HealOpts) (res madmin.HealResultItem, err error) {
 
 	// Initialize sync waitgroup.
@@ -115,7 +114,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 	for i := range beforeState {
 		res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    beforeState[i],
 		})
 	}
@@ -125,7 +124,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 		for i := range beforeState {
 			res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[i],
+				Endpoint: storageEndpoints[i].String(),
 				State:    madmin.DriveStateOk,
 			})
 		}
@@ -160,7 +159,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 	for i := range afterState {
 		res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    afterState[i],
 		})
 	}
@@ -204,7 +203,7 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
-func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, quorumModTime time.Time) bool {
+func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool {
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
@@ -222,7 +221,16 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, quorumModTime t
 				return true
 			}
 		}
-		if !quorumModTime.Equal(meta.ModTime) {
+		if !latestMeta.MetadataEquals(meta) {
+			return true
+		}
+		if !latestMeta.TransitionInfoEquals(meta) {
+			return true
+		}
+		if !latestMeta.ReplicationInfoEquals(meta) {
+			return true
+		}
+		if !latestMeta.ModTime.Equal(meta.ModTime) {
 			return true
 		}
 		if meta.XLV1 {
@@ -267,7 +275,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
 
-	if _, err = getLatestFileInfo(ctx, partsMetadata, errs); err != nil {
+	if _, err = getLatestFileInfo(ctx, partsMetadata, errs, er.defaultParityCount); err != nil {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
 	}
 
@@ -295,6 +303,13 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	availableDisks, dataErrs := disksWithAllParts(ctx, storageDisks, partsMetadata,
 		errs, bucket, object, scanMode)
 
+	// Latest FileInfo for reference. If a valid metadata is not
+	// present, it is as good as object not found.
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, dataDir, result.DataBlocks)
+	if err != nil {
+		return result, toObjectErr(err, bucket, object, versionID)
+	}
+
 	// Loop to find number of disks with valid data, per-drive
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
@@ -325,29 +340,29 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], modTime) {
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[i],
+				Endpoint: storageEndpoints[i].String(),
 				State:    driveState,
 			})
 			result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[i],
+				Endpoint: storageEndpoints[i].String(),
 				State:    driveState,
 			})
 			continue
 		}
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    driveState,
 		})
 		result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    driveState,
 		})
 	}
@@ -379,20 +394,12 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		return result, nil
 	}
 
-	// Latest FileInfo for reference. If a valid metadata is not
-	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, dataDir, result.DataBlocks)
-	if err != nil {
-		return result, toObjectErr(err, bucket, object, versionID)
-	}
-
 	cleanFileInfo := func(fi FileInfo) FileInfo {
 		// Returns a copy of the 'fi' with checksums and parts nil'ed.
 		nfi := fi
-		nfi.Erasure.Index = 0
-		nfi.Erasure.Checksums = nil
-		if fi.IsRemote() {
-			nfi.Parts = nil
+		if !fi.IsRemote() {
+			nfi.Erasure.Index = 0
+			nfi.Erasure.Checksums = nil
 		}
 		return nfi
 	}
@@ -421,20 +428,20 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	}
 
 	var inlineBuffers []*bytes.Buffer
-	if len(latestMeta.Parts) <= 1 && latestMeta.Size < smallFileThreshold {
+	if latestMeta.InlineData() {
 		inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 	}
 
+	// Reorder so that we have data disks first and parity disks next.
+	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
+	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
+	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
+	copyPartsMetadata = shufflePartsMetadata(copyPartsMetadata, latestMeta.Erasure.Distribution)
+
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
 		result.DataBlocks = latestMeta.Erasure.DataBlocks
 		result.ParityBlocks = latestMeta.Erasure.ParityBlocks
 
-		// Reorder so that we have data disks first and parity disks next.
-		latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
-		outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
-		partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
-		copyPartsMetadata = shufflePartsMetadata(copyPartsMetadata, latestMeta.Erasure.Distribution)
-
 		// Heal each part. erasureHealFile() will write the healed
 		// part to .minio/tmp/uuid/ which needs to be renamed later to
 		// the final location.
@@ -472,7 +479,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 				partPath := pathJoin(tmpID, dstDataDir, fmt.Sprintf("part.%d", partNumber))
 				if len(inlineBuffers) > 0 {
-					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(latestMeta.Size)))
+					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(latestMeta.Size)+32))
 					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 				} else {
 					writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, partPath,
@@ -531,22 +538,21 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		if disk == OfflineDisk {
 			continue
 		}
-
 		// record the index of the updated disks
 		partsMetadata[i].Erasure.Index = i + 1
 
-		// dataDir should be empty when
-		// - transitionStatus is complete and not in restored state
-		if partsMetadata[i].IsRemote() {
-			partsMetadata[i].DataDir = ""
-		}
-
 		// Attempt a rename now from healed data to final location.
 		if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
 			logger.LogIf(ctx, err)
 			return result, toObjectErr(err, bucket, object)
 		}
 
+		// Remove any remaining parts from outdated disks from before transition.
+		if partsMetadata[i].IsRemote() {
+			rmDataDir := partsMetadata[i].DataDir
+			disk.DeleteVol(ctx, pathJoin(bucket, encodeDirObject(object), rmDataDir), true)
+		}
+
 		for i, v := range result.Before.Drives {
 			if v.Endpoint == disk.String() {
 				result.After.Drives[i].State = madmin.DriveStateOk
@@ -603,7 +609,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 
 	// Prepare object creation in all disks
 	for i, err := range errs {
-		drive := storageEndpoints[i]
+		drive := storageEndpoints[i].String()
 		switch err {
 		case nil:
 			hr.Before.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateOk}
@@ -644,7 +650,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 
 // Populates default heal result item entries with possible values when we are returning prematurely.
 // This is to ensure that in any circumstance we are not returning empty arrays with wrong values.
-func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []string, errs []error, bucket, object, versionID string) madmin.HealResultItem {
+func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []Endpoint, errs []error, bucket, object, versionID string) madmin.HealResultItem {
 	// Initialize heal result object
 	result := madmin.HealResultItem{
 		Type:       madmin.HealItemObject,
@@ -667,12 +673,12 @@ func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageA
 		if disk == nil {
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[index],
+				Endpoint: storageEndpoints[index].String(),
 				State:    madmin.DriveStateOffline,
 			})
 			result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[index],
+				Endpoint: storageEndpoints[index].String(),
 				State:    madmin.DriveStateOffline,
 			})
 			continue
@@ -686,12 +692,12 @@ func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageA
 		}
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[index],
+			Endpoint: storageEndpoints[index].String(),
 			State:    driveState,
 		})
 		result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[index],
+			Endpoint: storageEndpoints[index].String(),
 			State:    driveState,
 		})
 	}
@@ -879,6 +885,12 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.
 func (er erasureObjects) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (hr madmin.HealResultItem, err error) {
+	defer func() {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			err = nil
+		}
+	}()
+
 	// Create context that also contains information about the object and bucket.
 	// The top level handler might not have this information.
 	reqInfo := logger.GetReqInfo(ctx)

cmd/erasure-healing_test.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path"
 	"reflect"
+	"runtime"
 	"testing"
 	"time"
 
@@ -145,6 +146,205 @@ func TestHealing(t *testing.T) {
 	}
 }
 
+func TestHealingDanglingObject(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip()
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//defer removeRoots(fsDirs)
+
+	// Everything is fine, should return nil
+	objLayer, disks, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 128*1024)
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Enable versioning.
+	globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`))
+
+	_, err = objLayer.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{
+		Versioned: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, fsDir := range fsDirs[:4] {
+		if err = os.Chmod(fsDir, 0400); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Create delete marker under quorum.
+	objInfo, err := objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, fsDir := range fsDirs[:4] {
+		if err = os.Chmod(fsDir, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fileInfoPreHeal.NumVersions != 1 {
+		t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions)
+	}
+
+	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
+		func(bucket, object, vid string) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+			return err
+		}); err != nil {
+		t.Fatal(err)
+	}
+
+	fileInfoPostHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fileInfoPostHeal.NumVersions != 2 {
+		t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions)
+	}
+
+	if objInfo.DeleteMarker {
+		if _, err = objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{
+			Versioned: true,
+			VersionID: objInfo.VersionID,
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	for _, fsDir := range fsDirs[:4] {
+		if err = os.Chmod(fsDir, 0400); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	rd := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "")
+	_, err = objLayer.PutObject(ctx, bucket, object, rd, ObjectOptions{
+		Versioned: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, fsDir := range fsDirs[:4] {
+		if err = os.Chmod(fsDir, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fileInfoPreHeal.NumVersions != 1 {
+		t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions)
+	}
+
+	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
+		func(bucket, object, vid string) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+			return err
+		}); err != nil {
+		t.Fatal(err)
+	}
+
+	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fileInfoPostHeal.NumVersions != 2 {
+		t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions)
+	}
+
+	rd = mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "")
+	objInfo, err = objLayer.PutObject(ctx, bucket, object, rd, ObjectOptions{
+		Versioned: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, fsDir := range fsDirs[:4] {
+		if err = os.Chmod(fsDir, 0400); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Create delete marker under quorum.
+	_, err = objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{
+		Versioned: true,
+		VersionID: objInfo.VersionID,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, fsDir := range fsDirs[:4] {
+		if err = os.Chmod(fsDir, 0755); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fileInfoPreHeal.NumVersions != 3 {
+		t.Fatalf("Expected versions 3, got %d", fileInfoPreHeal.NumVersions)
+	}
+
+	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
+		func(bucket, object, vid string) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+			return err
+		}); err != nil {
+		t.Fatal(err)
+	}
+
+	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fileInfoPostHeal.NumVersions != 2 {
+		t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions)
+	}
+}
+
 func TestHealObjectCorrupted(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -166,8 +366,8 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	bucket := "bucket"
-	object := "object"
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
@@ -216,12 +416,12 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs, er.defaultParityCount)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 
@@ -241,7 +441,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	nfi, err := getLatestFileInfo(ctx, fileInfos, errs, er.defaultParityCount)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -267,7 +467,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	nfi, err = getLatestFileInfo(ctx, fileInfos, errs, er.defaultParityCount)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -365,7 +565,7 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 

cmd/erasure-metadata.go

@@ -24,11 +24,10 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
-	"github.com/minio/minio/internal/crypto"
-	"github.com/minio/minio/internal/etag"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -143,13 +142,6 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 	// Extract etag from metadata.
 	objInfo.ETag = extractETag(fi.Metadata)
 
-	// Verify if Etag is parseable, if yes
-	// then check if its multipart etag.
-	et, e := etag.Parse(objInfo.ETag)
-	if e == nil {
-		objInfo.Multipart = et.IsMultipart()
-	}
-
 	// Add user tags to the object info
 	tags := fi.Metadata[xhttp.AmzObjectTagging]
 	if len(tags) != 0 {
@@ -157,11 +149,11 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 	}
 
 	// Add replication status to the object info
-	objInfo.ReplicationStatus = replication.StatusType(fi.Metadata[xhttp.AmzBucketReplicationStatus])
-	if fi.Deleted {
-		objInfo.ReplicationStatus = replication.StatusType(fi.DeleteMarkerReplicationStatus)
-	}
+	objInfo.ReplicationStatusInternal = fi.ReplicationState.ReplicationStatusInternal
+	objInfo.VersionPurgeStatusInternal = fi.ReplicationState.VersionPurgeStatusInternal
+	objInfo.ReplicationStatus = fi.ReplicationState.CompositeReplicationStatus()
 
+	objInfo.VersionPurgeStatus = fi.ReplicationState.CompositeVersionPurgeStatus()
 	objInfo.TransitionedObject = TransitionedObject{
 		Name:        fi.TransitionedObjName,
 		VersionID:   fi.TransitionVersionID,
@@ -176,14 +168,6 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(fi.Metadata)
 
-	// Set multipart for encryption properly if
-	// not set already.
-	if !objInfo.Multipart {
-		if _, ok := crypto.IsEncrypted(objInfo.UserDefined); ok {
-			objInfo.Multipart = crypto.IsMultiPart(objInfo.UserDefined)
-		}
-	}
-
 	// All the parts per object.
 	objInfo.Parts = fi.Parts
 
@@ -194,7 +178,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 		objInfo.StorageClass = globalMinioDefaultStorageClass
 	}
 
-	objInfo.VersionPurgeStatus = fi.VersionPurgeStatus
+	objInfo.VersionPurgeStatus = fi.VersionPurgeStatus()
 	// set restore status for transitioned object
 	restoreHdr, ok := fi.Metadata[xhttp.AmzRestore]
 	if ok {
@@ -207,6 +191,41 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 	return objInfo
 }
 
+// TransitionInfoEquals returns true if transition related information are equal, false otherwise.
+func (fi FileInfo) TransitionInfoEquals(ofi FileInfo) bool {
+	switch {
+	case fi.TransitionStatus != ofi.TransitionStatus,
+		fi.TransitionTier != ofi.TransitionTier,
+		fi.TransitionedObjName != ofi.TransitionedObjName,
+		fi.TransitionVersionID != ofi.TransitionVersionID:
+		return false
+	}
+	return true
+}
+
+// MetadataEquals returns true if FileInfos Metadata maps are equal, false otherwise.
+func (fi FileInfo) MetadataEquals(ofi FileInfo) bool {
+	if len(fi.Metadata) != len(ofi.Metadata) {
+		return false
+	}
+	for k, v := range fi.Metadata {
+		if ov, ok := ofi.Metadata[k]; !ok || ov != v {
+			return false
+		}
+	}
+	return true
+}
+
+// ReplicationInfoEquals returns true if server-side replication related fields are equal, false otherwise.
+func (fi FileInfo) ReplicationInfoEquals(ofi FileInfo) bool {
+	switch {
+	case fi.MarkDeleted != ofi.MarkDeleted,
+		!fi.ReplicationState.Equal(ofi.ReplicationState):
+		return false
+	}
+	return true
+}
+
 // objectPartIndex - returns the index of matching object part number.
 func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
 	for i, part := range parts {
@@ -287,9 +306,12 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 
 			// Server-side replication fields
 			h.Write([]byte(fmt.Sprintf("%v", meta.MarkDeleted)))
-			h.Write([]byte(meta.DeleteMarkerReplicationStatus))
-			h.Write([]byte(meta.VersionPurgeStatus))
-			h.Write([]byte(meta.Metadata[xhttp.AmzBucketReplicationStatus]))
+			h.Write([]byte(meta.Metadata[string(meta.ReplicationState.ReplicaStatus)]))
+			h.Write([]byte(meta.Metadata[meta.ReplicationState.ReplicationTimeStamp.Format(http.TimeFormat)]))
+			h.Write([]byte(meta.Metadata[meta.ReplicationState.ReplicaTimeStamp.Format(http.TimeFormat)]))
+			h.Write([]byte(meta.Metadata[meta.ReplicationState.ReplicationStatusInternal]))
+			h.Write([]byte(meta.Metadata[meta.ReplicationState.VersionPurgeStatusInternal]))
+
 			metaHashes[i] = hex.EncodeToString(h.Sum(nil))
 			h.Reset()
 		}
@@ -366,7 +388,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 // writeQuorum is the min required disks to write data.
 func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
 	// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
-	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs)
+	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs, defaultParityCount)
 	if err != nil {
 		return 0, 0, err
 	}
@@ -429,3 +451,54 @@ func (fi *FileInfo) TierFreeVersion() bool {
 	_, ok := fi.Metadata[ReservedMetadataPrefixLower+tierFVMarker]
 	return ok
 }
+
+// VersionPurgeStatus returns overall version purge status for this object version across targets
+func (fi *FileInfo) VersionPurgeStatus() VersionPurgeStatusType {
+	return fi.ReplicationState.CompositeVersionPurgeStatus()
+}
+
+// DeleteMarkerReplicationStatus returns overall replication status for this delete marker version across targets
+func (fi *FileInfo) DeleteMarkerReplicationStatus() replication.StatusType {
+	if fi.Deleted {
+		return fi.ReplicationState.CompositeReplicationStatus()
+	}
+	return replication.StatusType("")
+}
+
+// GetInternalReplicationState is a wrapper method to fetch internal replication state from the map m
+func GetInternalReplicationState(m map[string][]byte) ReplicationState {
+	m1 := make(map[string]string, len(m))
+	for k, v := range m {
+		m1[k] = string(v)
+	}
+	return getInternalReplicationState(m1)
+}
+
+// getInternalReplicationState fetches internal replication state from the map m
+func getInternalReplicationState(m map[string]string) ReplicationState {
+	d := ReplicationState{
+		ResetStatusesMap: make(map[string]string),
+	}
+	for k, v := range m {
+		switch {
+		case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp):
+			tm, _ := time.Parse(http.TimeFormat, v)
+			d.ReplicationTimeStamp = tm
+		case equals(k, ReservedMetadataPrefixLower+ReplicaTimestamp):
+			tm, _ := time.Parse(http.TimeFormat, v)
+			d.ReplicaTimeStamp = tm
+		case equals(k, ReservedMetadataPrefixLower+ReplicaStatus):
+			d.ReplicaStatus = replication.StatusType(v)
+		case equals(k, ReservedMetadataPrefixLower+ReplicationStatus):
+			d.ReplicationStatusInternal = v
+			d.Targets = replicationStatusesMap(v)
+		case equals(k, VersionPurgeStatusKey):
+			d.VersionPurgeStatusInternal = v
+			d.PurgeTargets = versionPurgeStatusesMap(v)
+		case strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset):
+			arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset))
+			d.ResetStatusesMap[arn] = v
+		}
+	}
+	return d
+}

cmd/erasure-metadata_test.go

@@ -215,3 +215,62 @@ func TestFindFileInfoInQuorum(t *testing.T) {
 		})
 	}
 }
+
+func TestTransitionInfoEquals(t *testing.T) {
+	inputs := []struct {
+		tier            string
+		remoteObjName   string
+		remoteVersionID string
+		status          string
+	}{
+		{
+			tier:            "S3TIER-1",
+			remoteObjName:   mustGetUUID(),
+			remoteVersionID: mustGetUUID(),
+			status:          "complete",
+		},
+		{
+			tier:            "S3TIER-2",
+			remoteObjName:   mustGetUUID(),
+			remoteVersionID: mustGetUUID(),
+			status:          "complete",
+		},
+	}
+
+	var i uint
+	for i = 0; i < 8; i++ {
+		fi := FileInfo{
+			TransitionTier:      inputs[0].tier,
+			TransitionedObjName: inputs[0].remoteObjName,
+			TransitionVersionID: inputs[0].remoteVersionID,
+			TransitionStatus:    inputs[0].status,
+		}
+		ofi := fi
+		if i&(1<<0) != 0 {
+			ofi.TransitionTier = inputs[1].tier
+		}
+		if i&(1<<1) != 0 {
+			ofi.TransitionedObjName = inputs[1].remoteObjName
+		}
+		if i&(1<<2) != 0 {
+			ofi.TransitionVersionID = inputs[1].remoteVersionID
+		}
+		actual := fi.TransitionInfoEquals(ofi)
+		if i == 0 && !actual {
+			t.Fatalf("Test %d: Expected FileInfo's transition info to be equal: fi %v ofi %v", i, fi, ofi)
+		}
+		if i != 0 && actual {
+			t.Fatalf("Test %d: Expected FileInfo's transition info to be inequal: fi %v ofi %v", i, fi, ofi)
+		}
+	}
+	fi := FileInfo{
+		TransitionTier:      inputs[0].tier,
+		TransitionedObjName: inputs[0].remoteObjName,
+		TransitionVersionID: inputs[0].remoteVersionID,
+		TransitionStatus:    inputs[0].status,
+	}
+	ofi := FileInfo{}
+	if fi.TransitionInfoEquals(ofi) {
+		t.Fatalf("Expected to be inequal: fi %v ofi %v", fi, ofi)
+	}
+}

cmd/erasure-multipart.go

@@ -783,9 +783,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	// Calculate s3 compatible md5sum for complete multipart.
-	s3MD5 := getCompleteMultipartMD5(parts)
-
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 
 	storageDisks := er.getDisks()
@@ -882,9 +879,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Save successfully calculated md5sum.
-	fi.Metadata["etag"] = s3MD5
-	if opts.UserDefined["etag"] != "" { // preserve ETag if set
-		fi.Metadata["etag"] = opts.UserDefined["etag"]
+	fi.Metadata["etag"] = opts.UserDefined["etag"]
+	if fi.Metadata["etag"] == "" {
+		fi.Metadata["etag"] = getCompleteMultipartMD5(parts)
 	}
 
 	// Save the consolidated actual size.
@@ -953,6 +950,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	// we are adding a new version to this object under the namespace lock, so this is the latest version.
+	fi.IsLatest = true
+
 	// Success, return object info.
 	return fi.ToObjectInfo(bucket, object), nil
 }

cmd/erasure-object.go

@@ -134,6 +134,8 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
+	// we are adding a new version to this object under the namespace lock, so this is the latest version.
+	fi.IsLatest = true
 	return fi.ToObjectInfo(srcBucket, srcObject), nil
 }
 
@@ -311,19 +313,26 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 			// - attempt a heal to successfully heal them for future calls.
 			if written == partLength {
 				var scan madmin.HealScanMode
-				if errors.Is(err, errFileNotFound) {
+				switch {
+				case errors.Is(err, errFileNotFound):
 					scan = madmin.HealNormalScan
-					logger.Info("Healing required, attempting to heal missing shards for %s", pathJoin(bucket, object, fi.VersionID))
-				} else if errors.Is(err, errFileCorrupt) {
+					logger.Info("Healing required, triggering async heal missing shards for %s", pathJoin(bucket, object, fi.VersionID))
+				case errors.Is(err, errFileCorrupt):
 					scan = madmin.HealDeepScan
-					logger.Info("Healing required, attempting to heal bitrot for %s", pathJoin(bucket, object, fi.VersionID))
+					logger.Info("Healing required, triggering async heal bitrot for %s", pathJoin(bucket, object, fi.VersionID))
 				}
-				if scan == madmin.HealNormalScan || scan == madmin.HealDeepScan {
+				switch scan {
+				case madmin.HealNormalScan, madmin.HealDeepScan:
 					healOnce.Do(func() {
 						if _, healing := er.getOnlineDisksWithHealing(); !healing {
 							go healObject(bucket, object, fi.VersionID, scan)
 						}
 					})
+					// Healing is triggered and we have written
+					// successfully the content to client for
+					// the specific part, we should `nil` this error
+					// and proceed forward, instead of throwing errors.
+					err = nil
 				}
 			}
 			if err != nil {
@@ -441,7 +450,7 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 
 	}
 	objInfo = fi.ToObjectInfo(bucket, object)
-	if !fi.VersionPurgeStatus.Empty() {
+	if opts.VersionID != "" && !fi.VersionPurgeStatus().Empty() {
 		// Make sure to return object info to provide extra information.
 		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
 	}
@@ -457,6 +466,35 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 	return objInfo, nil
 }
 
+// getObjectInfoAndQuroum - wrapper for reading object metadata and constructs ObjectInfo, additionally returns write quorum for the object.
+func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, wquorum int, err error) {
+	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
+	if err != nil {
+		return objInfo, getWriteQuorum(len(er.getDisks())), toObjectErr(err, bucket, object)
+	}
+
+	wquorum = fi.Erasure.DataBlocks
+	if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
+		wquorum++
+	}
+
+	objInfo = fi.ToObjectInfo(bucket, object)
+	if !fi.VersionPurgeStatus().Empty() && opts.VersionID != "" {
+		// Make sure to return object info to provide extra information.
+		return objInfo, wquorum, toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+
+	if fi.Deleted {
+		if opts.VersionID == "" || opts.DeleteMarker {
+			return objInfo, wquorum, toObjectErr(errFileNotFound, bucket, object)
+		}
+		// Make sure to return object info to provide extra information.
+		return objInfo, wquorum, toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+
+	return objInfo, wquorum, nil
+}
+
 func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, errs []error) {
 	// Undo rename object on disks where RenameFile succeeded.
 
@@ -969,8 +1007,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		break
 	}
 
+	fi.ReplicationState = opts.PutReplicationState()
 	online = countOnlineDisks(onlineDisks)
 
+	// we are adding a new version to this object under the namespace lock, so this is the latest version.
+	fi.IsLatest = true
+
 	return fi.ToObjectInfo(bucket, object), nil
 }
 
@@ -1059,11 +1101,10 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 			}
 			if opts.Versioned || opts.VersionSuspended {
 				versions[i] = FileInfo{
-					Name:                          objects[i].ObjectName,
-					ModTime:                       modTime,
-					Deleted:                       true, // delete marker
-					DeleteMarkerReplicationStatus: objects[i].DeleteMarkerReplicationStatus,
-					VersionPurgeStatus:            objects[i].VersionPurgeStatus,
+					Name:             objects[i].ObjectName,
+					ModTime:          modTime,
+					Deleted:          true, // delete marker
+					ReplicationState: objects[i].ReplicationState(),
 				}
 				versions[i].SetTierFreeVersionID(mustGetUUID())
 				if opts.Versioned {
@@ -1073,10 +1114,9 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 			}
 		}
 		versions[i] = FileInfo{
-			Name:                          objects[i].ObjectName,
-			VersionID:                     objects[i].VersionID,
-			DeleteMarkerReplicationStatus: objects[i].DeleteMarkerReplicationStatus,
-			VersionPurgeStatus:            objects[i].VersionPurgeStatus,
+			Name:             objects[i].ObjectName,
+			VersionID:        objects[i].VersionID,
+			ReplicationState: objects[i].ReplicationState(),
 		}
 		versions[i].SetTierFreeVersionID(mustGetUUID())
 	}
@@ -1127,19 +1167,17 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 
 		if versions[objIndex].Deleted {
 			dobjects[objIndex] = DeletedObject{
-				DeleteMarker:                  versions[objIndex].Deleted,
-				DeleteMarkerVersionID:         versions[objIndex].VersionID,
-				DeleteMarkerMTime:             DeleteMarkerMTime{versions[objIndex].ModTime},
-				DeleteMarkerReplicationStatus: versions[objIndex].DeleteMarkerReplicationStatus,
-				ObjectName:                    versions[objIndex].Name,
-				VersionPurgeStatus:            versions[objIndex].VersionPurgeStatus,
+				DeleteMarker:          versions[objIndex].Deleted,
+				DeleteMarkerVersionID: versions[objIndex].VersionID,
+				DeleteMarkerMTime:     DeleteMarkerMTime{versions[objIndex].ModTime},
+				ObjectName:            versions[objIndex].Name,
+				ReplicationState:      versions[objIndex].ReplicationState,
 			}
 		} else {
 			dobjects[objIndex] = DeletedObject{
-				ObjectName:                    versions[objIndex].Name,
-				VersionID:                     versions[objIndex].VersionID,
-				VersionPurgeStatus:            versions[objIndex].VersionPurgeStatus,
-				DeleteMarkerReplicationStatus: versions[objIndex].DeleteMarkerReplicationStatus,
+				ObjectName:       versions[objIndex].Name,
+				VersionID:        versions[objIndex].VersionID,
+				ReplicationState: versions[objIndex].ReplicationState,
 			}
 		}
 	}
@@ -1191,9 +1229,40 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
 	}
 
+	var lc *lifecycle.Lifecycle
+	if opts.Expiration.Expire {
+		// Check if the current bucket has a configured lifecycle policy
+		lc, _ = globalLifecycleSys.Get(bucket)
+	}
+
+	// expiration attempted on a bucket with no lifecycle
+	// rules shall be rejected.
+	if lc == nil && opts.Expiration.Expire {
+		if opts.VersionID != "" {
+			return objInfo, VersionNotFound{
+				Bucket:    bucket,
+				Object:    object,
+				VersionID: opts.VersionID,
+			}
+		}
+		return objInfo, ObjectNotFound{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+
+	// Acquire a write lock before deleting the object.
+	lk := er.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+	if err != nil {
+		return ObjectInfo{}, err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
+
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
-	goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts)
+	goi, writeQuorum, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
 		switch gerr.(type) {
 		case InsufficientReadQuorum:
@@ -1207,19 +1276,34 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
+	if opts.Expiration.Expire {
+		action := evalActionFromLifecycle(ctx, *lc, goi, false)
+		var isErr bool
+		switch action {
+		case lifecycle.NoneAction:
+			isErr = true
+		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+			isErr = true
+		}
+		if isErr {
+			if goi.VersionID != "" {
+				return goi, VersionNotFound{
+					Bucket:    bucket,
+					Object:    object,
+					VersionID: goi.VersionID,
+				}
+			}
+			return goi, ObjectNotFound{
+				Bucket: bucket,
+				Object: object,
+			}
+		}
+	}
+
 	defer NSUpdated(bucket, object)
 
-	// Acquire a write lock before deleting the object.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
-	if err != nil {
-		return ObjectInfo{}, err
-	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
-
 	storageDisks := er.getDisks()
-	writeQuorum := len(storageDisks)/2 + 1
+
 	var markDelete bool
 	// Determine whether to mark object deleted for replication
 	if goi.VersionID != "" {
@@ -1231,13 +1315,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	if opts.VersionID != "" {
 		// case where replica version needs to be deleted on target cluster
-		if versionFound && opts.DeleteMarkerReplicationStatus == replication.Replica.String() {
+		if versionFound && opts.DeleteMarkerReplicationStatus() == replication.Replica {
 			markDelete = false
 		}
-		if opts.VersionPurgeStatus.Empty() && opts.DeleteMarkerReplicationStatus == "" {
+		if opts.VersionPurgeStatus().Empty() && opts.DeleteMarkerReplicationStatus().Empty() {
 			markDelete = false
 		}
-		if opts.VersionPurgeStatus == Complete {
+		if opts.VersionPurgeStatus() == Complete {
 			markDelete = false
 		}
 		// determine if the version represents an object delete
@@ -1255,14 +1339,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	if markDelete {
 		if opts.Versioned || opts.VersionSuspended {
 			fi := FileInfo{
-				Name:                          object,
-				Deleted:                       deleteMarker,
-				MarkDeleted:                   markDelete,
-				ModTime:                       modTime,
-				DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus,
-				VersionPurgeStatus:            opts.VersionPurgeStatus,
-				TransitionStatus:              opts.Transition.Status,
-				ExpireRestored:                opts.Transition.ExpireRestored,
+				Name:             object,
+				Deleted:          deleteMarker,
+				MarkDeleted:      markDelete,
+				ModTime:          modTime,
+				ReplicationState: opts.DeleteReplication,
+				TransitionStatus: opts.Transition.Status,
+				ExpireRestored:   opts.Transition.ExpireRestored,
 			}
 			fi.SetTierFreeVersionID(fvID)
 			if opts.Versioned {
@@ -1284,15 +1367,14 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	// Delete the object version on all disks.
 	dfi := FileInfo{
-		Name:                          object,
-		VersionID:                     opts.VersionID,
-		MarkDeleted:                   markDelete,
-		Deleted:                       deleteMarker,
-		ModTime:                       modTime,
-		DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus,
-		VersionPurgeStatus:            opts.VersionPurgeStatus,
-		TransitionStatus:              opts.Transition.Status,
-		ExpireRestored:                opts.Transition.ExpireRestored,
+		Name:             object,
+		VersionID:        opts.VersionID,
+		MarkDeleted:      markDelete,
+		Deleted:          deleteMarker,
+		ModTime:          modTime,
+		ReplicationState: opts.DeleteReplication,
+		TransitionStatus: opts.Transition.Status,
+		ExpireRestored:   opts.Transition.ExpireRestored,
 	}
 	dfi.SetTierFreeVersionID(fvID)
 	if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, dfi, opts.DeleteMarker); err != nil {
@@ -1308,11 +1390,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 
 	return ObjectInfo{
-		Bucket:             bucket,
-		Name:               object,
-		VersionID:          opts.VersionID,
-		VersionPurgeStatus: opts.VersionPurgeStatus,
-		ReplicationStatus:  replication.StatusType(opts.DeleteMarkerReplicationStatus),
+		Bucket:                     bucket,
+		Name:                       object,
+		VersionID:                  opts.VersionID,
+		VersionPurgeStatusInternal: opts.DeleteReplication.VersionPurgeStatusInternal,
+		ReplicationStatusInternal:  opts.DeleteReplication.ReplicationStatusInternal,
 	}, nil
 }
 
@@ -1476,7 +1558,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	if err != nil {
 		return err
 	}
-	defer NSUpdated(bucket, object)
 
 	// Acquire write lock before starting to transition the object.
 	lk := er.NewNSLock(bucket, object)
@@ -1506,6 +1587,8 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	if fi.TransitionStatus == lifecycle.TransitionComplete {
 		return nil
 	}
+	defer NSUpdated(bucket, object)
+
 	if fi.XLV1 {
 		if _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{NoLock: true}); err != nil {
 			return err
@@ -1544,10 +1627,17 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	eventName := event.ObjectTransitionComplete
 
 	storageDisks := er.getDisks()
-	writeQuorum := len(storageDisks)/2 + 1
+	// we now know the number of blocks this object needs for data and parity.
+	// writeQuorum is dataBlocks + 1
+	writeQuorum := fi.Erasure.DataBlocks
+	if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
+		writeQuorum++
+	}
+
 	if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, false); err != nil {
 		eventName = event.ObjectTransitionFailed
 	}
+
 	for _, disk := range storageDisks {
 		if disk != nil && disk.IsOnline() {
 			continue

cmd/erasure-server-pool.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -150,33 +151,45 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 // GetRawData will return all files with a given raw path to the callback.
 // Errors are ignored, only errors from the callback are returned.
 // For now only direct file paths are supported.
-func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error {
+	found := 0
 	for _, s := range z.serverPools {
 		for _, disks := range s.erasureDisks {
 			for i, disk := range disks {
 				if disk == OfflineDisk {
 					continue
 				}
-				si, err := disk.StatInfoFile(ctx, volume, file)
+				stats, err := disk.StatInfoFile(ctx, volume, file, true)
 				if err != nil {
 					continue
 				}
-				r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size)
-				if err != nil {
-					continue
-				}
-				defer r.Close()
 				did, err := disk.GetDiskID()
 				if err != nil {
 					did = fmt.Sprintf("disk-%d", i)
 				}
-				err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime)
-				if err != nil {
-					return err
+				for _, si := range stats {
+					found++
+					var r io.ReadCloser
+					if !si.Dir {
+						r, err = disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size)
+						if err != nil {
+							continue
+						}
+					} else {
+						r = io.NopCloser(bytes.NewBuffer([]byte{}))
+					}
+					err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si)
+					r.Close()
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}
 	}
+	if found == 0 {
+		return errFileNotFound
+	}
 	return nil
 }
 
@@ -339,6 +352,22 @@ func (z *erasureServerPools) getPoolIdxExisting(ctx context.Context, bucket, obj
 	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{})
 }
 
+func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
+	idx, err = z.getPoolIdxExistingNoLock(ctx, bucket, object)
+	if err != nil && !isErrObjectNotFound(err) {
+		return idx, err
+	}
+
+	if isErrObjectNotFound(err) {
+		idx = z.getAvailablePoolIdx(ctx, bucket, object, size)
+		if idx < 0 {
+			return -1, toObjectErr(errDiskFull)
+		}
+	}
+
+	return idx, nil
+}
+
 // getPoolIdx returns the found previous object and its corresponding pool idx,
 // if none are found falls back to most available space pool.
 func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
@@ -456,7 +485,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) (StorageInfo, []er
 	return storageInfo, errs
 }
 
-func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- madmin.DataUsageInfo) error {
+func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
 	// Updates must be closed before we return.
 	defer close(updates)
 
@@ -474,7 +503,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 	}
 
 	if len(allBuckets) == 0 {
-		updates <- madmin.DataUsageInfo{} // no buckets found update data usage to reflect latest state
+		updates <- DataUsageInfo{} // no buckets found update data usage to reflect latest state
 		return nil
 	}
 
@@ -501,7 +530,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 					}
 				}()
 				// Start scanner. Blocks until done.
-				err := erObj.nsScanner(ctx, allBuckets, bf, updates)
+				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					mu.Lock()
@@ -603,6 +632,10 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 	// Return the first encountered error
 	for _, err := range errs {
 		if err != nil {
+			if _, ok := err.(BucketExists); !ok {
+				// Delete created buckets, ignoring errors.
+				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: true, NoRecreate: true})
+			}
 			return err
 		}
 	}
@@ -614,7 +647,7 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
 	}
 
-	if err := meta.Save(ctx, z); err != nil {
+	if err := meta.Save(context.Background(), z); err != nil {
 		return toObjectErr(err, bucket)
 	}
 
@@ -791,7 +824,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
 	if !opts.NoLock {
-		ns := z.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object, "newMultipartObject.lck"))
+		ns := z.NewNSLock(bucket, object)
 		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
 			return ObjectInfo{}, err
@@ -801,7 +834,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		opts.NoLock = true
 	}
 
-	idx, err := z.getPoolIdx(ctx, bucket, object, data.Size())
+	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, data.Size())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -867,7 +900,11 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 	defer multiDeleteLock.Unlock(lkctx.Cancel)
 
 	if z.SinglePool() {
-		return z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
+		deleteObjects, dErrs := z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
+		for i := range deleteObjects {
+			deleteObjects[i].ObjectName = decodeDirObject(deleteObjects[i].ObjectName)
+		}
+		return deleteObjects, dErrs
 	}
 
 	// Fetch location of up to 10 objects concurrently.
@@ -921,6 +958,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 					if derr != nil {
 						derrs[orgIndexes[i]] = derr
 					}
+					deletedObjects[i].ObjectName = decodeDirObject(deletedObjects[i].ObjectName)
 					dobjects[orgIndexes[i]] = deletedObjects[i]
 				}
 				mu.Unlock()
@@ -939,7 +977,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
 	if !dstOpts.NoLock {
-		ns := z.NewNSLock(minioMetaMultipartBucket, pathJoin(dstBucket, dstObject, "newMultipartObject.lck"))
+		ns := z.NewNSLock(dstBucket, dstObject)
 		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
 			return ObjectInfo{}, err
@@ -949,7 +987,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		dstOpts.NoLock = true
 	}
 
-	poolIdx, err := z.getPoolIdx(ctx, dstBucket, dstObject, srcInfo.Size)
+	poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
@@ -982,6 +1020,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		Versioned:            dstOpts.Versioned,
 		VersionID:            dstOpts.VersionID,
 		MTime:                dstOpts.MTime,
+		NoLock:               true,
 	}
 
 	return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
@@ -1162,14 +1201,6 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
 
-	ns := z.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object, "newMultipartObject.lck"))
-	lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return "", err
-	}
-	ctx = lkctx.Context()
-	defer ns.Unlock(lkctx.Cancel)
-
 	for idx, pool := range z.serverPools {
 		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
 		if err != nil {
@@ -1183,6 +1214,8 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 		}
 	}
 
+	// any parallel writes on the object will block for this poolIdx
+	// to return since this holds a read lock on the namespace.
 	idx, err := z.getPoolIdx(ctx, bucket, object, -1)
 	if err != nil {
 		return "", err
@@ -1400,14 +1433,14 @@ func (z *erasureServerPools) IsTaggingSupported() bool {
 // DeleteBucket - deletes a bucket on all serverPools simultaneously,
 // even if one of the serverPools fail to delete buckets, we proceed to
 // undo a successful operation.
-func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	g := errgroup.WithNErrs(len(z.serverPools))
 
 	// Delete buckets in parallel across all serverPools.
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
-			return z.serverPools[index].DeleteBucket(ctx, bucket, forceDelete)
+			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}
 
@@ -1417,14 +1450,15 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, fo
 	// buckets operation by creating all the buckets again.
 	for _, err := range errs {
 		if err != nil {
-			if !z.SinglePool() {
-				undoDeleteBucketServerPools(ctx, bucket, z.serverPools, errs)
+			if !z.SinglePool() && !opts.NoRecreate {
+				undoDeleteBucketServerPools(context.Background(), bucket, z.serverPools, errs)
 			}
 			return err
 		}
 	}
 
-	deleteBucketMetadata(ctx, z, bucket)
+	// Purge the entire bucket metadata entirely.
+	z.renameAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
 
 	// Success.
 	return nil
@@ -1569,51 +1603,82 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 		return err
 	}
 
-	if opts.WalkVersions {
-		go func() {
-			defer close(results)
-
-			var marker, versionIDMarker string
-			for {
-				loi, err := z.ListObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, "", 1000)
-				if err != nil {
-					break
-				}
-
-				for _, obj := range loi.Objects {
-					results <- obj
-				}
-
-				if !loi.IsTruncated {
-					break
-				}
-
-				marker = loi.NextMarker
-				versionIDMarker = loi.NextVersionIDMarker
-			}
-		}()
-		return nil
-	}
-
+	ctx, cancel := context.WithCancel(ctx)
 	go func() {
+		defer cancel()
 		defer close(results)
 
-		var marker string
-		for {
-			loi, err := z.ListObjects(ctx, bucket, prefix, marker, "", 1000)
-			if err != nil {
-				break
-			}
+		for _, erasureSet := range z.serverPools {
+			var wg sync.WaitGroup
+			for _, set := range erasureSet.sets {
+				set := set
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
 
-			for _, obj := range loi.Objects {
-				results <- obj
-			}
+					disks, _ := set.getOnlineDisksWithHealing()
+					if len(disks) == 0 {
+						cancel()
+						return
+					}
 
-			if !loi.IsTruncated {
-				break
-			}
+					loadEntry := func(entry metaCacheEntry) {
+						if entry.isDir() {
+							return
+						}
 
-			marker = loi.NextMarker
+						fivs, err := entry.fileInfoVersions(bucket)
+						if err != nil {
+							cancel()
+							return
+						}
+
+						for _, version := range fivs.Versions {
+							results <- version.ToObjectInfo(bucket, version.Name)
+						}
+					}
+
+					// How to resolve partial results.
+					resolver := metadataResolutionParams{
+						dirQuorum: 1,
+						objQuorum: 1,
+						bucket:    bucket,
+					}
+
+					path := baseDirFromPrefix(prefix)
+					if path == "" {
+						path = prefix
+					}
+
+					lopts := listPathRawOptions{
+						disks:          disks,
+						bucket:         bucket,
+						path:           path,
+						recursive:      true,
+						forwardTo:      "",
+						minDisks:       1,
+						reportNotFound: false,
+						agreed:         loadEntry,
+						partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+							entry, ok := entries.resolve(&resolver)
+							if !ok {
+								// check if we can get one entry atleast
+								// proceed to heal nonetheless.
+								entry, _ = entries.firstFound()
+							}
+
+							loadEntry(*entry)
+						},
+						finished: nil,
+					}
+
+					if err := listPathRaw(ctx, lopts); err != nil {
+						logger.LogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
+						return
+					}
+				}()
+			}
+			wg.Wait()
 		}
 	}()
 
@@ -1640,8 +1705,8 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 
 					disks, _ := set.getOnlineDisksWithHealing()
 					if len(disks) == 0 {
-						errCh <- errors.New("HealObjects: No non-healing disks found")
 						cancel()
+						errCh <- errors.New("HealObjects: No non-healing disks found")
 						return
 					}
 
@@ -1668,15 +1733,18 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 						}
 						fivs, err := entry.fileInfoVersions(bucket)
 						if err != nil {
-							errCh <- err
-							cancel()
+							if err := healObject(bucket, entry.name, ""); err != nil {
+								cancel()
+								errCh <- err
+								return
+							}
 							return
 						}
-						waitForLowHTTPReq(globalHealConfig.IOCount, globalHealConfig.Sleep)
+
 						for _, version := range fivs.Versions {
 							if err := healObject(bucket, version.Name, version.VersionID); err != nil {
-								errCh <- err
 								cancel()
+								errCh <- err
 								return
 							}
 						}
@@ -1705,15 +1773,20 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 						agreed:         healEntry,
 						partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
 							entry, ok := entries.resolve(&resolver)
-							if ok {
-								healEntry(*entry)
+							if !ok {
+								// check if we can get one entry atleast
+								// proceed to heal nonetheless.
+								entry, _ = entries.firstFound()
 							}
+
+							healEntry(*entry)
 						},
 						finished: nil,
 					}
+
 					if err := listPathRaw(ctx, lopts); err != nil {
-						errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
 						cancel()
+						errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
 						return
 					}
 				}()
@@ -1731,9 +1804,6 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 		result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
 		result.Object = decodeDirObject(result.Object)
 		if err != nil {
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				continue
-			}
 			return result, err
 		}
 		return result, nil

cmd/erasure-sets.go

@@ -41,14 +41,11 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/console"
-	"github.com/minio/pkg/env"
 )
 
 // setsDsyncLockers is encapsulated type for Close()
 type setsDsyncLockers [][]dsync.NetLocker
 
-const envMinioDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL"
-
 // erasureSets implements ObjectLayer combining a static list of erasure coded
 // object sets. NOTE: There is no dynamic scaling allowed or intended in
 // current design.
@@ -257,7 +254,6 @@ func (s *erasureSets) connectDisks() {
 				s.erasureDisks[setIndex][diskIndex] = disk
 			}
 			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
-			s.endpointStrings[setIndex*s.setDriveCount+diskIndex] = disk.String()
 			setsJustConnected[setIndex] = true
 			s.erasureDisksMu.Unlock()
 		}(endpoint)
@@ -314,14 +310,14 @@ func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string
 	}
 }
 
-func (s *erasureSets) GetEndpoints(setIndex int) func() []string {
-	return func() []string {
+func (s *erasureSets) GetEndpoints(setIndex int) func() []Endpoint {
+	return func() []Endpoint {
 		s.erasureDisksMu.RLock()
 		defer s.erasureDisksMu.RUnlock()
 
-		eps := make([]string, s.setDriveCount)
+		eps := make([]Endpoint, s.setDriveCount)
 		for i := 0; i < s.setDriveCount; i++ {
-			eps[i] = s.endpointStrings[setIndex*s.setDriveCount+i]
+			eps[i] = s.endpoints[setIndex*s.setDriveCount+i]
 		}
 		return eps
 	}
@@ -348,6 +344,9 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	setDriveCount := len(format.Erasure.Sets[0])
 
 	endpointStrings := make([]string, len(endpoints))
+	for i, endpoint := range endpoints {
+		endpointStrings[i] = endpoint.String()
+	}
 
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
@@ -416,6 +415,9 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 			if err != nil {
 				continue
 			}
+			if m != i || n != j {
+				return nil, fmt.Errorf("found a disk in an unexpected location, pool: %d, found (set=%d, disk=%d) expected (set=%d, disk=%d): %s(%s)", poolIdx, m, n, i, j, disk, diskID)
+			}
 			disk.SetDiskLoc(s.poolIndex, m, n)
 			s.endpointStrings[m*setDriveCount+n] = disk.String()
 			s.erasureDisks[m][n] = disk
@@ -437,19 +439,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 		}
 	}
 
-	// cleanup ".trash/" folder every 5m minutes with sufficient sleep cycles, between each
-	// deletes a dynamic sleeper is used with a factor of 10 ratio with max delay between
-	// deletes to be 2 seconds.
-	deletedObjectsCleanupInterval, err := time.ParseDuration(env.Get(envMinioDeleteCleanupInterval, "5m"))
-	if err != nil {
-		return nil, err
-	}
-
 	// start cleanup stale uploads go-routine.
-	go s.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
+	go s.cleanupStaleUploads(ctx)
 
 	// start cleanup of deleted objects.
-	go s.cleanupDeletedObjects(ctx, deletedObjectsCleanupInterval)
+	go s.cleanupDeletedObjects(ctx)
 
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval)
@@ -457,8 +451,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	return s, nil
 }
 
-func (s *erasureSets) cleanupDeletedObjects(ctx context.Context, cleanupInterval time.Duration) {
-	timer := time.NewTimer(cleanupInterval)
+// cleanup ".trash/" folder every 5m minutes with sufficient sleep cycles, between each
+// deletes a dynamic sleeper is used with a factor of 10 ratio with max delay between
+// deletes to be 2 seconds.
+func (s *erasureSets) cleanupDeletedObjects(ctx context.Context) {
+	timer := time.NewTimer(globalAPIConfig.getDeleteCleanupInterval())
 	defer timer.Stop()
 
 	for {
@@ -467,7 +464,7 @@ func (s *erasureSets) cleanupDeletedObjects(ctx context.Context, cleanupInterval
 			return
 		case <-timer.C:
 			// Reset for the next interval
-			timer.Reset(cleanupInterval)
+			timer.Reset(globalAPIConfig.getDeleteCleanupInterval())
 
 			for _, set := range s.sets {
 				set.cleanupDeletedObjects(ctx)
@@ -476,8 +473,8 @@ func (s *erasureSets) cleanupDeletedObjects(ctx context.Context, cleanupInterval
 	}
 }
 
-func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
-	timer := time.NewTimer(cleanupInterval)
+func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
+	timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval())
 	defer timer.Stop()
 
 	for {
@@ -486,10 +483,10 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval,
 			return
 		case <-timer.C:
 			// Reset for the next interval
-			timer.Reset(cleanupInterval)
+			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
 
 			for _, set := range s.sets {
-				set.cleanupStaleUploads(ctx, expiry)
+				set.cleanupStaleUploads(ctx, globalAPIConfig.getStaleUploadsExpiry())
 			}
 		}
 	}
@@ -525,10 +522,15 @@ func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjec
 
 	object = decodeDirObject(object)
 
+	var disksEndpoints []string
+	for _, endpoint := range set.getEndpoints() {
+		disksEndpoints = append(disksEndpoints, endpoint.String())
+	}
+
 	op := auditObjectOp{
 		Pool:  set.poolIndex + 1,
 		Set:   set.setIndex + 1,
-		Disks: set.getEndpoints(),
+		Disks: disksEndpoints,
 	}
 
 	var objectErasureSetTag *auditObjectErasureMap
@@ -802,14 +804,14 @@ func (s *erasureSets) IsTaggingSupported() bool {
 // DeleteBucket - deletes a bucket on all sets simultaneously,
 // even if one of the sets fail to delete buckets, we proceed to
 // undo a successful operation.
-func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	g := errgroup.WithNErrs(len(s.sets))
 
 	// Delete buckets in parallel across all sets.
 	for index := range s.sets {
 		index := index
 		g.Go(func() error {
-			return s.sets[index].DeleteBucket(ctx, bucket, forceDelete)
+			return s.sets[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}
 
@@ -817,7 +819,7 @@ func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, forceDele
 	// For any failure, we attempt undo all the delete buckets operation
 	// by creating buckets again on all sets which were successfully deleted.
 	for _, err := range errs {
-		if err != nil {
+		if err != nil && !opts.NoRecreate {
 			undoDeleteBucketSets(ctx, bucket, s.sets, errs)
 			return err
 		}
@@ -893,12 +895,16 @@ func (s *erasureSets) GetObjectInfo(ctx context.Context, bucket, object string,
 }
 
 func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix string) error {
+	var wg sync.WaitGroup
+	wg.Add(len(s.sets))
 	for _, s := range s.sets {
-		_, err := s.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
-		if err != nil {
-			return err
-		}
+		go func(s *erasureObjects) {
+			defer wg.Done()
+			// This is a force delete, no reason to throw errors.
+			s.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true})
+		}(s)
 	}
+	wg.Wait()
 	return nil
 }
 
@@ -1292,9 +1298,11 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			if s.erasureDisks[m][n] != nil {
 				s.erasureDisks[m][n].Close()
 			}
-			storageDisks[index].SetDiskLoc(s.poolIndex, m, n)
-			s.erasureDisks[m][n] = storageDisks[index]
-			s.endpointStrings[m*s.setDriveCount+n] = storageDisks[index].String()
+
+			if storageDisks[index] != nil {
+				storageDisks[index].SetDiskLoc(s.poolIndex, m, n)
+				s.erasureDisks[m][n] = storageDisks[index]
+			}
 		}
 
 		// Replace reference format with what was loaded from disks.

cmd/erasure.go

@@ -57,7 +57,7 @@ type erasureObjects struct {
 
 	// getEndpoints returns list of endpoint strings belonging this set.
 	// some may be local and some remote.
-	getEndpoints func() []string
+	getEndpoints func() []Endpoint
 
 	// Locker mutex map.
 	nsMutex *nsLockMap
@@ -164,7 +164,7 @@ func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDi
 }
 
 // getDisksInfo - fetch disks info across all other storage API.
-func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error) {
+func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.Disk, errs []error) {
 	disksInfo = make([]madmin.Disk, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
@@ -175,7 +175,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 				logger.LogIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]))
 				disksInfo[index] = madmin.Disk{
 					State:    diskErrToDriveState(errDiskNotFound),
-					Endpoint: endpoints[index],
+					Endpoint: endpoints[index].String(),
 				}
 				// Storage disk is empty, perhaps ignored disk or not available.
 				return errDiskNotFound
@@ -222,7 +222,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 }
 
 // Get an aggregated storage info across all disks.
-func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) {
+func getStorageInfo(disks []StorageAPI, endpoints []Endpoint) (StorageInfo, []error) {
 	disksInfo, errs := getDisksInfo(disks, endpoints)
 
 	// Sort so that the first element is the smallest.
@@ -245,14 +245,20 @@ func (er erasureObjects) StorageInfo(ctx context.Context) (StorageInfo, []error)
 
 // LocalStorageInfo - returns underlying local storage statistics.
 func (er erasureObjects) LocalStorageInfo(ctx context.Context) (StorageInfo, []error) {
-	disks := er.getLocalDisks()
-	endpoints := make([]string, len(disks))
-	for i, disk := range disks {
-		if disk != nil {
-			endpoints[i] = disk.String()
+	disks := er.getDisks()
+	endpoints := er.getEndpoints()
+
+	var localDisks []StorageAPI
+	var localEndpoints []Endpoint
+
+	for i, endpoint := range endpoints {
+		if endpoint.IsLocal {
+			localDisks = append(localDisks, disks[i])
+			localEndpoints = append(localEndpoints, endpoint)
 		}
 	}
-	return getStorageInfo(disks, endpoints)
+
+	return getStorageInfo(localDisks, localEndpoints)
 }
 
 func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, healing bool) {
@@ -326,7 +332,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
-func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
+func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache) error {
 	if len(buckets) == 0 {
 		return nil
 	}
@@ -419,7 +425,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 			case v, ok := <-bucketResults:
 				if !ok {
 					// Save final state...
-					cache.Info.NextCycle++
+					cache.Info.NextCycle = wantCycle
 					cache.Info.LastUpdate = time.Now()
 					logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
 					updates <- cache
@@ -461,12 +467,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				cache.Info.BloomFilter = bloom
 				cache.Info.SkipHealing = healing
 				cache.Disks = allDiskIDs
+				cache.Info.NextCycle = wantCycle
 				if cache.Info.Name != bucket.Name {
 					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
 					cache.Info = dataUsageCacheInfo{
 						Name:       bucket.Name,
 						LastUpdate: time.Time{},
-						NextCycle:  0,
+						NextCycle:  wantCycle,
 					}
 				}
 				// Collect updates.

cmd/etcd.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/minio/minio/internal/logger"
 	etcd "go.etcd.io/etcd/client/v3"
 )
 
@@ -47,6 +48,7 @@ func saveKeyEtcdWithTTL(ctx context.Context, client *etcd.Client, key string, da
 		return etcdErrToErr(err, client.Endpoints())
 	}
 	_, err = client.Put(timeoutCtx, key, string(data), etcd.WithLease(lease.ID))
+	logger.LogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -57,6 +59,7 @@ func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []by
 		return saveKeyEtcdWithTTL(ctx, client, key, data, opts[0].ttl)
 	}
 	_, err := client.Put(timeoutCtx, key, string(data))
+	logger.LogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -65,6 +68,7 @@ func deleteKeyEtcd(ctx context.Context, client *etcd.Client, key string) error {
 	defer cancel()
 
 	_, err := client.Delete(timeoutCtx, key)
+	logger.LogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -73,6 +77,7 @@ func readKeyEtcd(ctx context.Context, client *etcd.Client, key string) ([]byte,
 	defer cancel()
 	resp, err := client.Get(timeoutCtx, key)
 	if err != nil {
+		logger.LogIf(ctx, err)
 		return nil, etcdErrToErr(err, client.Endpoints())
 	}
 	if resp.Count == 0 {

cmd/format-erasure.go

@@ -705,7 +705,7 @@ func saveUnformattedFormat(ctx context.Context, storageDisks []StorageAPI, forma
 		if format == nil {
 			continue
 		}
-		if storageDisks[index] != nil && storageDisks[index].IsOnline() {
+		if storageDisks[index] != nil {
 			if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
 				return err
 			}

cmd/fs-v1-helpers.go

@@ -268,6 +268,7 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 	// Stat to get the size of the file at path.
 	st, err := fr.Stat()
 	if err != nil {
+		fr.Close()
 		err = osErrToFileErr(err)
 		if err != errFileNotFound {
 			logger.LogIf(ctx, err)
@@ -277,6 +278,7 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 
 	// Verify if its not a regular file, since subsequent Seek is undefined.
 	if !st.Mode().IsRegular() {
+		fr.Close()
 		return nil, 0, errIsNotRegular
 	}
 
@@ -284,6 +286,7 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 	if offset > 0 {
 		_, err = fr.Seek(offset, io.SeekStart)
 		if err != nil {
+			fr.Close()
 			logger.LogIf(ctx, err)
 			return nil, 0, err
 		}

cmd/fs-v1-metadata.go

@@ -29,8 +29,6 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/minio/internal/crypto"
-	"github.com/minio/minio/internal/etag"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/lock"
 	"github.com/minio/minio/internal/logger"
@@ -170,11 +168,6 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 
 	objInfo.ETag = extractETag(m.Meta)
 
-	et, e := etag.Parse(objInfo.ETag)
-	if e == nil {
-		objInfo.Multipart = et.IsMultipart()
-	}
-
 	objInfo.ContentType = m.Meta["content-type"]
 	objInfo.ContentEncoding = m.Meta["content-encoding"]
 	if storageClass, ok := m.Meta[xhttp.AmzStorageClass]; ok {
@@ -184,6 +177,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	}
 	var (
 		t time.Time
+		e error
 	)
 	if exp, ok := m.Meta["expires"]; ok {
 		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
@@ -200,14 +194,6 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(m.Meta)
 
-	// Set multipart for encryption properly if
-	// not set already.
-	if !objInfo.Multipart {
-		if _, ok := crypto.IsEncrypted(objInfo.UserDefined); ok {
-			objInfo.Multipart = crypto.IsMultiPart(objInfo.UserDefined)
-		}
-	}
-
 	// All the parts per object.
 	objInfo.Parts = m.Parts
 

cmd/fs-v1-multipart.go

@@ -564,9 +564,6 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, toObjectErr(err, bucket, object)
 	}
 
-	// Calculate s3 compatible md5sum for complete multipart.
-	s3MD5 := getCompleteMultipartMD5(parts)
-
 	// ensure that part ETag is canonicalized to strip off extraneous quotes
 	for i := range parts {
 		parts[i].ETag = canonicalizeETag(parts[i].ETag)
@@ -755,7 +752,12 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	if fsMeta.Meta == nil {
 		fsMeta.Meta = make(map[string]string)
 	}
-	fsMeta.Meta["etag"] = s3MD5
+
+	fsMeta.Meta["etag"] = opts.UserDefined["etag"]
+	if fsMeta.Meta["etag"] == "" {
+		fsMeta.Meta["etag"] = getCompleteMultipartMD5(parts)
+	}
+
 	// Save consolidated actual size.
 	fsMeta.Meta[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 	if _, err = fsMeta.WriteTo(metaFile); err != nil {
@@ -845,8 +847,8 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
-func (fs *FSObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
-	timer := time.NewTimer(cleanupInterval)
+func (fs *FSObjects) cleanupStaleUploads(ctx context.Context) {
+	timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval())
 	defer timer.Stop()
 
 	for {
@@ -855,7 +857,9 @@ func (fs *FSObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, e
 			return
 		case <-timer.C:
 			// Reset for the next interval
-			timer.Reset(cleanupInterval)
+			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
+
+			expiry := globalAPIConfig.getStaleUploadsExpiry()
 
 			now := time.Now()
 			entries, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket))

cmd/fs-v1-multipart_test.go

@@ -26,6 +26,8 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/minio/minio/internal/config/api"
 )
 
 // Tests cleanup multipart uploads for filesystem backend.
@@ -49,11 +51,23 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 		t.Fatal("Unexpected err: ", err)
 	}
 
+	globalAPIConfig.init(api.Config{
+		ListQuorum:                  "optimal",
+		StaleUploadsExpiry:          time.Millisecond,
+		StaleUploadsCleanupInterval: time.Millisecond,
+	}, obj.SetDriveCounts())
+
+	defer func() {
+		globalAPIConfig.init(api.Config{
+			ListQuorum: "optimal",
+		}, obj.SetDriveCounts())
+	}()
+
 	var cleanupWg sync.WaitGroup
 	cleanupWg.Add(1)
 	go func() {
 		defer cleanupWg.Done()
-		fs.cleanupStaleUploads(ctx, time.Millisecond, 0)
+		fs.cleanupStaleUploads(ctx)
 	}()
 
 	// Wait for 100ms such that - we have given enough time for

cmd/fs-v1.go

@@ -177,7 +177,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	// or cause changes on backend format.
 	fs.fsFormatRlk = rlk
 
-	go fs.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
+	go fs.cleanupStaleUploads(ctx)
 	go intDataUpdateTracker.start(ctx, fsPath)
 
 	// Return successfully initialized object layer.
@@ -240,7 +240,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 }
 
 // NSScanner returns data usage stats of the current FS deployment
-func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- madmin.DataUsageInfo) error {
+func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
 	defer close(updates)
 	// Load bucket totals
 	var totalCache dataUsageCache
@@ -258,6 +258,12 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 		updates <- totalCache.dui(dataUsageRoot, buckets)
 		return nil
 	}
+	for i, b := range buckets {
+		if isReservedOrInvalidBucket(b.Name, false) {
+			// Delete bucket...
+			buckets = append(buckets[:i], buckets[i+1:]...)
+		}
+	}
 
 	totalCache.Info.BloomFilter = bf.bytes()
 
@@ -282,6 +288,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 			bCache.Info.Name = b.Name
 		}
 		bCache.Info.BloomFilter = totalCache.Info.BloomFilter
+		bCache.Info.NextCycle = wantCycle
 		upds := make(chan dataUsageEntry, 1)
 		var wg sync.WaitGroup
 		wg.Add(1)
@@ -290,7 +297,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 			for update := range upds {
 				totalCache.replace(b.Name, dataUsageRoot, update)
 				if intDataUpdateTracker.debug {
-					logger.Info(color.Green("NSScanner:")+" Got update:", len(totalCache.Cache))
+					logger.Info(color.Green("NSScanner:")+" Got update: %v", len(totalCache.Cache))
 				}
 				cloned := totalCache.clone()
 				updates <- cloned.dui(dataUsageRoot, buckets)
@@ -386,7 +393,9 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 		}
 
 		oi := fsMeta.ToObjectInfo(bucket, object, fi)
-		sz := item.applyActions(ctx, fs, actionMeta{oi: oi}, &sizeSummary{})
+		atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
+		atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
+		sz := item.applyActions(ctx, fs, oi, &sizeSummary{})
 		if sz >= 0 {
 			return sizeSummary{totalSize: sz, versions: 1}, nil
 		}
@@ -579,7 +588,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 
 // DeleteBucket - delete a bucket and all the metadata associated
 // with the bucket including pending multipart, object metadata.
-func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
@@ -592,7 +601,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, forceDelet
 		return toObjectErr(err, bucket)
 	}
 
-	if !forceDelete {
+	if !opts.Force {
 		// Attempt to delete regular bucket.
 		if err = fsRemoveDir(ctx, bucketDir); err != nil {
 			return toObjectErr(err, bucket)
@@ -854,6 +863,15 @@ func (fs *FSObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object s
 		return fsMeta.ToObjectInfo(bucket, object, fi), nil
 	}
 
+	if !globalCLIContext.StrictS3Compat {
+		// Stat the file to get file size.
+		fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
+		if err != nil {
+			return oi, err
+		}
+		return fsMeta.ToObjectInfo(bucket, object, fi), nil
+	}
+
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
 	// Read `fs.json` to perhaps contend with
 	// parallel Put() operations.
@@ -1492,7 +1510,7 @@ func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, obje
 // GetRawData returns raw file data to the callback.
 // Errors are ignored, only errors from the callback are returned.
 // For now only direct file paths are supported.
-func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
 	f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
 	if err != nil {
 		return nil
@@ -1502,5 +1520,5 @@ func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn fun
 	if err != nil || st.IsDir() {
 		return nil
 	}
-	return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime())
+	return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime(), st.IsDir())
 }

cmd/fs-v1_test.go

@@ -224,16 +224,16 @@ func TestFSDeleteBucket(t *testing.T) {
 	}
 
 	// Test with an invalid bucket name
-	if err = fs.DeleteBucket(GlobalContext, "fo", false); !isSameType(err, BucketNotFound{}) {
+	if err = fs.DeleteBucket(GlobalContext, "fo", DeleteBucketOptions{}); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test with an inexistant bucket
-	if err = fs.DeleteBucket(GlobalContext, "foobucket", false); !isSameType(err, BucketNotFound{}) {
+	if err = fs.DeleteBucket(GlobalContext, "foobucket", DeleteBucketOptions{}); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with a valid case
-	if err = fs.DeleteBucket(GlobalContext, bucketName, false); err != nil {
+	if err = fs.DeleteBucket(GlobalContext, bucketName, DeleteBucketOptions{}); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -241,7 +241,7 @@ func TestFSDeleteBucket(t *testing.T) {
 
 	// Delete bucket should get error disk not found.
 	os.RemoveAll(disk)
-	if err = fs.DeleteBucket(GlobalContext, bucketName, false); err != nil {
+	if err = fs.DeleteBucket(GlobalContext, bucketName, DeleteBucketOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
 		}

cmd/gateway-common.go

@@ -333,6 +333,10 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = PartTooSmall{}
 	}
 
+	switch minioErr.StatusCode {
+	case http.StatusMethodNotAllowed:
+		err = toObjectErr(errMethodNotAllowed, bucket, object)
+	}
 	return err
 }
 
@@ -384,7 +388,7 @@ func gatewayHandleEnvVars() {
 
 // shouldMeterRequest checks whether incoming request should be added to prometheus gateway metrics
 func shouldMeterRequest(req *http.Request) bool {
-	return !(guessIsBrowserReq(req) || guessIsHealthCheckReq(req) || guessIsMetricsReq(req))
+	return req.URL != nil && !strings.HasPrefix(req.URL.Path, minioReservedBucketPath+slashSeparator)
 }
 
 // MetricsTransport is a custom wrapper around Transport to track metrics

cmd/gateway-main.go

@@ -154,8 +154,6 @@ func ValidateGatewayArguments(serverAddr, endpointAddr string) error {
 
 // StartGateway - handler for 'minio gateway <name>'.
 func StartGateway(ctx *cli.Context, gw Gateway) {
-	defer globalDNSCache.Stop()
-
 	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 
 	go handleSignals()
@@ -218,8 +216,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Set when gateway is enabled
 	globalIsGateway = true
 
-	enableConfigOps := false
-
 	// TODO: We need to move this code with globalConfigSys.Init()
 	// for now keep it here such that "s3" gateway layer initializes
 	// itself properly when KMS is set.
@@ -245,7 +241,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Enable IAM admin APIs if etcd is enabled, if not just enable basic
 	// operations such as profiling, server info etc.
-	registerAdminRouter(router, enableConfigOps)
+	registerAdminRouter(router, false)
 
 	// Add healthcheck router
 	registerHealthCheckRouter(router)
@@ -264,13 +260,21 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	httpServer := xhttp.NewServer([]string{globalMinioAddr},
-		criticalErrorHandler{corsHandler(router)}, getCert)
+	listeners := ctx.Int("listeners")
+	if listeners == 0 {
+		listeners = 1
+	}
+	addrs := make([]string, 0, listeners)
+	for i := 0; i < listeners; i++ {
+		addrs = append(addrs, globalMinioAddr)
+	}
+
+	httpServer := xhttp.NewServer(addrs, criticalErrorHandler{corsHandler(router)}, getCert)
 	httpServer.BaseContext = func(listener net.Listener) context.Context {
 		return GlobalContext
 	}
 	go func() {
-		globalHTTPServerErrorCh <- httpServer.Start()
+		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
 	}()
 
 	globalObjLayerMutex.Lock()
@@ -282,7 +286,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		SecretKey: globalActiveCred.SecretKey,
 	})
 	if err != nil {
-		globalHTTPServer.Shutdown()
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 	newObject = NewGatewayLayerWithLocker(newObject)
@@ -303,10 +306,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(globalNotificationSys.Init(GlobalContext, buckets, newObject), "Unable to initialize notification system")
 	}
 
-	// Initialize users credentials and policies in background.
-	globalIAMSys.InitStore(newObject)
-
-	go globalIAMSys.Init(GlobalContext, newObject)
+	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient)
 
 	if globalCacheConfig.Enabled {
 		// initialize the new disk cache objects.
@@ -345,18 +345,14 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	if globalBrowserEnabled {
-		consoleSrv, err := initConsoleServer()
+		globalConsoleSrv, err = initConsoleServer()
 		if err != nil {
 			logger.FatalIf(err, "Unable to initialize console service")
 		}
 
 		go func() {
-			<-globalOSSignalCh
-			consoleSrv.Shutdown()
+			logger.FatalIf(globalConsoleSrv.Serve(), "Unable to initialize console server")
 		}()
-
-		consoleSrv.Serve()
-	} else {
-		<-globalOSSignalCh
 	}
+	<-globalOSSignalCh
 }

cmd/gateway-unsupported.go

@@ -48,7 +48,7 @@ func (a GatewayUnsupported) LocalStorageInfo(ctx context.Context) (StorageInfo,
 }
 
 // NSScanner - scanner is not implemented for gateway
-func (a GatewayUnsupported) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- madmin.DataUsageInfo) error {
+func (a GatewayUnsupported) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
 	return NotImplemented{}
 }

cmd/gateway/azure/gateway-azure.go

@@ -38,7 +38,7 @@ import (
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
 	"github.com/Azure/azure-storage-blob-go/azblob"
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	"github.com/minio/madmin-go"
 	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
@@ -623,8 +623,8 @@ func (a *azureObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketI
 }
 
 // DeleteBucket - delete a container on azure, uses Azure equivalent `ContainerURL.Delete`.
-func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
-	if !forceDelete {
+func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
+	if !opts.Force {
 		// Check if the container is empty before deleting it.
 		result, err := a.ListObjects(ctx, bucket, "", "", "", 1)
 		if err != nil {

cmd/gateway/gcs/gateway-gcs.go

@@ -470,7 +470,7 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
 }
 
 // DeleteBucket delete a bucket on GCS.
-func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	itObject := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
 		Delimiter: minio.SlashSeparator,
 		Versions:  false,

cmd/gateway/hdfs/gateway-hdfs.go

@@ -305,11 +305,11 @@ func (n *hdfsObjects) hdfsPathJoin(args ...string) string {
 	return minio.PathJoin(append([]string{n.subPath, hdfsSeparator}, args...)...)
 }
 
-func (n *hdfsObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (n *hdfsObjects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	if !hdfsIsValidBucketName(bucket) {
 		return minio.BucketNameInvalid{Bucket: bucket}
 	}
-	if forceDelete {
+	if opts.Force {
 		return hdfsToObjectErr(ctx, n.clnt.RemoveAll(n.hdfsPathJoin(bucket)), bucket)
 	}
 	return hdfsToObjectErr(ctx, n.clnt.Remove(n.hdfsPathJoin(bucket)), bucket)

cmd/gateway/s3/gateway-s3-chain.go

@@ -0,0 +1,103 @@
+/*
+ * MinIO Object Storage (c) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// A Chain will search for a provider which returns credentials
+// and cache that provider until Retrieve is called again.
+//
+// The Chain provides a way of chaining multiple providers together
+// which will pick the first available using priority order of the
+// Providers in the list.
+//
+// If none of the Providers retrieve valid credentials Value, ChainProvider's
+// Retrieve() will return the no credentials value.
+//
+// If a Provider is found which returns valid credentials Value ChainProvider
+// will cache that Provider for all calls to IsExpired(), until Retrieve is
+// called again after IsExpired() is true.
+//
+//     creds := credentials.NewChainCredentials(
+//         []credentials.Provider{
+//             &credentials.EnvAWSS3{},
+//             &credentials.EnvMinio{},
+//         })
+//
+//     // Usage of ChainCredentials.
+//     mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1")
+//     if err != nil {
+//          log.Fatalln(err)
+//     }
+//
+type Chain struct {
+	Providers []credentials.Provider
+	curr      credentials.Provider
+}
+
+// NewChainCredentials returns a pointer to a new Credentials object
+// wrapping a chain of providers.
+func NewChainCredentials(providers []credentials.Provider) *credentials.Credentials {
+	for _, p := range providers {
+		if p == nil {
+			panic("providers cannot be uninitialized")
+		}
+	}
+	return credentials.New(&Chain{
+		Providers: append([]credentials.Provider{}, providers...),
+	})
+}
+
+// Retrieve returns the credentials value, returns no credentials(anonymous)
+// if no credentials provider returned any value.
+//
+// If a provider is found with credentials, it will be cached and any calls
+// to IsExpired() will return the expired state of the cached provider.
+func (c *Chain) Retrieve() (credentials.Value, error) {
+	for _, p := range c.Providers {
+		creds, _ := p.Retrieve()
+		if creds.AccessKeyID != "" && !p.IsExpired() {
+			// Only return credentials that are
+			// available and not expired.
+			c.curr = p
+			return creds, nil
+		}
+	}
+
+	providers := make([]string, 0, len(c.Providers))
+	for _, p := range c.Providers {
+		providers = append(providers, reflect.TypeOf(p).String())
+	}
+
+	return credentials.Value{}, fmt.Errorf("no credentials found in %s cannot proceed", providers)
+
+}
+
+// IsExpired will returned the expired state of the currently cached provider
+// if there is one. If there is no current provider, true will be returned.
+func (c *Chain) IsExpired() bool {
+	if c.curr != nil {
+		return c.curr.IsExpired()
+	}
+
+	return true
+}

cmd/gateway/s3/gateway-s3-sse.go

@@ -113,7 +113,7 @@ func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 			} else {
 				objects = append(objects, obj)
 			}
-			if len(objects) > maxKeys {
+			if maxKeys > 0 && len(objects) > maxKeys {
 				break
 			}
 		}
@@ -130,7 +130,7 @@ func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 				prefixes = append(prefixes, p)
 			}
 		}
-		if (len(objects) > maxKeys) || !loi.IsTruncated {
+		if (maxKeys > 0 && len(objects) > maxKeys) || !loi.IsTruncated {
 			break
 		}
 	}
@@ -784,7 +784,7 @@ func (l *s3EncObjects) getStalePartsForBucket(ctx context.Context, bucket string
 	return
 }
 
-func (l *s3EncObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (l *s3EncObjects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	var prefix, continuationToken, delimiter, startAfter string
 	expParts := make(map[string]string)
 

cmd/gateway/s3/gateway-s3.go

@@ -23,6 +23,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -34,9 +35,11 @@ import (
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	minio "github.com/minio/minio/cmd"
+	"github.com/minio/minio/internal/config"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/env"
 )
 
 func init() {
@@ -94,12 +97,16 @@ func s3GatewayMain(ctx *cli.Context) {
 	logger.FatalIf(minio.ValidateGatewayArguments(serverAddr, args.First()), "Invalid argument")
 
 	// Start the gateway..
-	minio.StartGateway(ctx, &S3{args.First()})
+	minio.StartGateway(ctx, &S3{
+		host:  args.First(),
+		debug: env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn,
+	})
 }
 
 // S3 implements Gateway.
 type S3 struct {
-	host string
+	host  string
+	debug bool
 }
 
 // Name implements Gateway interface.
@@ -139,7 +146,6 @@ func randString(n int, src rand.Source, prefix string) string {
 var defaultProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
-	&credentials.EnvMinio{},
 }
 
 // Chains all credential types, in the following order:
@@ -153,15 +159,17 @@ var defaultAWSCredProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
 	&credentials.IAM{
+		// you can specify a custom STS endpoint.
+		Endpoint: env.Get("MINIO_GATEWAY_S3_STS_ENDPOINT", ""),
 		Client: &http.Client{
 			Transport: minio.NewGatewayHTTPTransport(),
 		},
 	},
-	&credentials.EnvMinio{},
 }
 
-// newS3 - Initializes a new client by auto probing S3 server signature.
-func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
+// new - Initializes a new client by auto probing S3 server signature.
+func (g *S3) new(creds madmin.Credentials, transport http.RoundTripper) (*miniogo.Core, error) {
+	urlStr := g.host
 	if urlStr == "" {
 		urlStr = "https://s3.amazonaws.com"
 	}
@@ -177,30 +185,70 @@ func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
 		return nil, err
 	}
 
-	var creds *credentials.Credentials
+	var chainCreds *credentials.Credentials
 	if s3utils.IsAmazonEndpoint(*u) {
 		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
 		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
-		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
-
+		chainCreds = NewChainCredentials(defaultAWSCredProviders)
 	} else {
-		creds = credentials.NewChainCredentials(defaultProviders)
+		chainCreds = NewChainCredentials(defaultProviders)
 	}
 
-	options := &miniogo.Options{
-		Creds:        creds,
+	optionsStaticCreds := &miniogo.Options{
+		Creds:        credentials.NewStaticV4(creds.AccessKey, creds.SecretKey, creds.SessionToken),
 		Secure:       secure,
 		Region:       s3utils.GetRegionFromURL(*u),
 		BucketLookup: miniogo.BucketLookupAuto,
-		Transport:    tripper,
+		Transport:    transport,
 	}
 
-	clnt, err := miniogo.New(endpoint, options)
+	optionsChainCreds := &miniogo.Options{
+		Creds:        chainCreds,
+		Secure:       secure,
+		Region:       s3utils.GetRegionFromURL(*u),
+		BucketLookup: miniogo.BucketLookupAuto,
+		Transport:    transport,
+	}
+
+	clntChain, err := miniogo.New(endpoint, optionsChainCreds)
 	if err != nil {
 		return nil, err
 	}
 
-	return &miniogo.Core{Client: clnt}, nil
+	clntStatic, err := miniogo.New(endpoint, optionsStaticCreds)
+	if err != nil {
+		return nil, err
+	}
+
+	if g.debug {
+		clntChain.TraceOn(os.Stderr)
+		clntStatic.TraceOn(os.Stderr)
+	}
+
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
+	if _, err = clntStatic.BucketExists(context.Background(), probeBucketName); err != nil {
+		switch miniogo.ToErrorResponse(err).Code {
+		case "InvalidAccessKeyId":
+			// Check if the provided keys are valid for chain.
+			if _, err = clntChain.BucketExists(context.Background(), probeBucketName); err != nil {
+				if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
+					return nil, err
+				}
+			}
+			return &miniogo.Core{Client: clntChain}, nil
+		case "AccessDenied":
+			// this is a good error means backend is reachable
+			// and credentials are valid but credentials don't
+			// have access to 'probeBucketName' which is harmless.
+			return &miniogo.Core{Client: clntStatic}, nil
+		default:
+			return nil, err
+		}
+	}
+
+	// if static keys are valid always use static keys.
+	return &miniogo.Core{Client: clntStatic}, nil
 }
 
 // NewGatewayLayer returns s3 ObjectLayer.
@@ -214,20 +262,11 @@ func (g *S3) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, error
 
 	// creds are ignored here, since S3 gateway implements chaining
 	// all credentials.
-	clnt, err := newS3(g.host, t)
+	clnt, err := g.new(creds, t)
 	if err != nil {
 		return nil, err
 	}
 
-	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
-
-	// Check if the provided keys are valid.
-	if _, err = clnt.BucketExists(context.Background(), probeBucketName); err != nil {
-		if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
-			return nil, err
-		}
-	}
-
 	s := s3Objects{
 		Client:  clnt,
 		Metrics: metrics,
@@ -271,11 +310,17 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
 // StorageInfo is not relevant to S3 backend.
 func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
 	si.Backend.Type = madmin.Gateway
-	host := l.Client.EndpointURL().Host
-	if l.Client.EndpointURL().Port() == "" {
-		host = l.Client.EndpointURL().Host + ":" + l.Client.EndpointURL().Scheme
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
+	// check if bucket exists.
+	_, err := l.Client.BucketExists(ctx, probeBucketName)
+	switch miniogo.ToErrorResponse(err).Code {
+	case "", "AccessDenied":
+		si.Backend.GatewayOnline = true
+	default:
+		logger.LogIf(ctx, err)
+		si.Backend.GatewayOnline = false
 	}
-	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, host)
 	return si, nil
 }
 
@@ -354,7 +399,7 @@ func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error)
 }
 
 // DeleteBucket deletes a bucket on S3
-func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	err := l.Client.RemoveBucket(ctx, bucket)
 	if err != nil {
 		return minio.ErrorRespToObjectError(err, bucket)
@@ -470,6 +515,10 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string,
 		UserMetadata:         opts.UserDefined,
 		ServerSideEncryption: opts.ServerSideEncryption,
 		UserTags:             tagMap,
+		// Content-Md5 is needed for buckets with object locking,
+		// instead of spending an extra API call to detect this
+		// we can set md5sum to be calculated always.
+		SendContentMd5: true,
 	}
 	ui, err := l.Client.PutObject(ctx, bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), putOpts)
 	if err != nil {

cmd/global-heal.go

@@ -19,13 +19,13 @@ package cmd
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"sort"
 	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/wildcard"
@@ -43,12 +43,11 @@ func newBgHealSequence() *healSequence {
 
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
-		Remove:   true,
-		ScanMode: madmin.HealNormalScan,
+		Remove:   healDeleteDangling,
+		ScanMode: globalHealConfig.ScanMode(),
 	}
 
 	return &healSequence{
-		sourceCh:    make(chan healSource),
 		respCh:      make(chan healResult),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
@@ -133,6 +132,11 @@ func getBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealS
 		return status.Sets[i].ID < status.Sets[j].ID
 	})
 
+	backendInfo := o.BackendInfo()
+	status.SCParity = make(map[string]int)
+	status.SCParity[storageclass.STANDARD] = backendInfo.StandardSCParity
+	status.SCParity[storageclass.RRS] = backendInfo.RRSCParity
+
 	return status, true
 
 }
@@ -165,6 +169,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
 		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
 	})
 
+	scanMode := globalHealConfig.ScanMode()
+
 	// Heal all buckets with all objects
 	for _, bucket := range buckets {
 		if tracker.isHealed(bucket.Name) {
@@ -183,10 +189,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
 		tracker.Object = ""
 		tracker.Bucket = bucket.Name
 		// Heal current bucket
-		if _, err := er.HealBucket(ctx, bucket.Name, madmin.HealOpts{}); err != nil {
-			if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-				logger.LogIf(ctx, err)
-			}
+		if _, err := er.HealBucket(ctx, bucket.Name, madmin.HealOpts{
+			ScanMode: scanMode,
+		}); err != nil {
+			logger.LogIf(ctx, err)
 		}
 
 		if serverDebugLog {
@@ -195,7 +201,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
 
 		disks, _ := er.getOnlineDisksWithHealing()
 		if len(disks) == 0 {
-			return errors.New("healErasureSet: No non-healing disks found")
+			// all disks are healing in this set, this is allowed
+			// so we simply proceed to next bucket, marking the bucket
+			// as done as there are no objects to heal.
+			tracker.bucketDone(bucket.Name)
+			logger.LogIf(ctx, tracker.update(ctx))
+			continue
 		}
 
 		// Limit listing to 3 drives.
@@ -221,21 +232,28 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
 					return
 				}
 			}
+
 			fivs, err := entry.fileInfoVersions(bucket.Name)
 			if err != nil {
+				err := bgSeq.queueHealTask(healSource{
+					bucket:    bucket.Name,
+					object:    entry.name,
+					versionID: "",
+				}, madmin.HealItemObject)
 				logger.LogIf(ctx, err)
 				return
 			}
-			waitForLowHTTPReq(globalHealConfig.IOCount, globalHealConfig.Sleep)
+
 			for _, version := range fivs.Versions {
-				if _, err := er.HealObject(ctx, bucket.Name, version.Name, version.VersionID, madmin.HealOpts{
-					ScanMode: madmin.HealNormalScan, Remove: healDeleteDangling}); err != nil {
-					if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-						// If not deleted, assume they failed.
-						tracker.ItemsFailed++
-						tracker.BytesFailed += uint64(version.Size)
-						logger.LogIf(ctx, err)
-					}
+				if _, err := er.HealObject(ctx, bucket.Name, version.Name,
+					version.VersionID, madmin.HealOpts{
+						ScanMode: scanMode,
+						Remove:   healDeleteDangling,
+					}); err != nil {
+					// If not deleted, assume they failed.
+					tracker.ItemsFailed++
+					tracker.BytesFailed += uint64(version.Size)
+					logger.LogIf(ctx, err)
 				} else {
 					tracker.ItemsHealed++
 					tracker.BytesDone += uint64(version.Size)
@@ -246,6 +264,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
 			if time.Since(tracker.LastUpdate) > time.Minute {
 				logger.LogIf(ctx, tracker.update(ctx))
 			}
+
+			// Wait and proceed if there are active requests
+			waitForLowHTTPReq()
 		}
 
 		// How to resolve partial results.
@@ -265,9 +286,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
 			agreed:         healEntry,
 			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
 				entry, ok := entries.resolve(&resolver)
-				if ok {
-					healEntry(*entry)
+				if !ok {
+					// check if we can get one entry atleast
+					// proceed to heal nonetheless.
+					entry, _ = entries.firstFound()
 				}
+				healEntry(*entry)
 			},
 			finished: nil,
 		})
@@ -298,14 +322,14 @@ func healObject(bucket, object, versionID string, scan madmin.HealScanMode) {
 	// Get background heal sequence to send elements to heal
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if ok {
-		bgSeq.sourceCh <- healSource{
+		bgSeq.queueHealTask(healSource{
 			bucket:    bucket,
 			object:    object,
 			versionID: versionID,
 			opts: &madmin.HealOpts{
-				Remove:   true, // if found dangling purge it.
+				Remove:   healDeleteDangling, // if found dangling purge it.
 				ScanMode: scan,
 			},
-		}
+		}, madmin.HealItemObject)
 	}
 }

cmd/globals.go

@@ -25,10 +25,12 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/console/restapi"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
+	"github.com/rs/dnscache"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/auth"
@@ -37,8 +39,10 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/policy/opa"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	xhttp "github.com/minio/minio/internal/http"
 	etcd "go.etcd.io/etcd/client/v3"
 
@@ -88,14 +92,12 @@ const (
 	// date and server date during signature verification.
 	globalMaxSkewTime = 15 * time.Minute // 15 minutes skew allowed.
 
-	// GlobalStaleUploadsExpiry - Expiry duration after which the uploads in multipart, tmp directory are deemed stale.
+	// GlobalStaleUploadsExpiry - Expiry duration after which the uploads in multipart,
+	// tmp directory are deemed stale.
 	GlobalStaleUploadsExpiry = time.Hour * 24 // 24 hrs.
 
 	// GlobalStaleUploadsCleanupInterval - Cleanup interval when the stale uploads cleanup is initiated.
-	GlobalStaleUploadsCleanupInterval = time.Hour * 12 // 12 hrs.
-
-	// GlobalServiceExecutionInterval - Executes the Lifecycle events.
-	GlobalServiceExecutionInterval = time.Hour * 24 // 24 hrs.
+	GlobalStaleUploadsCleanupInterval = time.Hour * 6 // 6 hrs.
 
 	// Refresh interval to update in-memory iam config cache.
 	globalRefreshIAMInterval = 5 * time.Minute
@@ -187,6 +189,7 @@ var (
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
 	globalOpenIDConfig openid.Config
+	globalSTSTLSConfig xtls.Config
 
 	// CA root certificates, a nil value means system certs pool will be used
 	globalRootCAs *x509.CertPool
@@ -216,8 +219,8 @@ var (
 	// The name of this local node, fetched from arguments
 	globalLocalNodeName string
 
-	// The global subnet license
-	globalSubnetLicense string
+	// The global subnet config
+	globalSubnetConfig subnet.Config
 
 	globalRemoteEndpoints map[string]Endpoint
 
@@ -253,6 +256,9 @@ var (
 	// Allocated etcd endpoint for config and bucket DNS.
 	globalEtcdClient *etcd.Client
 
+	// Cluster replication manager.
+	globalSiteReplicationSys SiteReplicationSys
+
 	// Is set to true when Bucket federation is requested
 	// and is 'true' when etcdConfig.PathPrefix is empty
 	globalBucketFederation bool
@@ -307,7 +313,9 @@ var (
 
 	globalProxyTransport http.RoundTripper
 
-	globalDNSCache *xhttp.DNSCache
+	globalDNSCache = &dnscache.Resolver{
+		Timeout: 5 * time.Second,
+	}
 
 	globalForwarder *handlers.Forwarder
 
@@ -315,7 +323,8 @@ var (
 
 	globalTierJournal *tierJournal
 
-	globalDebugRemoteTiersImmediately []string
+	globalConsoleSrv *restapi.Server
+
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -22,9 +22,11 @@ import (
 	"sync"
 	"time"
 
+	mem "github.com/shirou/gopsutil/v3/mem"
+
 	"github.com/minio/minio/internal/config/api"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/sys"
 )
 
 type apiConfig struct {
@@ -40,6 +42,10 @@ type apiConfig struct {
 	replicationWorkers       int
 	replicationFailedWorkers int
 	transitionWorkers        int
+
+	staleUploadsExpiry          time.Duration
+	staleUploadsCleanupInterval time.Duration
+	deleteCleanupInterval       time.Duration
 }
 
 func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
@@ -48,29 +54,42 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 
 	t.clusterDeadline = cfg.ClusterDeadline
 	t.corsAllowOrigins = cfg.CorsAllowOrigin
+	maxSetDrives := 0
 	for _, setDriveCount := range setDriveCounts {
 		t.totalDriveCount += setDriveCount
+		if setDriveCount > maxSetDrives {
+			maxSetDrives = setDriveCount
+		}
 	}
 
 	var apiRequestsMaxPerNode int
 	if cfg.RequestsMax <= 0 {
-		stats, err := sys.GetStats()
+		var maxMem uint64
+		memStats, err := mem.VirtualMemory()
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
 			// Default to 8 GiB, not critical.
-			stats.TotalRAM = 8 << 30
+			maxMem = 8 << 30
+		} else {
+			maxMem = memStats.Available / 2
 		}
+
 		// max requests per node is calculated as
 		// total_ram / ram_per_request
 		// ram_per_request is (2MiB+128KiB) * driveCount \
 		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
-		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(t.totalDriveCount*(blockSizeLarge+blockSizeSmall)+int(blockSizeV1*2+blockSizeV2*2)))
+		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
+		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
+
+		if globalIsErasure {
+			logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
+		}
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
 		if len(globalEndpoints.Hostnames()) > 0 {
 			apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames())
 		}
 	}
+
 	if cap(t.requestsPool) < apiRequestsMaxPerNode {
 		// Only replace if needed.
 		// Existing requests will use the previous limit,
@@ -92,6 +111,10 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
 	}
 	t.transitionWorkers = cfg.TransitionWorkers
+
+	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
+	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
+	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
 }
 
 func (t *apiConfig) getListQuorum() int {
@@ -110,6 +133,39 @@ func (t *apiConfig) getCorsAllowOrigins() []string {
 	return corsAllowOrigins
 }
 
+func (t *apiConfig) getStaleUploadsCleanupInterval() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.staleUploadsCleanupInterval == 0 {
+		return 6 * time.Hour // default 6 hours
+	}
+
+	return t.staleUploadsCleanupInterval
+}
+
+func (t *apiConfig) getStaleUploadsExpiry() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.staleUploadsExpiry == 0 {
+		return 24 * time.Hour // default 24 hours
+	}
+
+	return t.staleUploadsExpiry
+}
+
+func (t *apiConfig) getDeleteCleanupInterval() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.deleteCleanupInterval == 0 {
+		return 5 * time.Minute // every 5 minutes
+	}
+
+	return t.deleteCleanupInterval
+}
+
 func (t *apiConfig) getClusterDeadline() time.Duration {
 	t.mu.RLock()
 	defer t.mu.RUnlock()

cmd/healthcheck-handler.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"strconv"
 
@@ -29,6 +30,11 @@ const unavailable = "offline"
 
 // ClusterCheckHandler returns if the server is ready for requests.
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if globalIsGateway {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
 	if shouldProxy() {
@@ -67,6 +73,11 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // ClusterReadCheckHandler returns if the server is ready for requests.
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if globalIsGateway {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
 	if shouldProxy() {
@@ -85,17 +96,13 @@ func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
 // ReadinessCheckHandler Checks if the process is up. Always returns success.
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	if shouldProxy() {
-		// Service not initialized yet
-		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
-	}
-
-	writeResponse(w, http.StatusOK, nil, mimeNone)
+	LivenessCheckHandler(w, r)
 }
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
@@ -104,5 +111,51 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		// Service not initialized yet
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 	}
+
+	if globalIsGateway {
+		objLayer := newObjectLayerFn()
+		if objLayer == nil {
+			apiErr := toAPIError(r.Context(), errServerNotInitialized)
+			switch r.Method {
+			case http.MethodHead:
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, apiErr, r.URL)
+			}
+			return
+		}
+
+		storageInfo, _ := objLayer.StorageInfo(r.Context())
+		if !storageInfo.Backend.GatewayOnline {
+			err := errors.New("gateway backend is not reachable")
+			apiErr := toAPIError(r.Context(), err)
+			switch r.Method {
+			case http.MethodHead:
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, apiErr, r.URL)
+			}
+			return
+		}
+
+		if globalEtcdClient != nil {
+			// Borrowed from
+			// https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
+			ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
+			defer cancel()
+			if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
+				// etcd unreachable throw an error..
+				switch r.Method {
+				case http.MethodHead:
+					apiErr := toAPIError(r.Context(), err)
+					writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+				case http.MethodGet:
+					writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+				}
+				return
+			}
+		}
+	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }

cmd/http-tracer.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"reflect"
@@ -50,6 +49,11 @@ type recordRequest struct {
 	bytesRead int
 }
 
+func (r *recordRequest) Close() error {
+	// no-op
+	return nil
+}
+
 func (r *recordRequest) Read(p []byte) (n int, err error) {
 	n, err = r.Reader.Read(p)
 	r.bytesRead += n
@@ -62,6 +66,7 @@ func (r *recordRequest) Read(p []byte) (n int, err error) {
 	}
 	return n, err
 }
+
 func (r *recordRequest) Size() int {
 	sz := r.bytesRead
 	for k, v := range r.headers {
@@ -122,7 +127,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 	}
 
 	reqBodyRecorder := &recordRequest{Reader: r.Body, logBody: logBody, headers: reqHeaders}
-	r.Body = ioutil.NopCloser(reqBodyRecorder)
+	r.Body = reqBodyRecorder
 
 	now := time.Now().UTC()
 	t := madmin.TraceInfo{TraceType: madmin.TraceHTTP, FuncName: name, Time: now}

cmd/iam-dummy-store.go

@@ -81,10 +81,6 @@ func (ids *iamDummyStore) loadMappedPolicies(ctx context.Context, userType IAMUs
 	return nil
 }
 
-func (ids *iamDummyStore) loadAll(ctx context.Context, sys *IAMSys) error {
-	return sys.Load(ctx, ids)
-}
-
 func (ids *iamDummyStore) saveIAMConfig(ctx context.Context, item interface{}, path string, opts ...options) error {
 	return nil
 }
@@ -128,6 +124,3 @@ func (ids *iamDummyStore) deleteUserIdentity(ctx context.Context, name string, u
 func (ids *iamDummyStore) deleteGroupInfo(ctx context.Context, name string) error {
 	return nil
 }
-
-func (ids *iamDummyStore) watch(context.Context, *IAMSys) {
-}