avoid replication stats crash in Prometheus

Move to using `xl.meta` data structure to keep temporary partInfo,
this allows for a future change where we move to different parts to
different drives.

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,5 +15,6 @@
 
 ## Checklist:
 - [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
-- [ ] Documentation updated
 - [ ] Unit tests added/updated
+- [ ] Internal documentation updated
+- [ ] Create a documentation update request [here](https://github.com/minio/docs/issues/new?label=doc-change,title=Doc+Updated+Needed+For+PR+github.com%2fminio%2fminio%2fpull%2fNNNNN)

.github/workflows/depsreview.yaml

@@ -0,0 +1,14 @@
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v1

.github/workflows/go-cross.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2

.github/workflows/go-fips.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.11b7, 1.18.3b7]
+        go-version: [1.18.5b7]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/go-healing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
@@ -49,3 +49,4 @@ jobs:
           make verify-healing
           make verify-healing-inconsistent-versions
           make verify-healing-with-root-disks
+          make verify-healing-with-rewrite

.github/workflows/go-lint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]

.github/workflows/replication.yaml

@@ -21,7 +21,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x, 1.19.x]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/upgrade-ci-cd.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/vulncheck.yml

@@ -0,0 +1,26 @@
+name: VulnCheck
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+jobs:
+  vulncheck:
+    name: Analysis
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19.x
+          check-latest: true
+      - name: Get official govulncheck
+        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+        shell: bash
+      - name: Run govulncheck
+        run: govulncheck ./...
+        shell: bash

.golangci.yml

@@ -2,6 +2,13 @@ linters-settings:
   golint:
     min-confidence: 0
 
+  gofumpt:
+    lang-version: "1.18"
+
+    # Choose whether or not to use the extra rules that are disabled
+    # by default
+    extra-rules: false
+
   misspell:
     locale: US
 
@@ -14,28 +21,15 @@ linters:
     - govet
     - revive
     - ineffassign
-    - gosimple
-    - deadcode
-    - structcheck
     - gomodguard
     - gofmt
-    - unused
-    - structcheck
     - unconvert
-    - varcheck
+    - unused
     - gocritic
     - gofumpt
     - tenv
     - durationcheck
 
-linters-settings:
-  gofumpt:
-    lang-version: "1.17"
-
-    # Choose whether or not to use the extra rules that are disabled
-    # by default
-    extra-rules: false
-
 issues:
   exclude-use-default: false
   exclude:

.nancy-ignore

@@ -2,3 +2,5 @@ CVE-2020-26160
 CVE-2020-15136
 CVE-2020-15115
 CVE-2020-15114
+CVE-2022-2835
+CVE-2022-2837

Makefile

@@ -15,12 +15,12 @@ checks: ## check dependencies
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 
 help: ## print this help
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}'
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.45.2
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7-0.20211026165309-e818a1881b0e
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@f3635b96e4838a6c773babb65ef35297fe5fe2f9
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
@@ -89,12 +89,18 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
+	@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
 
-verify-healing-with-root-disks:
-	@echo "Verify healing with root disks"
+verify-healing-with-root-disks: ## verify healing root disks
+	@echo "Verify healing with root drives"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh)
 
+verify-healing-with-rewrite: ## verify healing to rewrite old xl.meta -> new xl.meta
+	@echo "Verify healing with rewrite"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/rewrite-old-new.sh)
+
 verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@echo "Verify resolving inconsistent versions build with race"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

README.md

@@ -125,7 +125,7 @@ You can also connect using any S3-compatible tool, such as the MinIO Client `mc`
 
 ## Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go install github.com/minio/minio@latest

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips"
 }
 
 function _build() {

buildscripts/heal-inconsistent-versions.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+function start_minio_4drive() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export MINIO_CI_CD=1
+
+    mkdir ${WORK_DIR}
+    C_PWD=${PWD}
+    if [ ! -x "$PWD/mc" ]; then
+	MC_BUILD_DIR="mc-$RANDOM"
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	    echo "failed to download https://github.com/minio/mc"
+	    purge "${MC_BUILD_DIR}"
+	    exit 1
+	fi
+
+	(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+
+	# remove mc source.
+	purge "${MC_BUILD_DIR}"
+    fi
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 5
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${PWD}/mc" mb --with-versioning minio/bucket
+
+    for i in $(seq 1 4); do
+	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
+
+	sudo chown -R root. "${WORK_DIR}/disk${i}"
+
+	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
+
+	sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
+    done
+
+    for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
+	"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
+    done
+
+    pkill minio
+    sleep 3
+}
+
+function main() {
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    start_minio_4drive ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/heal-manual.go

@@ -0,0 +1,87 @@
+//go:build ignore
+// +build ignore
+
+//
+// MinIO Object Storage (c) 2022 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/minio/madmin-go"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New(os.Args[1], os.Args[2], os.Args[3], false)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	opts := madmin.HealOpts{
+		Recursive: true,                  // recursively heal all objects at 'prefix'
+		Remove:    true,                  // remove content that has lost quorum and not recoverable
+		Recreate:  true,                  // rewrite all old non-inlined xl.meta to new xl.meta
+		ScanMode:  madmin.HealNormalScan, // by default do not do 'deep' scanning
+	}
+
+	start, _, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, "", false, false)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	fmt.Println("Healstart sequence ===")
+	enc := json.NewEncoder(os.Stdout)
+	if err = enc.Encode(&start); err != nil {
+		log.Fatalln(err)
+	}
+
+	fmt.Println()
+	for {
+		_, status, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, start.ClientToken, false, false)
+		if status.Summary == "finished" {
+			fmt.Println("Healstatus on items ===")
+			for _, item := range status.Items {
+				if err = enc.Encode(&item); err != nil {
+					log.Fatalln(err)
+				}
+			}
+			break
+		}
+		if status.Summary == "stopped" {
+			fmt.Println("Healstatus on items ===")
+			fmt.Println("Heal failed with", status.FailureDetail)
+			break
+		}
+
+		for _, item := range status.Items {
+			if err = enc.Encode(&item); err != nil {
+				log.Fatalln(err)
+			}
+		}
+
+		time.Sleep(time.Second)
+	}
+}

buildscripts/resolve-right-versions.sh

@@ -39,7 +39,7 @@ function start_minio_5drive() {
     "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
     pid=$!
     disown $pid
-    sleep 30
+    sleep 5
 
     if ! ps -p ${pid} 1>&2 >/dev/null; then
 	echo "server1 log:"

buildscripts/rewrite-old-new.sh

@@ -0,0 +1,151 @@
+#!/bin/bash -e
+
+set -E
+set -o pipefail
+set -x
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+function download_old_release() {
+    if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
+	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
+	chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
+    fi
+}
+
+function verify_rewrite() {
+    start_port=$1
+
+    export MINIO_ACCESS_KEY=minio
+    export MINIO_SECRET_KEY=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export MINIO_CI_CD=1
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 10
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    "${WORK_DIR}/mc" cp \
+		     buildscripts/verify-build.sh \
+		     minio/healing-rewrite-bucket/ \
+		     --disable-multipart --quiet
+
+    kill ${pid}
+    sleep 3
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 10
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    go build ./docs/debugging/s3-check-md5/
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -versions \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
+    sleep 1
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -versions \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    kill ${pid}
+}
+
+function main() {
+    download_old_release
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    verify_rewrite ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/unaligned-healing.sh

@@ -120,7 +120,7 @@ function start_minio_16drive() {
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
 
 	"${WORK_DIR}/mc" mb play/inspects
 	"${WORK_DIR}/mc" mirror inspects play/inspects
@@ -140,7 +140,7 @@ function start_minio_16drive() {
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
 
 	"${WORK_DIR}/mc" mb play/inspects
 	"${WORK_DIR}/mc" mirror inspects play/inspects

cmd/acl-handlers.go

@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/admin-bucket-handlers.go

@@ -69,7 +69,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -131,7 +131,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -173,7 +173,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -249,7 +249,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	}
 	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil {
 		switch err.(type) {
-		case BucketRemoteConnectionErr:
+		case RemoteTargetConnectionErr:
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -301,7 +301,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	}
 	if bucket != "" {
 		// Check if bucket exists.
-		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -340,7 +340,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -374,7 +374,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	bucket := pathClean(r.Form.Get("bucket"))
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -390,13 +390,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	)
 	if bucket != "" {
 		// Check if bucket exists.
-		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
 		buckets = append(buckets, BucketInfo{Name: bucket})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx)
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -619,6 +619,39 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	}
 }
 
+type importMetaReport struct {
+	madmin.BucketMetaImportErrs
+}
+
+func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
+	st := i.Buckets[bucket]
+	var errMsg string
+	if err != nil {
+		errMsg = err.Error()
+	}
+	switch fname {
+	case bucketPolicyConfig:
+		st.Policy = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketNotificationConfig:
+		st.Notification = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketLifecycleConfig:
+		st.Lifecycle = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketSSEConfig:
+		st.SSEConfig = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketTaggingConfig:
+		st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketQuotaConfigFile:
+		st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case objectLockConfig:
+		st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	case bucketVersioningConfig:
+		st.Versioning = madmin.MetaStatus{IsSet: true, Err: errMsg}
+	default:
+		st.Err = errMsg
+	}
+	i.Buckets[bucket] = st
+}
+
 // ImportBucketMetadataHandler - imports all bucket metadata from a zipped file and overwrite bucket metadata config
 // There are some caveats regarding the following:
 // 1. object lock config - object lock should have been specified at time of bucket creation. Only default retention settings are imported here.
@@ -628,9 +661,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 	ctx := newContext(r, w, "ImportBucketMetadata")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	bucket := pathClean(r.Form.Get("bucket"))
 
-	if !globalIsErasure {
+	if globalIsGateway {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
@@ -651,47 +683,46 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		return
 	}
 	bucketMap := make(map[string]struct{}, 1)
-
+	rpt := importMetaReport{
+		madmin.BucketMetaImportErrs{
+			Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
+		},
+	}
 	// import object lock config if any - order of import matters here.
 	for _, file := range zr.File {
 		slc := strings.Split(file.Name, slashSeparator)
 		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-			return
-		}
-		b, fileName := slc[0], slc[1]
-		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
-			bucket = b
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
 		}
+		bucket, fileName := slc[0], slc[1]
 		switch fileName {
 		case objectLockConfig:
 			reader, err := file.Open()
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			config, err := objectlock.ParseObjectLockConfig(reader)
 			if err != nil {
-				apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
-				apiErr.Description = err.Error()
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			configData, err := xml.Marshal(config)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				opts := BucketOptions{
+				opts := MakeBucketOptions{
 					LockEnabled: config.ObjectLockEnabled == "Enabled",
 				}
 				err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
 				if err != nil {
 					if _, ok := err.(BucketExists); !ok {
-						writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-						return
+						rpt.SetStatus(bucket, fileName, err)
+						continue
 					}
 				}
 				bucketMap[bucket] = struct{}{}
@@ -699,15 +730,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 			// Deny object locking configuration settings on existing buckets without object lock enabled.
 			if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 
 			// Call site replication hook.
 			//
@@ -720,8 +752,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				ObjectLockConfig: &cfgStr,
 				UpdatedAt:        updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 		}
 	}
@@ -730,162 +762,145 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 	for _, file := range zr.File {
 		slc := strings.Split(file.Name, slashSeparator)
 		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-			return
-		}
-		b, fileName := slc[0], slc[1]
-		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
-			bucket = b
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
 		}
+		bucket, fileName := slc[0], slc[1]
 		switch fileName {
 		case bucketVersioningConfig:
 			reader, err := file.Open()
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			v, err := versioning.ParseConfig(io.LimitReader(reader, maxBucketVersioningConfigSize))
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
-				if err != nil {
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{}); err != nil {
 					if _, ok := err.(BucketExists); !ok {
-						writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-						return
+						rpt.SetStatus(bucket, fileName, err)
+						continue
 					}
 				}
 				bucketMap[bucket] = struct{}{}
 			}
 
 			if globalSiteReplicationSys.isEnabled() && v.Suspended() {
-				writeErrorResponse(ctx, w, APIError{
-					Code:           "InvalidBucketState",
-					Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
-					HTTPStatusCode: http.StatusConflict,
-				}, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended."))
+				continue
 			}
 
 			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
-				writeErrorResponse(ctx, w, APIError{
-					Code:           "InvalidBucketState",
-					Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
-					HTTPStatusCode: http.StatusConflict,
-				}, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
+				continue
 			}
 			if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
-				writeErrorResponse(ctx, w, APIError{
-					Code:           "InvalidBucketState",
-					Description:    "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
-					HTTPStatusCode: http.StatusConflict,
-				}, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
+				continue
 			}
 
 			configData, err := xml.Marshal(v)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 		}
 	}
 
 	for _, file := range zr.File {
 		reader, err := file.Open()
 		if err != nil {
-			writeErrorResponse(ctx, w, importError(ctx, err, file.Name, ""), r.URL)
-			return
+			rpt.SetStatus(file.Name, "", err)
+			continue
 		}
 		sz := file.FileInfo().Size()
 		slc := strings.Split(file.Name, slashSeparator)
 		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-			return
-		}
-		b, fileName := slc[0], slc[1]
-		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
-			bucket = b
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
 		}
+		bucket, fileName := slc[0], slc[1]
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
-			err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				if _, ok := err.(BucketExists); !ok {
-					writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-					return
+					rpt.SetStatus(bucket, "", err)
+					continue
 				}
 			}
 			bucketMap[bucket] = struct{}{}
 		}
+		if _, ok := bucketMap[bucket]; !ok {
+			continue
+		}
 		switch fileName {
 		case bucketNotificationConfig:
-			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalNotificationSys.targetList)
+			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalEventNotifier.targetList)
 			if err != nil {
-				apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
-				if event.IsEventError(err) {
-					apiErr = importError(ctx, err, file.Name, bucket)
-				}
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			configData, err := xml.Marshal(config)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 			rulesMap := config.ToRulesMap()
-			globalNotificationSys.AddRulesMap(bucket, rulesMap)
+			globalEventNotifier.AddRulesMap(bucket, rulesMap)
+			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.
 			if sz > maxBucketPolicySize {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
+				continue
 			}
 
 			bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(reader, sz))
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Version in policy must not be empty
 			if bucketPolicy.Version == "" {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrMalformedPolicy.String()))
+				continue
 			}
 
 			configData, err := json.Marshal(bucketPolicy)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 			// Call site replication hook.
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 				Type:      madmin.SRBucketMetaTypePolicy,
@@ -893,56 +908,51 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Policy:    bucketPolicyBytes,
 				UpdatedAt: updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
-
 		case bucketLifecycleConfig:
 			bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Validate the received bucket policy document
 			if err = bucketLifecycle.Validate(); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Validate the transition storage ARNs
 			if err = validateTransitionTier(bucketLifecycle); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			configData, err := xml.Marshal(bucketLifecycle)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 		case bucketSSEConfig:
 			// Parse bucket encryption xml
 			encConfig, err := validateBucketSSEConfig(io.LimitReader(reader, maxBucketSSEConfigSize))
 			if err != nil {
-				apiErr := APIError{
-					Code:           "MalformedXML",
-					Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, err),
-					HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
-				}
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			// Return error if KMS is not initialized
 			if GlobalKMS == nil {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s", errorCodes[ErrKMSNotConfigured].Description))
+				continue
 			}
 			kmsKey := encConfig.KeyID()
 			if kmsKey != "" {
@@ -950,26 +960,27 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				_, err := GlobalKMS.GenerateKey(ctx, kmsKey, kmsContext)
 				if err != nil {
 					if errors.Is(err, kes.ErrKeyNotFound) {
-						writeErrorResponse(ctx, w, importError(ctx, errKMSKeyNotFound, file.Name, bucket), r.URL)
-						return
+						rpt.SetStatus(bucket, fileName, errKMSKeyNotFound)
+						continue
 					}
-					writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-					return
+					rpt.SetStatus(bucket, fileName, err)
+					continue
 				}
 			}
 
 			configData, err := xml.Marshal(encConfig)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			// Store the bucket encryption configuration in the object layer
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 
 			// Call site replication hook.
 			//
@@ -982,29 +993,30 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				SSEConfig: &cfgStr,
 				UpdatedAt: updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 		case bucketTaggingConfig:
 			tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
 			if err != nil {
-				apiErr := errorCodes.ToAPIErrWithErr(ErrMalformedXML, fmt.Errorf("error importing %s with %w", file.Name, err))
-				writeErrorResponse(ctx, w, apiErr, r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
+				continue
 			}
 
 			configData, err := xml.Marshal(tags)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
+
 			// Call site replication hook.
 			//
 			// We encode the xml bytes as base64 to ensure there are no encoding
@@ -1016,32 +1028,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				Tags:      &cfgStr,
 				UpdatedAt: updatedAt,
 			}); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 		case bucketQuotaConfigFile:
 			data, err := ioutil.ReadAll(reader)
 			if err != nil {
-				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			quotaConfig, err := parseBucketQuota(bucket, data)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 
 			if quotaConfig.Type == "fifo" {
-				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
+				continue
 			}
 
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
 			if err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
+			rpt.SetStatus(bucket, fileName, nil)
 
 			bucketMeta := madmin.SRBucketMeta{
 				Type:      madmin.SRBucketMetaTypeQuotaConfig,
@@ -1055,11 +1068,19 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 			// Call site replication hook.
 			if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
-				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
-				return
+				rpt.SetStatus(bucket, fileName, err)
+				continue
 			}
 		}
 	}
+
+	rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, rptData)
 }
 
 // ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
@@ -1089,7 +1110,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-config-kv.go

@@ -177,6 +177,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 }
 
 // GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
+//
+// `key` can be one of three forms:
+// 1. `subsys:target` -> request for config of a single subsystem and target pair.
+// 2. `subsys:` -> request for config of a single subsystem and the default target.
+// 3. `subsys` -> request for config of all targets for the given subsystem.
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
 
@@ -189,15 +194,34 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	cfg := globalServerConfig.Clone()
 	vars := mux.Vars(r)
-	buf := &bytes.Buffer{}
-	cw := config.NewConfigWriteTo(cfg, vars["key"])
-	if _, err := cw.WriteTo(buf); err != nil {
+	key := vars["key"]
+
+	var subSys, target string
+	{
+		ws := strings.SplitN(key, madmin.SubSystemSeparator, 2)
+		subSys = ws[0]
+		if len(ws) == 2 {
+			if ws[1] == "" {
+				target = madmin.Default
+			} else {
+				target = ws[1]
+			}
+		}
+	}
+
+	subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
+	var s strings.Builder
+	for _, subSysConfig := range subSysConfigs {
+		subSysConfig.AddString(&s, false)
+	}
+
 	password := cred.SecretKey
-	econfigData, err := madmin.EncryptData(password, buf.Bytes())
+	econfigData, err := madmin.EncryptData(password, []byte(s.String()))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -427,42 +451,30 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 		count += len(cfg[hkv.Key])
 	}
 	for _, hkv := range hkvs {
-		v := cfg[hkv.Key]
-		for target, kv := range v {
-			off := kv.Get(config.Enable) == config.EnableOff
+		// We ignore the error below, as we cannot get one.
+		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
+
+		for _, item := range cfgSubsysItems {
+			off := item.Config.Get(config.Enable) == config.EnableOff
 			switch hkv.Key {
 			case config.EtcdSubSys:
-				off = !etcd.Enabled(kv)
+				off = !etcd.Enabled(item.Config)
 			case config.CacheSubSys:
-				off = !cache.Enabled(kv)
+				off = !cache.Enabled(item.Config)
 			case config.StorageClassSubSys:
-				off = !storageclass.Enabled(kv)
+				off = !storageclass.Enabled(item.Config)
 			case config.PolicyPluginSubSys:
-				off = !polplugin.Enabled(kv)
+				off = !polplugin.Enabled(item.Config)
 			case config.IdentityOpenIDSubSys:
-				off = !openid.Enabled(kv)
+				off = !openid.Enabled(item.Config)
 			case config.IdentityLDAPSubSys:
-				off = !xldap.Enabled(kv)
+				off = !xldap.Enabled(item.Config)
 			case config.IdentityTLSSubSys:
 				off = !globalSTSTLSConfig.Enabled
 			case config.IdentityPluginSubSys:
-				off = !idplugin.Enabled(kv)
-			}
-			if off {
-				s.WriteString(config.KvComment)
-				s.WriteString(config.KvSpaceSeparator)
-			}
-			s.WriteString(hkv.Key)
-			if target != config.Default {
-				s.WriteString(config.SubSystemSeparator)
-				s.WriteString(target)
-			}
-			s.WriteString(config.KvSpaceSeparator)
-			s.WriteString(kv.String())
-			count--
-			if count > 0 {
-				s.WriteString(config.KvNewline)
+				off = !idplugin.Enabled(item.Config)
 			}
+			item.AddString(&s, off)
 		}
 	}
 

cmd/admin-handlers-site-replication.go

@@ -24,6 +24,8 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
@@ -116,19 +118,38 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
 		_, isForceCreate := r.Form["forceCreate"]
-		opts := BucketOptions{
+		createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
+		createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
+		if cerr != nil {
+			createdAt = timeSentinel
+		}
+
+		opts := MakeBucketOptions{
 			Location:          r.Form.Get("location"),
 			LockEnabled:       isLockEnabled,
 			VersioningEnabled: isVersioningEnabled,
 			ForceCreate:       isForceCreate,
+			CreatedAt:         createdAt,
 		}
 		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 	case madmin.ConfigureReplBktOp:
 		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
 	case madmin.DeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      false,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
 	case madmin.ForceDeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      true,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
+	case madmin.PurgeDeletedBucketOp:
+		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -436,6 +457,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 	opts.Users = q.Get("users") == "true"
 	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
 	opts.EntityValue = q.Get("entityvalue")
+	opts.ShowDeleted = q.Get("showDeleted") == "true"
 	return
 }
 

cmd/admin-handlers-users.go

@@ -1177,7 +1177,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx)
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1189,17 +1189,32 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		// For derived credentials, check the parent user's permissions.
 		accountName = cred.ParentUser
 	}
-	policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 
-	buf, err := json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+	var buf []byte
+	if accountName == globalActiveCred.AccessKey {
+		for _, policy := range iampolicy.DefaultPolicies {
+			if policy.Name == "consoleAdmin" {
+				buf, err = json.MarshalIndent(policy.Definition, "", " ")
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				break
+			}
+		}
+	} else {
+		policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		buf, err = json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	acctInfo := madmin.AccountInfo{
@@ -1514,7 +1529,29 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		}
 	}
 
-	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup)
+	// Validate that user or group exists.
+	if !isGroup {
+		if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
+			_, ok := globalIAMSys.GetUser(ctx, entityName)
+			if !ok {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+				return
+			}
+		}
+	} else {
+		_, err := globalIAMSys.GetGroupDescription(entityName)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	userType := regUser
+	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
+		userType = stsUser
+	}
+
+	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, userType, isGroup)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1524,6 +1561,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: entityName,
+			UserType:    int(userType),
 			IsGroup:     isGroup,
 			Policy:      policyName,
 		},
@@ -2099,7 +2137,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, userPolicyMappingsFile, u), r.URL)
 					return
 				}
-				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
 					return
 				}
@@ -2128,7 +2166,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for g, pm := range grpPolicyMap {
-				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
 					return
 				}
@@ -2167,7 +2205,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL)
 					return
 				}
-				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
 					return
 				}
@@ -2196,7 +2234,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for g, pm := range grpPolicyMap {
-				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
 					return
 				}

cmd/admin-handlers-users_test.go

@@ -34,7 +34,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go"
-	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -1141,6 +1141,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
 }
 
 func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials {
+	c.Helper()
 	randUser := mustGetUUID()
 	randPass := mustGetUUID()
 	err := admClnt.AddUser(ctx, randUser, randPass)
@@ -1154,6 +1155,7 @@ func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClie
 }
 
 func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminClient, accessKey string) madmin.UserInfo {
+	c.Helper()
 	ui, err := admClnt.GetUserInfo(ctx, accessKey)
 	if err != nil {
 		c.Fatalf("should be able to get user info: %v", err)
@@ -1162,6 +1164,7 @@ func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminCli
 }
 
 func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) {
+	c.Helper()
 	randUser := mustGetUUID()
 	randPass := mustGetUUID()
 	err := admClnt.AddUser(ctx, randUser, randPass)
@@ -1171,6 +1174,7 @@ func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminC
 }
 
 func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) madmin.Credentials {
+	c.Helper()
 	cr, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: tgtUser,
 	})
@@ -1181,6 +1185,7 @@ func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admCln
 }
 
 func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) {
+	c.Helper()
 	_, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: tgtUser,
 	})
@@ -1190,6 +1195,7 @@ func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, adm
 }
 
 func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
 	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
 	v, ok := <-res
 	if !ok || v.Err == nil {
@@ -1198,6 +1204,7 @@ func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bu
 }
 
 func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
 	res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{})
 	v, ok := <-res
 	if ok && v.Err != nil {
@@ -1205,7 +1212,16 @@ func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucke
 	}
 }
 
+func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) {
+	c.Helper()
+	_, err := client.ListBuckets(ctx)
+	if err != nil {
+		c.Fatalf("user was unable to list buckets: %v", err)
+	}
+}
+
 func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
 	_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
 	if e, ok := err.(minio.ErrorResponse); ok {
 		if e.Code == "AccessDenied" {
@@ -1221,6 +1237,7 @@ func (c *check) assertSvcAccS3Access(ctx context.Context, s *TestSuiteIAM, cr ma
 }
 
 func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *madmin.AdminClient, parentAK, svcAK string) {
+	c.Helper()
 	listResp, err := madmClient.ListServiceAccounts(ctx, parentAK)
 	if err != nil {
 		c.Fatalf("unable to list svc accounts: %v", err)
@@ -1246,6 +1263,7 @@ func (c *check) assertSvcAccInfoQueryable(ctx context.Context, madmClient *madmi
 // bucket. It creates a session policy that restricts listing on the bucket and
 // then enables it again in a session policy update call.
 func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
+	c.Helper()
 	svcAK, svcSK := mustGenerateCredentials(c)
 
 	// This policy does not allow listing objects.
@@ -1301,6 +1319,7 @@ func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuit
 }
 
 func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
+	c.Helper()
 	svcAK, svcSK := mustGenerateCredentials(c)
 	cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: accessKey,
@@ -1337,6 +1356,7 @@ func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *Tes
 }
 
 func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) {
+	c.Helper()
 	svcAK, svcSK := mustGenerateCredentials(c)
 	cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: accessKey,
@@ -1357,6 +1377,7 @@ func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmC
 }
 
 func mustGenerateCredentials(c *check) (string, string) {
+	c.Helper()
 	ak, sk, err := auth.GenerateCredentials()
 	if err != nil {
 		c.Fatalf("unable to generate credentials: %v", err)

cmd/admin-handlers.go

@@ -71,16 +71,6 @@ const (
 	mgmtForceStop   = "forceStop"
 )
 
-func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string, mode string) (us madmin.ServerUpdateStatus, err error) {
-	if err = doUpdate(u, lrTime, sha256Sum, releaseInfo, mode); err != nil {
-		return us, err
-	}
-
-	us.CurrentVersion = Version
-	us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout)
-	return us, nil
-}
-
 // ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
@@ -152,7 +142,16 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
+	// Download Binary Once
+	reader, err := downloadBinary(u, mode)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Push binary to other servers
+	for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader) {
 		if nerr.Err != nil {
 			err := AdminError{
 				Code:       AdminUpdateApplyFailure,
@@ -166,13 +165,39 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
+	err = verifyBinary(u, sha256Sum, releaseInfo, mode, reader)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
+	for _, nerr := range globalNotificationSys.CommitBinary(ctx) {
+		if nerr.Err != nil {
+			err := AdminError{
+				Code:       AdminUpdateApplyFailure,
+				Message:    nerr.Err.Error(),
+				StatusCode: http.StatusInternalServerError,
+			}
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	err = commitBinary()
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	updateStatus := madmin.ServerUpdateStatus{
+		CurrentVersion: Version,
+		UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+	}
+
 	// Marshal API response
 	jsonBytes, err := json.Marshal(updateStatus)
 	if err != nil {
@@ -363,18 +388,33 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 	if err != nil || interval < time.Second {
 		interval = defaultMetricsInterval
 	}
+
 	n, err := strconv.Atoi(r.Form.Get("n"))
 	if err != nil || n <= 0 {
 		n = math.MaxInt32
 	}
+
 	var types madmin.MetricType
 	if t, _ := strconv.ParseUint(r.Form.Get("types"), 10, 64); t != 0 {
 		types = madmin.MetricType(t)
 	} else {
 		types = madmin.MetricsAll
 	}
+
+	disks := strings.Split(r.Form.Get("disks"), ",")
+	byDisk := strings.EqualFold(r.Form.Get("by-disk"), "true")
+	var diskMap map[string]struct{}
+	if len(disks) > 0 && disks[0] != "" {
+		diskMap = make(map[string]struct{}, len(disks))
+		for _, k := range disks {
+			if k != "" {
+				diskMap[k] = struct{}{}
+			}
+		}
+	}
+
 	hosts := strings.Split(r.Form.Get("hosts"), ",")
-	byhost := strings.EqualFold(r.Form.Get("by-host"), "true")
+	byHost := strings.EqualFold(r.Form.Get("by-host"), "true")
 	var hostMap map[string]struct{}
 	if len(hosts) > 0 && hosts[0] != "" {
 		hostMap = make(map[string]struct{}, len(hosts))
@@ -384,34 +424,34 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 			}
 		}
 	}
+
 	done := ctx.Done()
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
+	enc := json.NewEncoder(w)
 	for n > 0 {
 		var m madmin.RealtimeMetrics
-		mLocal := collectLocalMetrics(types, hostMap)
+		mLocal := collectLocalMetrics(types, hostMap, diskMap)
 		m.Merge(&mLocal)
 
 		// Allow half the interval for collecting remote...
 		cctx, cancel := context.WithTimeout(ctx, interval/2)
-		mRemote := collectRemoteMetrics(cctx, types, hostMap)
+		mRemote := collectRemoteMetrics(cctx, types, hostMap, diskMap)
 		cancel()
 		m.Merge(&mRemote)
-		if !byhost {
+		if !byHost {
 			m.ByHost = nil
 		}
+		if !byDisk {
+			m.ByDisk = nil
+		}
 
 		m.Final = n <= 1
+
 		// Marshal API response
-		jsonBytes, err := json.Marshal(m)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		_, err = w.Write(jsonBytes)
-		if err != nil {
+		if err := enc.Encode(&m); err != nil {
 			n = 0
 		}
 
@@ -1190,16 +1230,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		concurrent = 32
 	}
 
-	if runtime.GOMAXPROCS(0) < concurrent {
-		concurrent = runtime.GOMAXPROCS(0)
-	}
-
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
 		duration = time.Second * 10
 	}
 
-	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
+	storageInfo, _ := objectAPI.StorageInfo(ctx)
+
+	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 	if !sufficientCapacity {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
 			Code:       "XMinioSpeedtestInsufficientCapacity",
@@ -1249,14 +1287,24 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		storageClass:     storageClass,
 		bucketName:       customBucket,
 	})
+	var prevResult madmin.SpeedTestResult
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-keepAliveTicker.C:
-			// Write a blank entry to prevent client from disconnecting
-			if err := enc.Encode(madmin.SpeedTestResult{}); err != nil {
-				return
+			// if previous result is set keep writing the
+			// previous result back to the client
+			if prevResult.Version != "" {
+				if err := enc.Encode(prevResult); err != nil {
+					return
+				}
+			} else {
+				// first result is not yet obtained, keep writing
+				// empty entry to prevent client from disconnecting.
+				if err := enc.Encode(madmin.SpeedTestResult{}); err != nil {
+					return
+				}
 			}
 			w.(http.Flusher).Flush()
 		case result, ok := <-ch:
@@ -1266,13 +1314,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 			if err := enc.Encode(result); err != nil {
 				return
 			}
+			prevResult = result
 			w.(http.Flusher).Flush()
 		}
 	}
 }
 
 func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
-	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
+	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
@@ -1286,11 +1335,11 @@ func deleteObjectPerfBucket(objectAPI ObjectLayer) {
 	objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
 		Force:      true,
 		NoRecreate: true,
+		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 	})
 }
 
-func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
-	storageInfo, _ := objectAPI.StorageInfo(ctx)
+func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo, concurrent int, size int, autotune bool) (bool, bool, string) {
 	capacityNeeded := uint64(concurrent * size)
 	capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)
 
@@ -1828,7 +1877,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 		Mode:         string(mode),
 		Domain:       domain,
 		Region:       globalSite.Region,
-		SQSARN:       globalNotificationSys.GetARNList(false),
+		SQSARN:       globalEventNotifier.GetARNList(false),
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
 		Objects:      objects,
@@ -2199,15 +2248,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	getAndWriteObjPerfInfo := func() {
 		if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
 			concurrent := 32
-			if runtime.GOMAXPROCS(0) < concurrent {
-				concurrent = runtime.GOMAXPROCS(0)
-			}
+
+			storageInfo, _ := objectAPI.StorageInfo(ctx)
 
 			size := 64 * humanize.MiByte
 			autotune := true
 
-			sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
-
+			sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 			if !sufficientCapacity {
 				healthInfo.Perf.Error = capacityErrMsg
 				partialWrite(healthInfo)
@@ -2636,6 +2683,84 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	return nil
 }
 
+func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
+	// Send profiling data to zip as file
+	header, zerr := zip.FileInfoHeader(dummyFileInfo{
+		name:    name,
+		size:    int64(len(data)),
+		mode:    0o600,
+		modTime: UTCNow(),
+		isDir:   false,
+		sys:     nil,
+	})
+	if zerr != nil {
+		return zerr
+	}
+	header.Method = zip.Deflate
+	zwriter, zerr := zipWriter.CreateHeader(header)
+	if zerr != nil {
+		return zerr
+	}
+	_, err := io.Copy(zwriter, bytes.NewReader(data))
+	return err
+}
+
+// appendClusterMetaInfoToZip gets information of the current cluster and embedded
+// it in the passed zipwriter, This is not a critical function and it is allowed
+// to fail with a ten seconds timeout.
+func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return
+	}
+
+	// Add a ten seconds timeout because getting profiling data
+	// is critical for debugging, in contrary to getting cluster info
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	resultCh := make(chan madmin.ClusterRegistrationInfo)
+	go func() {
+		ci := madmin.ClusterRegistrationInfo{}
+		ci.Info.NoOfServerPools = len(globalEndpoints)
+		ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
+		ci.Info.MinioVersion = Version
+
+		si, _ := objectAPI.StorageInfo(ctx)
+
+		ci.Info.NoOfDrives = len(si.Disks)
+		for _, disk := range si.Disks {
+			ci.Info.TotalDriveSpace += disk.TotalSpace
+			ci.Info.UsedDriveSpace += disk.UsedSpace
+		}
+
+		dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
+
+		ci.UsedCapacity = dataUsageInfo.ObjectsTotalSize
+		ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount
+		ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount
+
+		ci.DeploymentID = globalDeploymentID
+		ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
+		resultCh <- ci
+	}()
+
+	select {
+	case <-ctx.Done():
+		return
+	case ci := <-resultCh:
+		out, err := json.MarshalIndent(ci, "", "   ")
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return
+		}
+		err = embedFileInZip(zipWriter, "cluster.info", out)
+		if err != nil {
+			logger.LogIf(ctx, err)
+		}
+	}
+}
+
 // getRawDataer provides an interface for getting raw FS files.
 type getRawDataer interface {
 	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
@@ -2655,7 +2780,8 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	o, ok := newObjectLayerFn().(getRawDataer)
+	objLayer := newObjectLayerFn()
+	o, ok := objLayer.(getRawDataer)
 	if !ok {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
@@ -2778,6 +2904,8 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}); err != nil {
 		logger.LogIf(ctx, err)
 	}
+
+	appendClusterMetaInfoToZip(ctx, zipWriter)
 }
 
 func createHostAnonymizerForFSMode() map[string]string {

cmd/admin-heal-ops.go

@@ -822,7 +822,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	buckets, err := objAPI.ListBuckets(h.ctx)
+	buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}

cmd/api-errors.go

@@ -30,7 +30,7 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"google.golang.org/api/googleapi"
 
-	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -176,6 +176,7 @@ const (
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
 	ErrBucketAlreadyExists
+	ErrTooManyBuckets
 	ErrMetadataTooLarge
 	ErrUnsupportedMetadata
 	ErrMaximumExpires
@@ -196,8 +197,9 @@ const (
 	ErrInvalidTagDirective
 	// Add new error codes here.
 
-	// SSE-S3 related API errors
+	// SSE-S3/SSE-KMS related API errors
 	ErrInvalidEncryptionMethod
+	ErrInvalidEncryptionKeyID
 
 	// Server-Side-Encryption (with Customer provided key) related API errors.
 	ErrInsecureSSECustomerRequest
@@ -231,6 +233,7 @@ const (
 
 	// S3 extended errors.
 	ErrContentSHA256Mismatch
+	ErrContentChecksumMismatch
 
 	// Add new extended error codes here.
 
@@ -391,6 +394,8 @@ const (
 	ErrAccountNotEligible
 	ErrAdminServiceAccountNotFound
 	ErrPostPolicyConditionInvalidFormat
+
+	ErrInvalidChecksum
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -680,6 +685,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrTooManyBuckets: {
+		Code:           "TooManyBuckets",
+		Description:    "You have attempted to create more buckets than allowed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
 		Description:    "The bucket you tried to delete is not empty",
@@ -887,7 +897,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationRemoteConnectionError: {
 		Code:           "XMinioAdminReplicationRemoteConnectionError",
-		Description:    "Remote service connection error - please check remote service credentials and target bucket",
+		Description:    "Remote service connection error",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationBandwidthLimitError: {
@@ -1072,6 +1082,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The encryption method specified is not supported",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidEncryptionKeyID: {
+		Code:           "InvalidRequest",
+		Description:    "The specified KMS KeyID contains unsupported characters",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInsecureSSECustomerRequest: {
 		Code:           "InvalidRequest",
 		Description:    "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
@@ -1154,11 +1169,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The provided 'x-amz-content-sha256' header does not match what was computed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrContentChecksumMismatch: {
+		Code:           "XAmzContentChecksumMismatch",
+		Description:    "The provided 'x-amz-checksum' header does not match what was computed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// MinIO extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
-		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
+		Description:    "Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
 	ErrRequestBodyParse: {
@@ -1868,6 +1888,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid according to Policy: Policy Condition failed",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidChecksum: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid checksum provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -1921,6 +1946,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidEncryptionParameters
 	case crypto.ErrInvalidEncryptionMethod:
 		apiErr = ErrInvalidEncryptionMethod
+	case crypto.ErrInvalidEncryptionKeyID:
+		apiErr = ErrInvalidEncryptionKeyID
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
 	case crypto.ErrMissingCustomerKey:
@@ -2038,6 +2065,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSignatureDoesNotMatch
 	case hash.SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
+	case hash.ChecksumMismatch:
+		apiErr = ErrContentChecksumMismatch
 	case ObjectTooLarge:
 		apiErr = ErrEntityTooLarge
 	case ObjectTooSmall:
@@ -2066,7 +2095,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrRemoteDestinationNotFoundError
 	case BucketRemoteTargetNotFound:
 		apiErr = ErrRemoteTargetNotFoundError
-	case BucketRemoteConnectionErr:
+	case RemoteTargetConnectionErr:
 		apiErr = ErrReplicationRemoteConnectionError
 	case BucketRemoteAlreadyExists:
 		apiErr = ErrBucketRemoteAlreadyExists

cmd/api-headers.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/json"
-	"encoding/xml"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -30,6 +29,8 @@ import (
 
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	xxml "github.com/minio/xxml"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -64,9 +65,13 @@ func setCommonHeaders(w http.ResponseWriter) {
 // Encodes the response headers into XML format.
 func encodeResponse(response interface{}) []byte {
 	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
+	bytesBuffer.WriteString(xxml.Header)
+	buf, err := xxml.Marshal(response)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	bytesBuffer.Write(buf)
 	return bytesBuffer.Bytes()
 }
 
@@ -199,5 +204,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 	}
 
+	if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
+		if i := strings.LastIndexByte(v, '/'); i >= 0 {
+			v = v[i+1:]
+		}
+		w.Header()[xhttp.MinIOCompressed] = []string{v}
+	}
+
 	return nil
 }

cmd/api-response.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/handlers"
+	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 )
@@ -89,7 +90,8 @@ type ListVersionsResponse struct {
 	IsTruncated bool
 
 	CommonPrefixes []CommonPrefix
-	Versions       []ObjectVersion
+	DeleteMarkers  []DeleteMarkerVersion `xml:"DeleteMarker,omitempty"`
+	Versions       []ObjectVersion       `xml:"Version,omitempty"`
 
 	// Encoding type used to encode object keys in the response.
 	EncodingType string `xml:"EncodingType,omitempty"`
@@ -162,6 +164,12 @@ type Part struct {
 	LastModified string
 	ETag         string
 	Size         int64
+
+	// Checksum values
+	ChecksumCRC32  string
+	ChecksumCRC32C string
+	ChecksumSHA1   string
+	ChecksumSHA256 string
 }
 
 // ListPartsResponse - format for list parts response.
@@ -183,6 +191,7 @@ type ListPartsResponse struct {
 	MaxParts             int
 	IsTruncated          bool
 
+	ChecksumAlgorithm string
 	// List of parts.
 	Parts []Part `xml:"Part"`
 }
@@ -247,50 +256,80 @@ type ObjectVersion struct {
 	Object
 	IsLatest  bool
 	VersionID string `xml:"VersionId"`
-
-	isDeleteMarker bool
 }
 
-// MarshalXML - marshal ObjectVersion
-func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	if o.isDeleteMarker {
-		start.Name.Local = "DeleteMarker"
-	} else {
-		start.Name.Local = "Version"
+// DeleteMarkerVersion container for delete marker metadata
+type DeleteMarkerVersion struct {
+	Key          string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+
+	// Owner of the object.
+	Owner Owner
+
+	IsLatest  bool
+	VersionID string `xml:"VersionId"`
+}
+
+// Metadata metadata items implemented to ensure XML marshaling works.
+type Metadata struct {
+	Items []struct {
+		Key   string
+		Value string
 	}
-
-	type objectVersionWrapper ObjectVersion
-	return e.EncodeElement(objectVersionWrapper(o), start)
 }
 
-// StringMap is a map[string]string
-type StringMap map[string]string
+// Set add items, duplicate items get replaced.
+func (s *Metadata) Set(k, v string) {
+	for i, item := range s.Items {
+		if item.Key == k {
+			s.Items[i] = struct {
+				Key   string
+				Value string
+			}{
+				Key:   k,
+				Value: v,
+			}
+			return
+		}
+	}
+	s.Items = append(s.Items, struct {
+		Key   string
+		Value string
+	}{
+		Key:   k,
+		Value: v,
+	})
+}
+
+type xmlKeyEntry struct {
+	XMLName xml.Name
+	Value   string `xml:",chardata"`
+}
 
 // MarshalXML - StringMap marshals into XML.
-func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	tokens := []xml.Token{start}
-
-	for key, value := range s {
-		t := xml.StartElement{}
-		t.Name = xml.Name{
-			Space: "",
-			Local: key,
-		}
-		tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name})
+func (s *Metadata) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if s == nil {
+		return nil
 	}
 
-	tokens = append(tokens, xml.EndElement{
-		Name: start.Name,
-	})
+	if len(s.Items) == 0 {
+		return nil
+	}
 
-	for _, t := range tokens {
-		if err := e.EncodeToken(t); err != nil {
+	if err := e.EncodeToken(start); err != nil {
+		return err
+	}
+
+	for _, item := range s.Items {
+		if err := e.Encode(xmlKeyEntry{
+			XMLName: xml.Name{Local: item.Key},
+			Value:   item.Value,
+		}); err != nil {
 			return err
 		}
 	}
 
-	// flush to ensure tokens are written
-	return e.Flush()
+	return e.EncodeToken(start.End())
 }
 
 // Object container for object metadata
@@ -307,7 +346,7 @@ type Object struct {
 	StorageClass string
 
 	// UserMetadata user-defined metadata
-	UserMetadata StringMap `xml:"UserMetadata,omitempty"`
+	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -350,6 +389,11 @@ type CompleteMultipartUploadResponse struct {
 	Bucket   string
 	Key      string
 	ETag     string
+
+	ChecksumCRC32  string
+	ChecksumCRC32C string
+	ChecksumSHA1   string
+	ChecksumSHA256 string
 }
 
 // DeleteError structure.
@@ -438,6 +482,8 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
+	deleteMarkers := make([]DeleteMarkerVersion, 0, len(resp.Objects))
+
 	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
@@ -445,10 +491,25 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 	data := ListVersionsResponse{}
 
 	for _, object := range resp.Objects {
-		content := ObjectVersion{}
 		if object.Name == "" {
 			continue
 		}
+
+		if object.DeleteMarker {
+			deleteMarker := DeleteMarkerVersion{
+				Key:          s3EncodeName(object.Name, encodingType),
+				LastModified: object.ModTime.UTC().Format(iso8601TimeFormat),
+				Owner:        owner,
+				VersionID:    object.VersionID,
+			}
+			if deleteMarker.VersionID == "" {
+				deleteMarker.VersionID = nullVersionID
+			}
+			deleteMarker.IsLatest = object.IsLatest
+			deleteMarkers = append(deleteMarkers, deleteMarker)
+			continue
+		}
+		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
@@ -466,12 +527,12 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 			content.VersionID = nullVersionID
 		}
 		content.IsLatest = object.IsLatest
-		content.isDeleteMarker = object.DeleteMarker
 		versions = append(versions, content)
 	}
 
 	data.Name = bucket
 	data.Versions = versions
+	data.DeleteMarkers = deleteMarkers
 	data.EncodingType = encodingType
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.KeyMarker = s3EncodeName(marker, encodingType)
@@ -569,14 +630,14 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		}
 		content.Owner = owner
 		if metadata {
-			content.UserMetadata = make(StringMap)
+			content.UserMetadata = &Metadata{}
 			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
 			case crypto.S3:
-				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 			case crypto.S3KMS:
-				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
 			case crypto.SSEC:
-				content.UserMetadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
 			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
 				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
@@ -588,7 +649,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 					continue
 				}
-				content.UserMetadata[k] = v
+				content.UserMetadata.Set(k, v)
 			}
 		}
 		contents = append(contents, content)
@@ -642,14 +703,20 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
 }
 
 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
-func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
-	return CompleteMultipartUploadResponse{
+func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
+	cs := oi.decryptChecksums()
+	c := CompleteMultipartUploadResponse{
 		Location: location,
 		Bucket:   bucket,
 		Key:      key,
 		// AWS S3 quotes the ETag in XML, make sure we are compatible here.
-		ETag: "\"" + etag + "\"",
+		ETag:           "\"" + oi.ETag + "\"",
+		ChecksumSHA1:   cs[hash.ChecksumSHA1.String()],
+		ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
+		ChecksumCRC32:  cs[hash.ChecksumCRC32.String()],
+		ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
 	}
+	return c
 }
 
 // generates ListPartsResponse from ListPartsInfo.
@@ -674,6 +741,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
 	listPartsResponse.IsTruncated = partsInfo.IsTruncated
 	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
+	listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm
 
 	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
 	for index, part := range partsInfo.Parts {
@@ -682,6 +750,10 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
 		newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
+		newPart.ChecksumCRC32 = part.ChecksumCRC32
+		newPart.ChecksumCRC32C = part.ChecksumCRC32C
+		newPart.ChecksumSHA1 = part.ChecksumSHA1
+		newPart.ChecksumSHA256 = part.ChecksumSHA256
 		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse

cmd/api-utils.go

@@ -35,8 +35,8 @@ func shouldEscape(c byte) bool {
 
 // s3URLEncode is based on Golang's url.QueryEscape() code,
 // while considering some S3 exceptions:
-//	- Avoid encoding '/' and '*'
-//	- Force encoding of '~'
+//   - Avoid encoding '/' and '*'
+//   - Force encoding of '~'
 func s3URLEncode(s string) string {
 	spaceCount, hexCount := 0, 0
 	for i := 0; i < len(s); i++ {

cmd/auth-handler.go

@@ -288,9 +288,10 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 }
 
 // Check request auth type verifies the incoming http request
-// - validates the request signature
-// - validates the policy action if anonymous tests bucket policies if any,
-//   for authenticated requests validates IAM policies.
+//   - validates the request signature
+//   - validates the policy action if anonymous tests bucket policies if any,
+//     for authenticated requests validates IAM policies.
+//
 // returns APIErrorCode if any to be replied to the client.
 func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
 	_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action, bucketName, objectName)
@@ -298,9 +299,10 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 }
 
 // Check request auth type verifies the incoming http request
-// - validates the request signature
-// - validates the policy action if anonymous tests bucket policies if any,
-//   for authenticated requests validates IAM policies.
+//   - validates the request signature
+//   - validates the policy action if anonymous tests bucket policies if any,
+//     for authenticated requests validates IAM policies.
+//
 // returns APIErrorCode if any to be replied to the client.
 // Additionally returns the accessKey used in the request, and if this request is by an admin.
 func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (cred auth.Credentials, owner bool, s3Err APIErrorCode) {

cmd/background-heal-ops.go

@@ -26,9 +26,10 @@ import (
 )
 
 // healTask represents what to heal along with options
-//   path: '/' =>  Heal disk formats along with metadata
-//   path: 'bucket/' or '/bucket/' => Heal bucket
-//   path: 'bucket/object' => Heal object
+//
+//	path: '/' =>  Heal disk formats along with metadata
+//	path: 'bucket/' or '/bucket/' => Heal bucket
+//	path: 'bucket/object' => Heal object
 type healTask struct {
 	bucket    string
 	object    string

cmd/background-newdisks-heal-ops.go

@@ -87,7 +87,7 @@ type healingTracker struct {
 // The disk ID will be validated against the loaded one.
 func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, error) {
 	if disk == nil {
-		return nil, errors.New("loadHealingTracker: nil disk given")
+		return nil, errors.New("loadHealingTracker: nil drive given")
 	}
 	diskID, err := disk.GetDiskID()
 	if err != nil {
@@ -104,7 +104,7 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 		return nil, err
 	}
 	if h.ID != diskID && h.ID != "" {
-		return nil, fmt.Errorf("loadHealingTracker: disk id mismatch expected %s, got %s", h.ID, diskID)
+		return nil, fmt.Errorf("loadHealingTracker: drive id mismatch expected %s, got %s", h.ID, diskID)
 	}
 	h.disk = disk
 	h.ID = diskID
@@ -129,7 +129,7 @@ func newHealingTracker(disk StorageAPI) *healingTracker {
 // If the tracker has been deleted an error is returned.
 func (h *healingTracker) update(ctx context.Context) error {
 	if h.disk.Healing() == nil {
-		return fmt.Errorf("healingTracker: disk %q is not marked as healing", h.ID)
+		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
 	}
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()
@@ -310,7 +310,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	// Prevent parallel erasure set healing
-	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-disk-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
+	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%s/%d/%d", endpoint, poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
 	if err != nil {
 		return err
@@ -318,8 +318,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	ctx = lkctx.Context()
 	defer locker.Unlock(lkctx.Cancel)
 
-	buckets, _ := z.ListBuckets(ctx)
-
+	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to heal all bucket metadata configuration.
 	buckets = append(buckets, BucketInfo{
@@ -338,14 +337,14 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	})
 
 	if serverDebugLog {
-		logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
+		logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
 	}
 
 	// Load healing tracker in this disk
 	tracker, err := loadHealingTracker(ctx, disk)
 	if err != nil {
 		// So someone changed the drives underneath, healing tracker missing.
-		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
+		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
 			disk, humanize.Ordinal(poolIdx+1), err))
 		tracker = newHealingTracker(disk)
 	}
@@ -370,9 +369,9 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	if tracker.ItemsFailed > 0 {
-		logger.Info("Healing disk '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
 	} else {
-		logger.Info("Healing disk '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
 	}
 
 	if serverDebugLog {

cmd/benchmark-utils_test.go

@@ -25,7 +25,7 @@ import (
 	"strconv"
 	"testing"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 // Benchmark utility functions for ObjectLayer.PutObject().
@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -85,12 +85,12 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 
 	// PutObjectPart returns etag of the object inserted.
 	// etag variable is assigned with that value.
-	var etag, uploadID string
+	var etag string
 	// get text data generated for number of bytes equal to object size.
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
+	res, err := obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -113,7 +113,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			}
 			md5hex := getMD5Hash(textPartData)
 			var partInfo PartInfo
-			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
+			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, res.UploadID, j,
 				mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/bitrot-streaming.go

@@ -180,7 +180,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s  -> %s/%s - content hash does not match - expected %s, got %s",
 			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
 		return 0, errFileCorrupt
 	}

cmd/bitrot-whole.go

@@ -38,12 +38,12 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	return len(p), nil
@@ -72,12 +72,12 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
+			logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
+		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot_test.go

@@ -20,17 +20,11 @@ package cmd
 import (
 	"context"
 	"io"
-	"io/ioutil"
-	"os"
 	"testing"
 )
 
 func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
-	tmpDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
+	tmpDir := t.TempDir()
 
 	volume := "testvol"
 	filePath := "testfile"
@@ -60,7 +54,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	writer.(io.Closer).Close()
+	if bw, ok := writer.(io.Closer); ok {
+		bw.Close()
+	}
 
 	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
@@ -76,6 +72,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if _, err = reader.ReadAt(b[:5], 30); err != nil {
 		t.Fatal(err)
 	}
+	if br, ok := reader.(io.Closer); ok {
+		br.Close()
+	}
 }
 
 func TestAllBitrotAlgorithms(t *testing.T) {

cmd/bucket-encryption-handlers.go

@@ -65,7 +65,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -155,7 +155,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -198,7 +198,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-handlers.go

@@ -67,11 +67,16 @@ const (
 // - Check if a bucket has an entry in etcd backend
 // -- If no, make an entry
 // -- If yes, check if the entry matches local IP check if we
-//    need to update the entry then proceed to update
+//
+//	need to update the entry then proceed to update
+//
 // -- If yes, check if the IP of entry matches local IP.
-//    This means entry is for this instance.
+//
+//	This means entry is for this instance.
+//
 // -- If IP of the entry doesn't match, this means entry is
-//    for another instance. Log an error to console.
+//
+//	for another instance. Log an error to console.
 func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	if len(buckets) == 0 {
 		return
@@ -201,7 +206,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -227,7 +232,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 // using the Initiate Multipart Upload request, but has not yet been
 // completed or aborted. This operation returns at most 1,000 multipart
 // uploads in the response.
-//
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
@@ -332,7 +336,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	} else {
 		// Invoke the list buckets.
 		var err error
-		bucketsInfo, err = listBuckets(ctx)
+		bucketsInfo, err = listBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -447,7 +451,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
 
 	// Before proceeding validate if bucket exists.
-	_, err := objectAPI.GetBucketInfo(ctx, bucket)
+	_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -727,11 +731,30 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
+	cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction, bucket, "")
+	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 
+	if objectLockEnabled {
+		// Creating a bucket with locking requires the user having more permissions
+		for _, action := range []iampolicy.Action{iampolicy.PutBucketObjectLockConfigurationAction, iampolicy.PutBucketVersioningAction} {
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     cred.AccessKey,
+				Groups:          cred.Groups,
+				Action:          action,
+				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				BucketName:      bucket,
+				IsOwner:         owner,
+				Claims:          cred.Claims,
+			}) {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+				return
+			}
+		}
+	}
+
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
@@ -746,7 +769,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	opts := BucketOptions{
+	// check if client is attempting to create more buckets than allowed maximum.
+	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
+		apiErr := errorCodes.ToAPIErr(ErrTooManyBuckets)
+		apiErr.Description = fmt.Sprintf("You have attempted to create %d buckets than allowed %d", currBuckets+1, maxBuckets)
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
+
+	opts := MakeBucketOptions{
 		Location:    location,
 		LockEnabled: objectLockEnabled,
 		ForceCreate: forceCreate,
@@ -765,7 +796,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
+					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+						Force:      false,
+						NoRecreate: true,
+						SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+					})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
@@ -1165,7 +1200,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1228,7 +1263,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
@@ -1305,7 +1340,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
+	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{
+		Force:      forceDelete,
+		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+	}); err != nil {
 		apiErr := toAPIError(ctx, err)
 		if _, ok := err.(BucketNotEmpty); ok {
 			if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1472,7 +1510,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle-handlers.go

@@ -62,7 +62,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -163,7 +163,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/event"
@@ -422,6 +423,36 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 	return tier, objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
 
+type auditTierOp struct {
+	Tier             string `json:"tier"`
+	TimeToResponseNS int64  `json:"timeToResponseNS"`
+	OutputBytes      int64  `json:"tx,omitempty"`
+	Error            string `json:"error,omitempty"`
+}
+
+func auditTierActions(ctx context.Context, tier string, bytes int64) func(err error) {
+	startTime := time.Now()
+	return func(err error) {
+		// Record only when audit targets configured.
+		if len(logger.AuditTargets()) == 0 {
+			return
+		}
+
+		op := auditTierOp{
+			Tier:        tier,
+			OutputBytes: bytes,
+		}
+
+		if err == nil {
+			op.TimeToResponseNS = time.Since(startTime).Nanoseconds()
+		} else {
+			op.Error = err.Error()
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("tierStats", op)
+	}
+}
+
 // getTransitionedObjectReader returns a reader from the transitioned tier.
 func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	tgtClient, err := globalTierConfigMgr.getDriver(oi.TransitionedObject.Tier)
@@ -441,12 +472,13 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
 		gopts.length = length
 	}
 
+	timeTierAction := auditTierActions(ctx, oi.TransitionedObject.Tier, length)
 	reader, err := tgtClient.Get(ctx, oi.TransitionedObject.Name, remoteVersionID(oi.TransitionedObject.VersionID), gopts)
 	if err != nil {
 		return nil, err
 	}
 	closer := func() {
-		reader.Close()
+		timeTierAction(reader.Close())
 	}
 	return fn(reader, h, closer)
 }
@@ -560,7 +592,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 	}
 	// Check if bucket exists.
 	if !r.OutputLocation.IsEmpty() {
-		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
+		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName, BucketOptions{}); err != nil {
 			return err
 		}
 		if r.OutputLocation.S3.Prefix == "" {
@@ -755,7 +787,7 @@ func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) {
 		if strings.TrimSpace(expiryTokens[0]) != "expiry-date" {
 			return restoreObjStatus{}, errRestoreHDRMalformed
 		}
-		expiry, err := time.Parse(http.TimeFormat, strings.Trim(expiryTokens[1], `"`))
+		expiry, err := amztime.ParseHeader(strings.Trim(expiryTokens[1], `"`))
 		if err != nil {
 			return restoreObjStatus{}, errRestoreHDRMalformed
 		}

cmd/bucket-listobjects-handlers.go

@@ -32,9 +32,9 @@ import (
 // Validate all the ListObjects query arguments, returns an APIErrorCode
 // if one of the args do not meet the required conditions.
 // Special conditions required by MinIO server are as below
-// - delimiter if set should be equal to '/', otherwise the request is rejected.
-// - marker if set should have a common prefix with 'prefix' param, otherwise
-//   the request is rejected.
+//   - delimiter if set should be equal to '/', otherwise the request is rejected.
+//   - marker if set should have a common prefix with 'prefix' param, otherwise
+//     the request is rejected.
 func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
 	// Max keys cannot be negative.
 	if maxKeys < 0 {
@@ -298,7 +298,6 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 // This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
-//
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 

cmd/bucket-metadata-sys.go

@@ -45,6 +45,14 @@ type BucketMetadataSys struct {
 	metadataMap map[string]BucketMetadata
 }
 
+// Count returns number of bucket metadata map entries.
+func (sys *BucketMetadataSys) Count() int {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	return len(sys.metadataMap)
+}
+
 // Remove bucket metadata from memory.
 func (sys *BucketMetadataSys) Remove(bucket string) {
 	if globalIsGateway {
@@ -452,7 +460,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			sys.metadataMap[buckets[index].Name] = meta
 			sys.Unlock()
 
-			globalNotificationSys.set(buckets[index], meta) // set notification targets
+			globalEventNotifier.set(buckets[index], meta) // set notification targets
 
 			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
 

cmd/bucket-metadata.go

@@ -105,10 +105,8 @@ type BucketMetadata struct {
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
 func newBucketMetadata(name string) BucketMetadata {
-	now := UTCNow()
 	return BucketMetadata{
-		Name:    name,
-		Created: now,
+		Name: name,
 		notificationConfig: &event.Config{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
@@ -121,6 +119,17 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }
 
+// SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
+// creation time of bucket on this cluster in all other cases.
+func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
+	if b.Created.IsZero() {
+		b.Created = UTCNow()
+	}
+	if !createdAt.IsZero() {
+		b.Created = createdAt.UTC()
+	}
+}
+
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
@@ -160,17 +169,19 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
-
+	if err == nil {
+		b.defaultTimestamps()
+	}
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
 	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
 		return b, err
 	}
 
 	// migrate unencrypted remote targets
-	if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
+	if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
 		return b, err
 	}
-	b.defaultTimestamps()
+
 	return b, nil
 }
 
@@ -345,6 +356,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.BucketTargetsConfigJSON = configData
 		}
 	}
+	b.defaultTimestamps()
 
 	if err := b.Save(ctx, objectAPI); err != nil {
 		return err

cmd/bucket-notification-handlers.go

@@ -60,7 +60,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -72,7 +72,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 	config.SetRegion(globalSite.Region)
-	if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
+	if err = config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
 		if ok {
 			for i, queue := range config.QueueList {
@@ -132,7 +132,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objectAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -144,7 +144,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalEventNotifier.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
@@ -166,7 +166,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	}
 
 	rulesMap := config.ToRulesMap()
-	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
+	globalEventNotifier.AddRulesMap(bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/bucket-object-lock.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"math"
 	"net/http"
 
@@ -46,11 +47,14 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
 
 	config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
 	if err != nil {
-		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
+		if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucketName}) {
 			return r, nil
 		}
+		if errors.Is(err, errInvalidArgument) {
+			return r, err
+		}
+		logger.CriticalIf(context.Background(), err)
 		return r, err
-
 	}
 	return config.ToRetention(), nil
 }

cmd/bucket-policy-handlers.go

@@ -61,7 +61,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -191,7 +191,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers_test.go

@@ -128,7 +128,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
 			defer wg.Done()
 			// Sync start.
 			<-start
-			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 				if _, ok := err.(BucketExists); !ok {
 					t.Logf("unexpected error: %T: %v", err, err)
 					return
@@ -163,7 +163,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T,
 ) {
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 

cmd/bucket-policy.go

@@ -169,6 +169,8 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	}
 
 	// JWT specific values
+	//
+	// Add all string claims
 	for k, v := range claims {
 		vStr, ok := v.(string)
 		if ok {
@@ -183,6 +185,21 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 			}
 		}
 	}
+	// Add groups claim which could be a list. This will ensure that the claim
+	// `jwt:groups` works.
+	if grpsVal, ok := claims["groups"]; ok {
+		if grpsIs, ok := grpsVal.([]interface{}); ok {
+			grps := []string{}
+			for _, gI := range grpsIs {
+				if g, ok := gI.(string); ok {
+					grps = append(grps, g)
+				}
+			}
+			if len(grps) > 0 {
+				args["groups"] = grps
+			}
+		}
+	}
 
 	return args
 }

cmd/bucket-replication-handlers.go

@@ -55,7 +55,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -118,7 +118,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -157,7 +157,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -198,7 +198,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -217,7 +217,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
 	enc := json.NewEncoder(w)
-	if err = enc.Encode(getLatestReplicationStats(bucket, usageInfo)); err != nil {
+	if err = enc.Encode(globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -267,7 +267,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -310,7 +310,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 	rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
 	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
 		switch err.(type) {
-		case BucketRemoteConnectionErr:
+		case RemoteTargetConnectionErr:
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -356,7 +356,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-replication-stats.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"math"
 	"sync"
 	"time"
 
@@ -36,10 +37,12 @@ func (b *BucketReplicationStats) hasReplicationUsage() bool {
 
 // ReplicationStats holds the global in-memory replication stats
 type ReplicationStats struct {
-	Cache      map[string]*BucketReplicationStats
-	UsageCache map[string]*BucketReplicationStats
-	sync.RWMutex
-	ulock sync.RWMutex
+	Cache           map[string]*BucketReplicationStats
+	UsageCache      map[string]*BucketReplicationStats
+	mostRecentStats BucketStatsMap
+	sync.RWMutex                 // mutex for Cache
+	ulock           sync.RWMutex // mutex for UsageCache
+	dlock           sync.RWMutex // mutex for mostRecentStats
 }
 
 // Delete deletes in-memory replication statistics for a bucket.
@@ -92,14 +95,23 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 		bs.Stats[arn] = b
 	}
 	switch status {
+	case replication.Pending:
+		if opType.IsDataReplication() && prevStatus != status {
+			b.PendingSize += n
+			b.PendingCount++
+		}
 	case replication.Completed:
 		switch prevStatus { // adjust counters based on previous state
+		case replication.Pending:
+			b.PendingCount--
 		case replication.Failed:
 			b.FailedCount--
 		}
-		if opType == replication.ObjectReplicationType {
+		if opType.IsDataReplication() {
 			b.ReplicatedSize += n
 			switch prevStatus {
+			case replication.Pending:
+				b.PendingSize -= n
 			case replication.Failed:
 				b.FailedSize -= n
 			}
@@ -108,10 +120,12 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 			}
 		}
 	case replication.Failed:
-		if opType == replication.ObjectReplicationType {
+		if opType.IsDataReplication() {
 			if prevStatus == replication.Pending {
 				b.FailedSize += n
 				b.FailedCount++
+				b.PendingSize -= n
+				b.PendingCount--
 			}
 		}
 	case replication.Replica:
@@ -176,9 +190,20 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 	}
 }
 
-// load replication metrics at cluster start from initial data usage
+// load replication metrics at cluster start from latest replication stats saved in .minio.sys/buckets/replication/node-name.stats
+// fallback to replication stats in data usage to be backward compatible
 func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
-	rTimer := time.NewTimer(time.Minute)
+	m := make(map[string]*BucketReplicationStats)
+	if stats, err := globalReplicationPool.loadStatsFromDisk(); err == nil {
+		for b, st := range stats {
+			m[b] = &st
+		}
+		r.ulock.Lock()
+		r.UsageCache = m
+		r.ulock.Unlock()
+		return
+	}
+	rTimer := time.NewTimer(time.Second * 5)
 	defer rTimer.Stop()
 	var (
 		dui DataUsageInfo
@@ -192,15 +217,12 @@ outer:
 		case <-rTimer.C:
 			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
 			// If LastUpdate is set, data usage is available.
-			if err == nil && !dui.LastUpdate.IsZero() {
+			if err == nil {
 				break outer
 			}
-
-			rTimer.Reset(time.Minute)
+			rTimer.Reset(time.Second * 5)
 		}
 	}
-
-	m := make(map[string]*BucketReplicationStats)
 	for bucket, usage := range dui.BucketsUsage {
 		b := &BucketReplicationStats{
 			Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
@@ -219,6 +241,130 @@ outer:
 		}
 	}
 	r.ulock.Lock()
-	defer r.ulock.Unlock()
 	r.UsageCache = m
+	r.ulock.Unlock()
+}
+
+func (r *ReplicationStats) getAllCachedLatest() BucketStatsMap {
+	r.dlock.RLock()
+	defer r.dlock.RUnlock()
+	return r.mostRecentStats
+}
+
+func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
+	peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
+	bucketsReplicationStats = make(map[string]BucketReplicationStats, len(bucketsUsage))
+
+	for bucket, u := range bucketsUsage {
+		bucketStats := make([]BucketStats, len(peerBucketStatsList))
+		for i, peerBucketStats := range peerBucketStatsList {
+			bucketStat, ok := peerBucketStats.Stats[bucket]
+			if !ok {
+				continue
+			}
+			bucketStats[i] = bucketStat
+		}
+		bucketsReplicationStats[bucket] = r.calculateBucketReplicationStats(bucket, u, bucketStats)
+	}
+	return bucketsReplicationStats
+}
+
+func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u BucketUsageInfo, bucketStats []BucketStats) (s BucketReplicationStats) {
+	if r == nil {
+		s = BucketReplicationStats{
+			Stats: make(map[string]*BucketReplicationStat),
+		}
+		return s
+	}
+
+	// accumulate cluster bucket stats
+	stats := make(map[string]*BucketReplicationStat)
+	var totReplicaSize int64
+	for _, bucketStat := range bucketStats {
+		totReplicaSize += bucketStat.ReplicationStats.ReplicaSize
+		for arn, stat := range bucketStat.ReplicationStats.Stats {
+			oldst := stats[arn]
+			if oldst == nil {
+				oldst = &BucketReplicationStat{}
+			}
+			stats[arn] = &BucketReplicationStat{
+				FailedCount:    stat.FailedCount + oldst.FailedCount,
+				FailedSize:     stat.FailedSize + oldst.FailedSize,
+				ReplicatedSize: stat.ReplicatedSize + oldst.ReplicatedSize,
+				Latency:        stat.Latency.merge(oldst.Latency),
+				PendingCount:   stat.PendingCount + oldst.PendingCount,
+				PendingSize:    stat.PendingSize + oldst.PendingSize,
+			}
+		}
+	}
+
+	// add initial usage stat to cluster stats
+	usageStat := globalReplicationStats.GetInitialUsage(bucket)
+
+	totReplicaSize += usageStat.ReplicaSize
+	for arn, stat := range usageStat.Stats {
+		st, ok := stats[arn]
+		if !ok {
+			st = &BucketReplicationStat{}
+			stats[arn] = st
+		}
+		st.ReplicatedSize += stat.ReplicatedSize
+		st.FailedSize += stat.FailedSize
+		st.FailedCount += stat.FailedCount
+		st.PendingSize += stat.PendingSize
+		st.PendingCount += stat.PendingCount
+	}
+
+	s = BucketReplicationStats{
+		Stats: make(map[string]*BucketReplicationStat, len(stats)),
+	}
+	var latestTotReplicatedSize int64
+	for _, st := range u.ReplicationInfo {
+		latestTotReplicatedSize += int64(st.ReplicatedSize)
+	}
+
+	// normalize computed real time stats with latest usage stat
+	for arn, tgtstat := range stats {
+		st := BucketReplicationStat{}
+		bu, ok := u.ReplicationInfo[arn]
+		if !ok {
+			bu = BucketTargetUsageInfo{}
+		}
+		// use in memory replication stats if it is ahead of usage info.
+		st.ReplicatedSize = int64(bu.ReplicatedSize)
+		if tgtstat.ReplicatedSize >= int64(bu.ReplicatedSize) {
+			st.ReplicatedSize = tgtstat.ReplicatedSize
+		}
+		s.ReplicatedSize += st.ReplicatedSize
+		// Reset FailedSize and FailedCount to 0 for negative overflows which can
+		// happen since data usage picture can lag behind actual usage state at the time of cluster start
+		st.FailedSize = int64(math.Max(float64(tgtstat.FailedSize), 0))
+		st.FailedCount = int64(math.Max(float64(tgtstat.FailedCount), 0))
+		st.PendingSize = int64(math.Max(float64(tgtstat.PendingSize), 0))
+		st.PendingCount = int64(math.Max(float64(tgtstat.PendingCount), 0))
+		st.Latency = tgtstat.Latency
+
+		s.Stats[arn] = &st
+		s.FailedSize += st.FailedSize
+		s.FailedCount += st.FailedCount
+		s.PendingCount += st.PendingCount
+		s.PendingSize += st.PendingSize
+	}
+	// normalize overall stats
+	s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
+	s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
+	r.dlock.Lock()
+	if len(r.mostRecentStats.Stats) == 0 {
+		r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
+	}
+	r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
+	r.mostRecentStats.Timestamp = UTCNow()
+	r.dlock.Unlock()
+	return s
+}
+
+// get the most current of in-memory replication stats  and data usage info from crawler.
+func (r *ReplicationStats) getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
+	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
+	return r.calculateBucketReplicationStats(bucket, u, bucketStats)
 }

cmd/bucket-replication-utils.go

@@ -737,3 +737,44 @@ func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) {
 	opts.Prefix = q.Get("prefix")
 	return
 }
+
+const (
+	replicationMRFDir = bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "mrf"
+	mrfMetaFormat     = 1
+	mrfMetaVersionV1  = 1
+	mrfMetaVersion    = mrfMetaVersionV1
+)
+
+// MRFReplicateEntry mrf entry to save to disk
+type MRFReplicateEntry struct {
+	Bucket    string `json:"bucket" msg:"b"`
+	Object    string `json:"object" msg:"o"`
+	versionID string `json:"-"`
+}
+
+// MRFReplicateEntries has the map of MRF entries to save to disk
+type MRFReplicateEntries struct {
+	Entries map[string]MRFReplicateEntry `json:"entries" msg:"e"`
+	Version int                          `json:"version" msg:"v"`
+}
+
+// ToMRFEntry returns the relevant info needed by MRF
+func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
+	return MRFReplicateEntry{
+		Bucket:    ri.Bucket,
+		Object:    ri.Name,
+		versionID: ri.VersionID,
+	}
+}
+
+func getReplicationStatsPath(nodeName string) string {
+	nodeStr := strings.ReplaceAll(nodeName, ":", "_")
+	return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + nodeStr + ".stats"
+}
+
+const (
+	replStatsMetaFormat   = 1
+	replStatsVersionV1    = 1
+	replStatsVersion      = replStatsVersionV1
+	replStatsSaveInterval = time.Minute * 5
+)

cmd/bucket-replication-utils_gen.go

@@ -260,6 +260,413 @@ func (z *BucketReplicationResyncStatus) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "e":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Entries")
+				return
+			}
+			if z.Entries == nil {
+				z.Entries = make(map[string]MRFReplicateEntry, zb0002)
+			} else if len(z.Entries) > 0 {
+				for key := range z.Entries {
+					delete(z.Entries, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 MRFReplicateEntry
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Entries")
+					return
+				}
+				var zb0003 uint32
+				zb0003, err = dc.ReadMapHeader()
+				if err != nil {
+					err = msgp.WrapError(err, "Entries", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						err = msgp.WrapError(err, "Entries", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "b":
+						za0002.Bucket, err = dc.ReadString()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+							return
+						}
+					case "o":
+						za0002.Object, err = dc.ReadString()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Object")
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001)
+							return
+						}
+					}
+				}
+				z.Entries[za0001] = za0002
+			}
+		case "v":
+			z.Version, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "e"
+	err = en.Append(0x82, 0xa1, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Entries)))
+	if err != nil {
+		err = msgp.WrapError(err, "Entries")
+		return
+	}
+	for za0001, za0002 := range z.Entries {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries")
+			return
+		}
+		// map header, size 2
+		// write "b"
+		err = en.Append(0x82, 0xa1, 0x62)
+		if err != nil {
+			return
+		}
+		err = en.WriteString(za0002.Bucket)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+			return
+		}
+		// write "o"
+		err = en.Append(0xa1, 0x6f)
+		if err != nil {
+			return
+		}
+		err = en.WriteString(za0002.Object)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries", za0001, "Object")
+			return
+		}
+	}
+	// write "v"
+	err = en.Append(0xa1, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Version)
+	if err != nil {
+		err = msgp.WrapError(err, "Version")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "e"
+	o = append(o, 0x82, 0xa1, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
+	for za0001, za0002 := range z.Entries {
+		o = msgp.AppendString(o, za0001)
+		// map header, size 2
+		// string "b"
+		o = append(o, 0x82, 0xa1, 0x62)
+		o = msgp.AppendString(o, za0002.Bucket)
+		// string "o"
+		o = append(o, 0xa1, 0x6f)
+		o = msgp.AppendString(o, za0002.Object)
+	}
+	// string "v"
+	o = append(o, 0xa1, 0x76)
+	o = msgp.AppendInt(o, z.Version)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "e":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Entries")
+				return
+			}
+			if z.Entries == nil {
+				z.Entries = make(map[string]MRFReplicateEntry, zb0002)
+			} else if len(z.Entries) > 0 {
+				for key := range z.Entries {
+					delete(z.Entries, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 MRFReplicateEntry
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Entries")
+					return
+				}
+				var zb0003 uint32
+				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Entries", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Entries", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "b":
+						za0002.Bucket, bts, err = msgp.ReadStringBytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+							return
+						}
+					case "o":
+						za0002.Object, bts, err = msgp.ReadStringBytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Object")
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001)
+							return
+						}
+					}
+				}
+				z.Entries[za0001] = za0002
+			}
+		case "v":
+			z.Version, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *MRFReplicateEntries) Msgsize() (s int) {
+	s = 1 + 2 + msgp.MapHeaderSize
+	if z.Entries != nil {
+		for za0001, za0002 := range z.Entries {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object)
+		}
+	}
+	s += 2 + msgp.IntSize
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "b":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "o":
+			z.Object, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "b"
+	err = en.Append(0x82, 0xa1, 0x62)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "o"
+	err = en.Append(0xa1, 0x6f)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Object)
+	if err != nil {
+		err = msgp.WrapError(err, "Object")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "b"
+	o = append(o, 0x82, 0xa1, 0x62)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "o"
+	o = append(o, 0xa1, 0x6f)
+	o = msgp.AppendString(o, z.Object)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "b":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "o":
+			z.Object, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z MRFReplicateEntry) Msgsize() (s int) {
+	s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object)
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/bucket-replication-utils_gen_test.go

@@ -122,6 +122,232 @@ func BenchmarkDecodeBucketReplicationResyncStatus(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalMRFReplicateEntries(t *testing.T) {
+	v := MRFReplicateEntries{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeMRFReplicateEntries(t *testing.T) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeMRFReplicateEntries Msgsize() is inaccurate")
+	}
+
+	vn := MRFReplicateEntries{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalMRFReplicateEntry(t *testing.T) {
+	v := MRFReplicateEntry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeMRFReplicateEntry(t *testing.T) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeMRFReplicateEntry Msgsize() is inaccurate")
+	}
+
+	vn := MRFReplicateEntry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
 	v := ReplicateDecision{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-replication.go

@@ -23,7 +23,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
 	"path"
 	"reflect"
@@ -82,6 +81,12 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 	}
 
 	rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
+	if err != nil {
+		if errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucketName}) || errors.Is(err, errInvalidArgument) {
+			return rCfg, err
+		}
+		logger.CriticalIf(ctx, err)
+	}
 	return rCfg, err
 }
 
@@ -197,7 +202,7 @@ func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplica
 
 	// Disable server-side replication on object prefixes which are excluded
 	// from versioning via the MinIO bucket versioning extension.
-	if globalBucketVersioningSys.PrefixSuspended(bucket, object) {
+	if !globalBucketVersioningSys.PrefixEnabled(bucket, object) {
 		return
 	}
 
@@ -403,6 +408,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
+		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
 		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, rcfg.RoleArn))
 		sendEvent(eventArgs{
 			BucketName: bucket,
@@ -477,6 +483,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	eventName := event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
+		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
 	}
 	drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
 	if replicationStatus != prevStatus {
@@ -488,7 +495,11 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		MTime:             dobj.DeleteMarkerMTime.Time,
 		DeleteReplication: drs,
 		Versioned:         globalBucketVersioningSys.PrefixEnabled(bucket, dobj.ObjectName),
-		VersionSuspended:  globalBucketVersioningSys.PrefixSuspended(bucket, dobj.ObjectName),
+		// Objects matching prefixes should not leave delete markers,
+		// dramatically reduces namespace pollution while keeping the
+		// benefits of replication, make sure to apply version suspension
+		// only at bucket level instead.
+		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
 		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
@@ -534,7 +545,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	if dobj.VersionID != "" && rinfo.VersionPurgeStatus == Complete {
 		return
 	}
-	if tgt.IsOffline() {
+	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN))
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
@@ -910,6 +921,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			Object:     objInfo,
 			Host:       "Internal: [Replication]",
 		})
+		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, cfg.RoleArn))
 		return
 	}
@@ -934,7 +946,12 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		wg.Add(1)
 		go func(index int, tgt *TargetClient) {
 			defer wg.Done()
-			rinfos.Targets[index] = replicateObjectToTarget(ctx, ri, objectAPI, tgt)
+			if ri.OpType == replication.ObjectReplicationType {
+				// all incoming calls go through optimized path.
+				rinfos.Targets[index] = ri.replicateObject(ctx, objectAPI, tgt)
+			} else {
+				rinfos.Targets[index] = ri.replicateAll(ctx, objectAPI, tgt)
+			}
 		}(i, tgt)
 	}
 	wg.Wait()
@@ -992,39 +1009,25 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 
 	// re-queue failures once more - keep a retry count to avoid flooding the queue if
 	// the target site is down. Leave it to scanner to catch up instead.
-	if rinfos.ReplicationStatus() != replication.Completed && ri.RetryCount < 1 {
+	if rinfos.ReplicationStatus() != replication.Completed {
 		ri.OpType = replication.HealReplicationType
 		ri.EventType = ReplicateMRF
 		ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal()
 		ri.RetryCount++
-		globalReplicationPool.queueReplicaFailedTask(ri)
+		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 	}
 }
 
-// replicateObjectToTarget replicates the specified version of the object to destination bucket
+// replicateObject replicates object data for specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
-func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
 	startTime := time.Now()
 	objInfo := ri.ObjectInfo.Clone()
 	bucket := objInfo.Bucket
 	object := objInfo.Name
-	var (
-		closeOnDefer bool
-		gr           *GetObjectReader
-		size         int64
-		err          error
-	)
 	sz, _ := objInfo.GetActualSize()
-	// set defaults for replication action based on operation being performed - actual
-	// replication action can only be determined after stat on remote. This default is
-	// needed for updating replication metrics correctly when target is offline.
-	var rAction replicationAction
-	switch ri.OpType {
-	case replication.MetadataReplicationType:
-		rAction = replicateMetadata
-	default:
-		rAction = replicateAll
-	}
+
+	rAction := replicateAll
 	rinfo = replicatedTargetInfo{
 		Size:                  sz,
 		Arn:                   tgt.ARN,
@@ -1039,7 +1042,8 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 		rinfo.ReplicationResynced = true
 		return
 	}
-	if tgt.IsOffline() {
+
+	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
@@ -1053,30 +1057,176 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 
-	gr, err = objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
 		VersionID:        objInfo.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
 	})
 	if err != nil {
+		if !isErrObjectNotFound(err) {
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				Host:       "Internal: [Replication]",
+			})
+			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+		}
+		return
+	}
+	defer gr.Close()
+
+	objInfo = gr.ObjInfo
+
+	size, err := objInfo.GetActualSize()
+	if err != nil {
+		logger.LogIf(ctx, err)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
 			Host:       "Internal: [Replication]",
 		})
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replicate for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		return
 	}
+
+	if tgt.Bucket == "" {
+		logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
+		return rinfo
+	}
 	defer func() {
-		if closeOnDefer {
-			gr.Close()
+		if rinfo.ReplicationStatus == replication.Completed && ri.OpType == replication.ExistingObjectReplicationType && tgt.ResetID != "" {
+			rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID)
+			rinfo.ReplicationResynced = true
 		}
+		rinfo.Duration = time.Since(startTime)
 	}()
-	closeOnDefer = true
+
+	rinfo.ReplicationStatus = replication.Completed
+	rinfo.Size = size
+	rinfo.ReplicationAction = rAction
+	// use core client to avoid doing multipart on PUT
+	c := &miniogo.Core{Client: tgt.Client}
+
+	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s err:%w", bucket, err))
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
+		return
+	}
+
+	var headerSize int
+	for k, v := range putOpts.Header() {
+		headerSize += len(k) + len(v)
+	}
+
+	opts := &bandwidth.MonitorReaderOptions{
+		Bucket:     objInfo.Bucket,
+		HeaderSize: headerSize,
+	}
+	newCtx := ctx
+	if globalBucketMonitor.IsThrottled(bucket) {
+		var cancel context.CancelFunc
+		newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
+		defer cancel()
+	}
+	r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
+	if objInfo.isMultipart() {
+		if err := replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
+			r, objInfo, putOpts); err != nil {
+			if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+				rinfo.ReplicationStatus = replication.Failed
+				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+			}
+		}
+	} else {
+		if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
+			if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+				rinfo.ReplicationStatus = replication.Failed
+				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+			}
+		}
+	}
+	return
+}
+
+// replicateAll replicates metadata for specified version of the object to destination bucket
+// if the destination version is missing it automatically does fully copy as well.
+// The source object is then updated to reflect the replication status.
+func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+	startTime := time.Now()
+	objInfo := ri.ObjectInfo.Clone()
+	bucket := objInfo.Bucket
+	object := objInfo.Name
+	sz, _ := objInfo.GetActualSize()
+
+	// set defaults for replication action based on operation being performed - actual
+	// replication action can only be determined after stat on remote. This default is
+	// needed for updating replication metrics correctly when target is offline.
+	rAction := replicateMetadata
+
+	rinfo = replicatedTargetInfo{
+		Size:                  sz,
+		Arn:                   tgt.ARN,
+		PrevReplicationStatus: objInfo.TargetReplicationStatus(tgt.ARN),
+		ReplicationStatus:     replication.Failed,
+		OpType:                ri.OpType,
+		ReplicationAction:     rAction,
+	}
+
+	if ri.ObjectInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
+		rinfo.ReplicationStatus = replication.Completed
+		rinfo.ReplicationResynced = true
+		return
+	}
+
+	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
+		return
+	}
+
+	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
+
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+		VersionID:        objInfo.VersionID,
+		Versioned:        versioned,
+		VersionSuspended: versionSuspended,
+	})
+	if err != nil {
+		if !isErrObjectNotFound(err) {
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				Host:       "Internal: [Replication]",
+			})
+			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+		}
+		return
+	}
+	defer gr.Close()
 
 	objInfo = gr.ObjInfo
-	size, err = objInfo.GetActualSize()
+
+	size, err := objInfo.GetActualSize()
 	if err != nil {
 		logger.LogIf(ctx, err)
 		sendEvent(eventArgs{
@@ -1135,8 +1285,6 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 				// as Completed.
 				//
 				// Note: Replication Stats would have been updated despite metadata update failure.
-				gr.Close()
-				closeOnDefer = false
 				rinfo.ReplicationAction = rAction
 				rinfo.ReplicationStatus = replication.Completed
 			}
@@ -1207,8 +1355,6 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 			}
 		}
 	}
-	gr.Close()
-	closeOnDefer = false
 	return
 }
 
@@ -1302,6 +1448,19 @@ type DeletedObjectReplicationInfo struct {
 	TargetArn string
 }
 
+// ToMRFEntry returns the relevant info needed by MRF
+func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry {
+	versionID := di.DeleteMarkerVersionID
+	if versionID == "" {
+		versionID = di.VersionID
+	}
+	return MRFReplicateEntry{
+		Bucket:    di.Bucket,
+		Object:    di.ObjectName,
+		versionID: versionID,
+	}
+}
+
 // Replication specific APIName
 const (
 	ReplicateObjectAPI = "ReplicateObject"
@@ -1346,13 +1505,16 @@ type ReplicationPool struct {
 	mrfReplicaCh            chan ReplicateObjectInfo
 	existingReplicaCh       chan ReplicateObjectInfo
 	existingReplicaDeleteCh chan DeletedObjectReplicationInfo
-	workerSize              int
-	mrfWorkerSize           int
-	resyncState             replicationResyncState
-	workerWg                sync.WaitGroup
-	mrfWorkerWg             sync.WaitGroup
-	once                    sync.Once
-	mu                      sync.Mutex
+	mrfSaveCh               chan MRFReplicateEntry
+	saveStateCh             chan struct{}
+
+	workerSize    int
+	mrfWorkerSize int
+	resyncState   replicationResyncState
+	workerWg      sync.WaitGroup
+	mrfWorkerWg   sync.WaitGroup
+	once          sync.Once
+	mu            sync.Mutex
 }
 
 // NewReplicationPool creates a pool of replication workers of specified size
@@ -1366,6 +1528,8 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		existingReplicaCh:       make(chan ReplicateObjectInfo, 100000),
 		existingReplicaDeleteCh: make(chan DeletedObjectReplicationInfo, 100000),
 		resyncState:             replicationResyncState{statusMap: make(map[string]BucketReplicationResyncStatus)},
+		mrfSaveCh:               make(chan MRFReplicateEntry, 100000),
+		saveStateCh:             make(chan struct{}, 1),
 		ctx:                     ctx,
 		objLayer:                o,
 	}
@@ -1374,6 +1538,9 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	pool.ResizeFailedWorkers(opts.FailedWorkers)
 	go pool.AddExistingObjectReplicateWorker()
 	go pool.updateResyncStatus(ctx, o)
+	go pool.processMRF()
+	go pool.persistMRF()
+	go pool.saveStatsToDisk()
 	return pool
 }
 
@@ -1479,33 +1646,17 @@ func (p *ReplicationPool) suggestedWorkers(failQueue bool) int {
 	return int(float64(p.workerSize) * ReplicationWorkerMultiplier)
 }
 
-func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
-	if p == nil {
-		return
-	}
-	select {
-	case <-GlobalContext.Done():
-		p.once.Do(func() {
-			close(p.replicaCh)
-			close(p.mrfReplicaCh)
-			close(p.existingReplicaCh)
-		})
-	case p.mrfReplicaCh <- ri:
-	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
-	}
-}
-
 func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
 	}
-	var ch chan ReplicateObjectInfo
+	var ch, healCh chan ReplicateObjectInfo
 	switch ri.OpType {
 	case replication.ExistingObjectReplicationType:
 		ch = p.existingReplicaCh
 	case replication.HealReplicationType:
-		fallthrough
+		ch = p.mrfReplicaCh
+		healCh = p.replicaCh
 	default:
 		ch = p.replicaCh
 	}
@@ -1515,10 +1666,13 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 			close(p.replicaCh)
 			close(p.mrfReplicaCh)
 			close(p.existingReplicaCh)
+			close(p.saveStateCh)
 		})
+	case healCh <- ri:
 	case ch <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
 	}
 }
 
@@ -1555,7 +1709,8 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		globalReplicationPool.queueMRFSave(doi.ToMRFEntry())
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
 	}
 }
 
@@ -1659,7 +1814,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 	}
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, t.Arn)
-		if tgt == nil || tgt.IsOffline() {
+		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			continue
 		}
 		// if proxying explicitly disabled on remote target
@@ -1805,6 +1960,7 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli
 	objInfo.VersionPurgeStatusInternal = ""
 	objInfo.ReplicationStatus = ""
 	objInfo.VersionPurgeStatus = ""
+	delete(objInfo.UserDefined, xhttp.AmzBucketReplicationStatus)
 	resyncdsc := mustReplicate(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(objInfo, replication.ExistingObjectReplicationType, ObjectOptions{}))
 	dsc = &resyncdsc
 	return c.resync(oi, dsc, tgtStatuses)
@@ -1873,100 +2029,6 @@ func resyncTarget(oi ObjectInfo, arn string, resetID string, resetBeforeDate tim
 	return
 }
 
-func getAllLatestReplicationStats(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
-	peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
-	bucketsReplicationStats = make(map[string]BucketReplicationStats, len(bucketsUsage))
-
-	for bucket, u := range bucketsUsage {
-		bucketStats := make([]BucketStats, len(peerBucketStatsList))
-		for i, peerBucketStats := range peerBucketStatsList {
-			bucketStat, ok := peerBucketStats[bucket]
-			if !ok {
-				continue
-			}
-			bucketStats[i] = bucketStat
-		}
-		bucketsReplicationStats[bucket] = calculateBucketReplicationStats(bucket, u, bucketStats)
-	}
-
-	return bucketsReplicationStats
-}
-
-func calculateBucketReplicationStats(bucket string, u BucketUsageInfo, bucketStats []BucketStats) (s BucketReplicationStats) {
-	// accumulate cluster bucket stats
-	stats := make(map[string]*BucketReplicationStat)
-	var totReplicaSize int64
-	for _, bucketStat := range bucketStats {
-		totReplicaSize += bucketStat.ReplicationStats.ReplicaSize
-		for arn, stat := range bucketStat.ReplicationStats.Stats {
-			oldst := stats[arn]
-			if oldst == nil {
-				oldst = &BucketReplicationStat{}
-			}
-			stats[arn] = &BucketReplicationStat{
-				FailedCount:    stat.FailedCount + oldst.FailedCount,
-				FailedSize:     stat.FailedSize + oldst.FailedSize,
-				ReplicatedSize: stat.ReplicatedSize + oldst.ReplicatedSize,
-				Latency:        stat.Latency.merge(oldst.Latency),
-			}
-		}
-	}
-
-	// add initial usage stat to cluster stats
-	usageStat := globalReplicationStats.GetInitialUsage(bucket)
-	totReplicaSize += usageStat.ReplicaSize
-	for arn, stat := range usageStat.Stats {
-		st, ok := stats[arn]
-		if !ok {
-			st = &BucketReplicationStat{}
-			stats[arn] = st
-		}
-		st.ReplicatedSize += stat.ReplicatedSize
-		st.FailedSize += stat.FailedSize
-		st.FailedCount += stat.FailedCount
-	}
-	s = BucketReplicationStats{
-		Stats: make(map[string]*BucketReplicationStat, len(stats)),
-	}
-	var latestTotReplicatedSize int64
-	for _, st := range u.ReplicationInfo {
-		latestTotReplicatedSize += int64(st.ReplicatedSize)
-	}
-	// normalize computed real time stats with latest usage stat
-	for arn, tgtstat := range stats {
-		st := BucketReplicationStat{}
-		bu, ok := u.ReplicationInfo[arn]
-		if !ok {
-			bu = BucketTargetUsageInfo{}
-		}
-		// use in memory replication stats if it is ahead of usage info.
-		st.ReplicatedSize = int64(bu.ReplicatedSize)
-		if tgtstat.ReplicatedSize >= int64(bu.ReplicatedSize) {
-			st.ReplicatedSize = tgtstat.ReplicatedSize
-		}
-		s.ReplicatedSize += st.ReplicatedSize
-		// Reset FailedSize and FailedCount to 0 for negative overflows which can
-		// happen since data usage picture can lag behind actual usage state at the time of cluster start
-		st.FailedSize = int64(math.Max(float64(tgtstat.FailedSize), 0))
-		st.FailedCount = int64(math.Max(float64(tgtstat.FailedCount), 0))
-		st.Latency = tgtstat.Latency
-
-		s.Stats[arn] = &st
-		s.FailedSize += st.FailedSize
-		s.FailedCount += st.FailedCount
-	}
-	// normalize overall stats
-	s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
-	s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
-	return s
-}
-
-// get the most current of in-memory replication stats  and data usage info from crawler.
-func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
-	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
-	return calculateBucketReplicationStats(bucket, u, bucketStats)
-}
-
 const resyncTimeInterval = time.Minute * 1
 
 // updateResyncStatus persists in-memory resync metadata stats to disk at periodic intervals
@@ -1974,25 +2036,32 @@ func (p *ReplicationPool) updateResyncStatus(ctx context.Context, objectAPI Obje
 	resyncTimer := time.NewTimer(resyncTimeInterval)
 	defer resyncTimer.Stop()
 
+	// For each bucket name, store the last timestamp of the
+	// successful save of replication status in the backend disks.
+	lastResyncStatusSave := make(map[string]time.Time)
+
 	for {
 		select {
 		case <-resyncTimer.C:
-			now := UTCNow()
 			p.resyncState.RLock()
 			for bucket, brs := range p.resyncState.statusMap {
 				var updt bool
+				// Save the replication status if one resync to any bucket target is still not finished
 				for _, st := range brs.TargetsMap {
-					// if resync in progress or just ended, needs to save to disk
-					if st.EndTime.Equal(timeSentinel) || now.Sub(st.EndTime) <= resyncTimeInterval {
+					if st.EndTime.Equal(timeSentinel) {
 						updt = true
 						break
 					}
 				}
+				// Save the replication status if a new stats update is found and not saved in the backend yet
+				if brs.LastUpdate.After(lastResyncStatusSave[bucket]) {
+					updt = true
+				}
 				if updt {
-					brs.LastUpdate = now
 					if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("Could not save resync metadata to disk for %s - %w", bucket, err))
-						continue
+						logger.LogIf(ctx, fmt.Errorf("Could not save resync metadata to drive for %s - %w", bucket, err))
+					} else {
+						lastResyncStatusSave[bucket] = brs.LastUpdate
 					}
 				}
 			}
@@ -2018,6 +2087,7 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI
 		st.EndTime = UTCNow()
 		st.ResyncStatus = resyncStatus
 		m.TargetsMap[arn] = st
+		m.LastUpdate = UTCNow()
 		globalReplicationPool.resyncState.statusMap[bucket] = m
 		globalReplicationPool.resyncState.Unlock()
 	}()
@@ -2052,9 +2122,9 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI
 		return
 	}
 
-	// Walk through all object versions - note ascending order of walk needed to ensure delete marker replicated to
-	// target after object version is first created.
-	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkAscending: true}); err != nil {
+	// Walk through all object versions - Walk() is always in ascending order needed to ensure
+	// delete marker replicated to target after object version is first created.
+	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{}); err != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
@@ -2127,6 +2197,7 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI
 			st.ReplicatedSize += roi.Size
 		}
 		m.TargetsMap[arn] = st
+		m.LastUpdate = UTCNow()
 		globalReplicationPool.resyncState.statusMap[bucket] = m
 		globalReplicationPool.resyncState.Unlock()
 	}
@@ -2385,3 +2456,369 @@ func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string,
 	}()
 	return diffCh, nil
 }
+
+// QueueReplicationHeal is a wrapper for queueReplicationHeal
+func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo) {
+	// un-versioned or a prefix
+	if oi.VersionID == "" || oi.ModTime.IsZero() {
+		return
+	}
+	rcfg, _ := getReplicationConfig(ctx, bucket)
+	tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	queueReplicationHeal(ctx, bucket, oi, replicationConfig{
+		Config:  rcfg,
+		remotes: tgts,
+	})
+}
+
+// queueReplicationHeal enqueues objects that failed replication OR eligible for resyncing through
+// an ongoing resync operation or via existing objects replication configuration setting.
+func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcfg replicationConfig) (roi ReplicateObjectInfo) {
+	// un-versioned or a prefix
+	if oi.VersionID == "" || oi.ModTime.IsZero() {
+		return roi
+	}
+
+	if rcfg.Config == nil || rcfg.remotes == nil {
+		return roi
+	}
+	roi = getHealReplicateObjectInfo(oi, rcfg)
+	if !roi.Dsc.ReplicateAny() {
+		return
+	}
+	// early return if replication already done, otherwise we need to determine if this
+	// version is an existing object that needs healing.
+	if oi.ReplicationStatus == replication.Completed && oi.VersionPurgeStatus.Empty() && !roi.ExistingObjResync.mustResync() {
+		return
+	}
+
+	if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
+		versionID := ""
+		dmVersionID := ""
+		if roi.VersionPurgeStatus.Empty() {
+			dmVersionID = roi.VersionID
+		} else {
+			versionID = roi.VersionID
+		}
+
+		dv := DeletedObjectReplicationInfo{
+			DeletedObject: DeletedObject{
+				ObjectName:            roi.Name,
+				DeleteMarkerVersionID: dmVersionID,
+				VersionID:             versionID,
+				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+				DeleteMarker:          roi.DeleteMarker,
+			},
+			Bucket:    roi.Bucket,
+			OpType:    replication.HealReplicationType,
+			EventType: ReplicateHealDelete,
+		}
+		// heal delete marker replication failure or versioned delete replication failure
+		if roi.ReplicationStatus == replication.Pending ||
+			roi.ReplicationStatus == replication.Failed ||
+			roi.VersionPurgeStatus == Failed || roi.VersionPurgeStatus == Pending {
+			globalReplicationPool.queueReplicaDeleteTask(dv)
+			return
+		}
+		// if replication status is Complete on DeleteMarker and existing object resync required
+		if roi.ExistingObjResync.mustResync() && (roi.ReplicationStatus == replication.Completed || roi.ReplicationStatus.Empty()) {
+			queueReplicateDeletesWrapper(dv, roi.ExistingObjResync)
+			return
+		}
+		return
+	}
+	if roi.ExistingObjResync.mustResync() {
+		roi.OpType = replication.ExistingObjectReplicationType
+	}
+	switch roi.ReplicationStatus {
+	case replication.Pending, replication.Failed:
+		roi.EventType = ReplicateHeal
+		globalReplicationPool.queueReplicaTask(roi)
+		return
+	}
+	if roi.ExistingObjResync.mustResync() {
+		roi.EventType = ReplicateExisting
+		globalReplicationPool.queueReplicaTask(roi)
+	}
+	return
+}
+
+const mrfTimeInterval = 5 * time.Minute
+
+func (p *ReplicationPool) persistMRF() {
+	var mu sync.Mutex
+	entries := make(map[string]MRFReplicateEntry)
+	mTimer := time.NewTimer(mrfTimeInterval)
+	defer mTimer.Stop()
+	saveMRFToDisk := func(drain bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		if len(entries) == 0 {
+			return
+		}
+		cctx := p.ctx
+		if drain {
+			cctx = context.Background()
+			// drain all mrf entries and save to disk
+			for e := range p.mrfSaveCh {
+				entries[e.versionID] = e
+			}
+		}
+		if err := p.saveMRFEntries(cctx, entries); err != nil {
+			logger.LogOnceIf(p.ctx, fmt.Errorf("Unable to persist replication failures to disk:%w", err), string(replicationSubsystem))
+		}
+		entries = make(map[string]MRFReplicateEntry)
+	}
+	for {
+		select {
+		case <-mTimer.C:
+			saveMRFToDisk(false)
+			mTimer.Reset(mrfTimeInterval)
+		case <-p.ctx.Done():
+			close(p.mrfSaveCh)
+			saveMRFToDisk(true)
+			return
+		case <-p.saveStateCh:
+			saveMRFToDisk(true)
+			return
+		case e, ok := <-p.mrfSaveCh:
+			if !ok {
+				return
+			}
+			var cnt int
+			mu.Lock()
+			entries[e.versionID] = e
+			cnt = len(entries)
+			mu.Unlock()
+			if cnt >= cap(p.mrfSaveCh) || len(p.mrfSaveCh) >= int(0.8*float32(cap(p.mrfSaveCh))) {
+				saveMRFToDisk(true)
+			}
+		}
+	}
+}
+
+func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
+	if p == nil {
+		return
+	}
+	select {
+	case <-GlobalContext.Done():
+		return
+	case p.mrfSaveCh <- entry:
+	}
+}
+
+// save mrf entries to mrf_<uuid>.bin
+func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string]MRFReplicateEntry) error {
+	if len(entries) == 0 {
+		return nil
+	}
+	v := MRFReplicateEntries{
+		Entries: entries,
+		Version: mrfMetaVersionV1,
+	}
+	data := make([]byte, 4, v.Msgsize()+4)
+
+	// Initialize the resync meta header.
+	binary.LittleEndian.PutUint16(data[0:2], resyncMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], resyncMetaVersion)
+
+	buf, err := v.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	configFile := path.Join(replicationMRFDir, mustGetUUID()+".bin")
+	err = saveConfig(ctx, p.objLayer, configFile, buf)
+	return err
+}
+
+// load mrf entries from disk
+func (p *ReplicationPool) loadMRF(fileName string) (re MRFReplicateEntries, e error) {
+	data, err := readConfig(p.ctx, p.objLayer, fileName)
+	if err != nil && err != errConfigNotFound {
+		return re, err
+	}
+	if len(data) == 0 {
+		// Seems to be empty.
+		return re, nil
+	}
+	if len(data) <= 4 {
+		return re, fmt.Errorf("replication mrf: no data")
+	}
+	// Read resync meta header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case mrfMetaFormat:
+	default:
+		return re, fmt.Errorf("replication mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case mrfMetaVersion:
+	default:
+		return re, fmt.Errorf("replication mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+	// OK, parse data.
+	if _, err = re.UnmarshalMsg(data[4:]); err != nil {
+		return re, err
+	}
+
+	switch re.Version {
+	case mrfMetaVersionV1:
+	default:
+		return re, fmt.Errorf("unexpected mrf meta version: %d", re.Version)
+	}
+	return re, nil
+}
+
+func (p *ReplicationPool) processMRF() {
+	if p == nil || p.objLayer == nil {
+		return
+	}
+	pTimer := time.NewTimer(mrfTimeInterval)
+	defer pTimer.Stop()
+	for {
+		select {
+		case <-pTimer.C:
+			// skip healing if all targets are offline
+			var offlineCnt int
+			tgts := globalBucketTargetSys.ListTargets(p.ctx, "", "")
+			for _, tgt := range tgts {
+				if globalBucketTargetSys.isOffline(tgt.URL()) {
+					offlineCnt++
+				}
+			}
+			if len(tgts) == offlineCnt {
+				pTimer.Reset(mrfTimeInterval)
+				continue
+			}
+			objCh := make(chan ObjectInfo)
+			cctx, cancelFn := context.WithCancel(p.ctx)
+			if err := p.objLayer.Walk(cctx, minioMetaBucket, replicationMRFDir, objCh, ObjectOptions{}); err != nil {
+				pTimer.Reset(mrfTimeInterval)
+				cancelFn()
+				logger.LogIf(p.ctx, err)
+				continue
+			}
+			for item := range objCh {
+				if err := p.queueMRFHeal(item.Name); err == nil {
+					p.objLayer.DeleteObject(p.ctx, minioMetaBucket, item.Name, ObjectOptions{})
+				}
+			}
+			pTimer.Reset(mrfTimeInterval)
+			cancelFn()
+		case <-p.ctx.Done():
+			return
+		}
+	}
+}
+
+// process sends error logs to the heal channel for an attempt to heal replication.
+func (p *ReplicationPool) queueMRFHeal(file string) error {
+	if p == nil || p.objLayer == nil {
+		return errServerNotInitialized
+	}
+
+	mrfRec, err := p.loadMRF(file)
+	if err != nil {
+		return err
+	}
+	for vID, e := range mrfRec.Entries {
+		oi, err := p.objLayer.GetObjectInfo(p.ctx, e.Bucket, e.Object, ObjectOptions{
+			VersionID: vID,
+		})
+		if err != nil {
+			continue
+		}
+		QueueReplicationHeal(p.ctx, e.Bucket, oi)
+	}
+	return nil
+}
+
+// load replication stats from disk
+func (p *ReplicationPool) loadStatsFromDisk() (rs map[string]BucketReplicationStats, e error) {
+	data, err := readConfig(p.ctx, p.objLayer, getReplicationStatsPath(globalLocalNodeName))
+	if err != nil {
+		if !errors.Is(err, errConfigNotFound) {
+			return rs, nil
+		}
+		return rs, err
+	}
+
+	if len(data) <= 4 {
+		logger.LogIf(p.ctx, fmt.Errorf("replication stats: no data"))
+		return map[string]BucketReplicationStats{}, nil
+	}
+	// Read repl stats meta header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case replStatsMetaFormat:
+	default:
+		return rs, fmt.Errorf("replication stats: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case replStatsVersion:
+	default:
+		return rs, fmt.Errorf("replication stats: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+	ss := BucketStatsMap{}
+	if _, err = ss.UnmarshalMsg(data[4:]); err != nil {
+		return rs, err
+	}
+	rs = make(map[string]BucketReplicationStats, len(ss.Stats))
+	for bucket, st := range ss.Stats {
+		rs[bucket] = st.ReplicationStats
+	}
+
+	return rs, nil
+}
+
+func (p *ReplicationPool) saveStatsToDisk() {
+	if p == nil || p.objLayer == nil {
+		return
+	}
+	sTimer := time.NewTimer(replStatsSaveInterval)
+	defer sTimer.Stop()
+	for {
+		select {
+		case <-sTimer.C:
+			dui, err := loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
+			if err == nil && !dui.LastUpdate.IsZero() {
+				globalReplicationStats.getAllLatest(dui.BucketsUsage)
+			}
+			p.saveStats(p.ctx)
+			sTimer.Reset(replStatsSaveInterval)
+		case <-p.ctx.Done():
+			return
+		}
+	}
+}
+
+// save replication stats to .minio.sys/buckets/replication/node-name.stats
+func (p *ReplicationPool) saveStats(ctx context.Context) error {
+	bsm := globalReplicationStats.getAllCachedLatest()
+	if len(bsm.Stats) == 0 {
+		return nil
+	}
+	data := make([]byte, 4, 4+bsm.Msgsize())
+	// Add the replication stats meta header.
+	binary.LittleEndian.PutUint16(data[0:2], replStatsMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], replStatsVersion)
+	// Add data
+	data, err := bsm.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+	return saveConfig(ctx, p.objLayer, getReplicationStatsPath(globalLocalNodeName), data)
+}
+
+// SaveState saves replication stats and mrf data before server restart
+func (p *ReplicationPool) SaveState(ctx context.Context) error {
+	go func() {
+		select {
+		case p.saveStateCh <- struct{}{}:
+		case <-p.ctx.Done():
+			return
+		}
+	}()
+	return p.saveStats(ctx)
+}

cmd/bucket-stats.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"fmt"
 	"time"
 )
 
@@ -52,7 +53,10 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
 }
 
 // BucketStatsMap captures bucket statistics for all buckets
-type BucketStatsMap map[string]BucketStats
+type BucketStatsMap struct {
+	Stats     map[string]BucketStats
+	Timestamp time.Time
+}
 
 // BucketStats bucket statistics
 type BucketStats struct {
@@ -62,7 +66,7 @@ type BucketStats struct {
 // BucketReplicationStats represents inline replication statistics
 // such as pending, failed and completed bytes in total for a bucket
 type BucketReplicationStats struct {
-	Stats map[string]*BucketReplicationStat
+	Stats map[string]*BucketReplicationStat `json:",omitempty"`
 	// Pending size in bytes
 	PendingSize int64 `json:"pendingReplicationSize"`
 	// Completed size in bytes
@@ -126,3 +130,11 @@ func (bs *BucketReplicationStat) hasReplicationUsage() bool {
 		bs.PendingCount > 0 ||
 		bs.PendingSize > 0
 }
+
+func (brs BucketReplicationStats) String() string {
+	s := "ReplicatedSize=" + fmt.Sprintf("%d", brs.ReplicatedSize) + "+\n ReplicaSize=" + fmt.Sprintf("%d", brs.ReplicaSize)
+	for arn, st := range brs.Stats {
+		s += "\n arn: " + arn + " ReplicatedSize=" + fmt.Sprintf("%d", st.ReplicatedSize) + "  +::ReplicaSize=" + fmt.Sprintf("%d", st.ReplicaSize)
+	}
+	return s
+}

cmd/bucket-stats_gen.go

@@ -794,74 +794,109 @@ func (z *BucketStats) Msgsize() (s int) {
 
 // DecodeMsg implements msgp.Decodable
 func (z *BucketStatsMap) DecodeMsg(dc *msgp.Reader) (err error) {
-	var zb0003 uint32
-	zb0003, err = dc.ReadMapHeader()
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
 	if err != nil {
 		err = msgp.WrapError(err)
 		return
 	}
-	if (*z) == nil {
-		(*z) = make(BucketStatsMap, zb0003)
-	} else if len((*z)) > 0 {
-		for key := range *z {
-			delete((*z), key)
-		}
-	}
-	for zb0003 > 0 {
-		zb0003--
-		var zb0001 string
-		var zb0002 BucketStats
-		zb0001, err = dc.ReadString()
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
 		if err != nil {
 			err = msgp.WrapError(err)
 			return
 		}
-		var field []byte
-		_ = field
-		var zb0004 uint32
-		zb0004, err = dc.ReadMapHeader()
-		if err != nil {
-			err = msgp.WrapError(err, zb0001)
-			return
-		}
-		for zb0004 > 0 {
-			zb0004--
-			field, err = dc.ReadMapKeyPtr()
+		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
-				err = msgp.WrapError(err, zb0001)
+				err = msgp.WrapError(err, "Stats")
 				return
 			}
-			switch msgp.UnsafeString(field) {
-			case "ReplicationStats":
-				err = zb0002.ReplicationStats.DecodeMsg(dc)
-				if err != nil {
-					err = msgp.WrapError(err, zb0001, "ReplicationStats")
-					return
-				}
-			default:
-				err = dc.Skip()
-				if err != nil {
-					err = msgp.WrapError(err, zb0001)
-					return
+			if z.Stats == nil {
+				z.Stats = make(map[string]BucketStats, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
 				}
 			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 BucketStats
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				var zb0003 uint32
+				zb0003, err = dc.ReadMapHeader()
+				if err != nil {
+					err = msgp.WrapError(err, "Stats", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "ReplicationStats":
+						err = za0002.ReplicationStats.DecodeMsg(dc)
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001)
+							return
+						}
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
+		case "Timestamp":
+			z.Timestamp, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "Timestamp")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
 		}
-		(*z)[zb0001] = zb0002
 	}
 	return
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteMapHeader(uint32(len(z)))
+func (z *BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Stats"
+	err = en.Append(0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
 	if err != nil {
-		err = msgp.WrapError(err)
 		return
 	}
-	for zb0005, zb0006 := range z {
-		err = en.WriteString(zb0005)
+	err = en.WriteMapHeader(uint32(len(z.Stats)))
+	if err != nil {
+		err = msgp.WrapError(err, "Stats")
+		return
+	}
+	for za0001, za0002 := range z.Stats {
+		err = en.WriteString(za0001)
 		if err != nil {
-			err = msgp.WrapError(err)
+			err = msgp.WrapError(err, "Stats")
 			return
 		}
 		// map header, size 1
@@ -870,102 +905,148 @@ func (z BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) {
 		if err != nil {
 			return
 		}
-		err = zb0006.ReplicationStats.EncodeMsg(en)
+		err = za0002.ReplicationStats.EncodeMsg(en)
 		if err != nil {
-			err = msgp.WrapError(err, zb0005, "ReplicationStats")
+			err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
 			return
 		}
 	}
+	// write "Timestamp"
+	err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.Timestamp)
+	if err != nil {
+		err = msgp.WrapError(err, "Timestamp")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendMapHeader(o, uint32(len(z)))
-	for zb0005, zb0006 := range z {
-		o = msgp.AppendString(o, zb0005)
+	// map header, size 2
+	// string "Stats"
+	o = append(o, 0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Stats)))
+	for za0001, za0002 := range z.Stats {
+		o = msgp.AppendString(o, za0001)
 		// map header, size 1
 		// string "ReplicationStats"
 		o = append(o, 0x81, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
-		o, err = zb0006.ReplicationStats.MarshalMsg(o)
+		o, err = za0002.ReplicationStats.MarshalMsg(o)
 		if err != nil {
-			err = msgp.WrapError(err, zb0005, "ReplicationStats")
+			err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
 			return
 		}
 	}
+	// string "Timestamp"
+	o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
+	o = msgp.AppendTime(o, z.Timestamp)
 	return
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
 func (z *BucketStatsMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var zb0003 uint32
-	zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err)
 		return
 	}
-	if (*z) == nil {
-		(*z) = make(BucketStatsMap, zb0003)
-	} else if len((*z)) > 0 {
-		for key := range *z {
-			delete((*z), key)
-		}
-	}
-	for zb0003 > 0 {
-		var zb0001 string
-		var zb0002 BucketStats
-		zb0003--
-		zb0001, bts, err = msgp.ReadStringBytes(bts)
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
 		if err != nil {
 			err = msgp.WrapError(err)
 			return
 		}
-		var field []byte
-		_ = field
-		var zb0004 uint32
-		zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
-		if err != nil {
-			err = msgp.WrapError(err, zb0001)
-			return
-		}
-		for zb0004 > 0 {
-			zb0004--
-			field, bts, err = msgp.ReadMapKeyZC(bts)
+		switch msgp.UnsafeString(field) {
+		case "Stats":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, zb0001)
+				err = msgp.WrapError(err, "Stats")
 				return
 			}
-			switch msgp.UnsafeString(field) {
-			case "ReplicationStats":
-				bts, err = zb0002.ReplicationStats.UnmarshalMsg(bts)
-				if err != nil {
-					err = msgp.WrapError(err, zb0001, "ReplicationStats")
-					return
-				}
-			default:
-				bts, err = msgp.Skip(bts)
-				if err != nil {
-					err = msgp.WrapError(err, zb0001)
-					return
+			if z.Stats == nil {
+				z.Stats = make(map[string]BucketStats, zb0002)
+			} else if len(z.Stats) > 0 {
+				for key := range z.Stats {
+					delete(z.Stats, key)
 				}
 			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 BucketStats
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Stats")
+					return
+				}
+				var zb0003 uint32
+				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Stats", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Stats", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "ReplicationStats":
+						bts, err = za0002.ReplicationStats.UnmarshalMsg(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001, "ReplicationStats")
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Stats", za0001)
+							return
+						}
+					}
+				}
+				z.Stats[za0001] = za0002
+			}
+		case "Timestamp":
+			z.Timestamp, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Timestamp")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
 		}
-		(*z)[zb0001] = zb0002
 	}
 	o = bts
 	return
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z BucketStatsMap) Msgsize() (s int) {
-	s = msgp.MapHeaderSize
-	if z != nil {
-		for zb0005, zb0006 := range z {
-			_ = zb0006
-			s += msgp.StringPrefixSize + len(zb0005) + 1 + 17 + zb0006.ReplicationStats.Msgsize()
+func (z *BucketStatsMap) Msgsize() (s int) {
+	s = 1 + 6 + msgp.MapHeaderSize
+	if z.Stats != nil {
+		for za0001, za0002 := range z.Stats {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + 1 + 17 + za0002.ReplicationStats.Msgsize()
 		}
 	}
+	s += 10 + msgp.TimeSize
 	return
 }
 

cmd/bucket-targets.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"net/url"
 	"sync"
 	"time"
 
@@ -35,7 +37,9 @@ import (
 )
 
 const (
-	defaultHealthCheckDuration = 30 * time.Second
+	defaultHealthCheckDuration = 5 * time.Second
+	// default interval for reload of all remote target endpoints
+	defaultHealthCheckReloadDuration = 30 * time.Minute
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -43,6 +47,113 @@ type BucketTargetSys struct {
 	sync.RWMutex
 	arnRemotesMap map[string]*TargetClient
 	targetsMap    map[string][]madmin.BucketTarget
+	hMutex        sync.RWMutex
+	hc            map[string]epHealth
+	hcClient      *madmin.AnonymousClient
+}
+
+// epHealth struct represents health of a replication target endpoint.
+type epHealth struct {
+	Endpoint string
+	Scheme   string
+	Online   bool
+}
+
+// isOffline returns current liveness result of remote target. Add endpoint to
+// healthcheck map if missing and default to online status
+func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
+	sys.hMutex.RLock()
+	defer sys.hMutex.RUnlock()
+	if h, ok := sys.hc[ep.Host]; ok {
+		return !h.Online
+	}
+	go sys.initHC(ep)
+	return false
+}
+
+func (sys *BucketTargetSys) initHC(ep *url.URL) {
+	sys.hMutex.Lock()
+	sys.hc[ep.Host] = epHealth{
+		Endpoint: ep.Host,
+		Scheme:   ep.Scheme,
+		Online:   true,
+	}
+	sys.hMutex.Unlock()
+}
+
+// newHCClient initializes an anonymous client for performing health check on the remote endpoints
+func newHCClient() *madmin.AnonymousClient {
+	clnt, e := madmin.NewAnonymousClientNoEndpoint()
+	if e != nil {
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to initialize health check client"), string(replicationSubsystem))
+		return nil
+	}
+	clnt.SetCustomTransport(globalRemoteTargetTransport)
+	return clnt
+}
+
+// heartBeat performs liveness check on remote endpoints.
+func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
+	hcTimer := time.NewTimer(defaultHealthCheckDuration)
+	defer hcTimer.Stop()
+	for {
+		select {
+		case <-hcTimer.C:
+			sys.hMutex.RLock()
+			var eps []madmin.ServerProperties
+			for _, ep := range sys.hc {
+				eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme})
+			}
+			sys.hMutex.RUnlock()
+
+			if len(eps) > 0 {
+				cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+				defer cancel()
+				m := map[string]epHealth{}
+				for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
+					var online bool
+					if result.Error == nil {
+						online = result.Online
+					}
+					m[result.Endpoint.Host] = epHealth{
+						Endpoint: result.Endpoint.Host,
+						Scheme:   result.Endpoint.Scheme,
+						Online:   online,
+					}
+				}
+				sys.hMutex.Lock()
+				sys.hc = m
+				sys.hMutex.Unlock()
+			}
+			hcTimer.Reset(defaultHealthCheckDuration)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// periodically rebuild the healthcheck map from list of targets to clear
+// out stale endpoints
+func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) {
+	m := make(map[string]epHealth)
+	tgts := sys.ListTargets(ctx, "", "")
+	for _, t := range tgts {
+		if _, ok := m[t.Endpoint]; !ok {
+			scheme := "http"
+			if t.Secure {
+				scheme = "https"
+			}
+			m[t.Endpoint] = epHealth{
+				Online:   true,
+				Endpoint: t.Endpoint,
+				Scheme:   scheme,
+			}
+		}
+	}
+	sys.hMutex.Lock()
+	// swap out the map
+	sys.hc = m
+	sys.hMutex.Unlock()
 }
 
 // ListTargets lists bucket targets across tenant or for individual bucket, and returns
@@ -91,9 +202,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
 		return
 	}
 	for _, t := range tgts {
-		if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
-			tgt.healthCancelFn()
-		}
 		delete(sys.arnRemotesMap, t.Arn)
 	}
 	delete(sys.targetsMap, bucket)
@@ -116,7 +224,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
 			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 		}
-		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 	}
 	if tgt.Type == madmin.ReplicationService {
 		if !globalBucketVersioningSys.Enabled(bucket) {
@@ -124,7 +232,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
 		if err != nil {
-			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 		}
 		if vcfg.Status != string(versioning.Enabled) {
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
@@ -152,10 +260,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	if !found && !update {
 		newtgts = append(newtgts, *tgt)
 	}
-	// cancel health check for previous target client to avoid leak.
-	if prevClnt, ok := sys.arnRemotesMap[tgt.Arn]; ok && prevClnt.healthCancelFn != nil {
-		prevClnt.healthCancelFn()
-	}
+
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
 	sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
@@ -227,9 +332,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		return BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	sys.targetsMap[bucket] = targets
-	if tgt, ok := sys.arnRemotesMap[arnStr]; ok && tgt.healthCancelFn != nil {
-		tgt.healthCancelFn()
-	}
 	delete(sys.arnRemotesMap, arnStr)
 	sys.updateBandwidthLimit(bucket, 0)
 	return nil
@@ -258,11 +360,29 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
 }
 
 // NewBucketTargetSys - creates new replication system.
-func NewBucketTargetSys() *BucketTargetSys {
-	return &BucketTargetSys{
+func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
+	sys := &BucketTargetSys{
 		arnRemotesMap: make(map[string]*TargetClient),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
+		hc:            make(map[string]epHealth),
+		hcClient:      newHCClient(),
 	}
+	// reload healthcheck endpoints map periodically to remove stale endpoints from the map.
+	go func() {
+		rTimer := time.NewTimer(defaultHealthCheckReloadDuration)
+		defer rTimer.Stop()
+		for {
+			select {
+			case <-rTimer.C:
+				sys.reloadHealthCheckers(ctx)
+				rTimer.Reset(defaultHealthCheckReloadDuration)
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	go sys.heartBeat(ctx)
+	return sys
 }
 
 // UpdateAllTargets updates target to reflect metadata updates
@@ -276,9 +396,6 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 	// Remove existingtarget and arn association
 	if tgts, ok := sys.targetsMap[bucket]; ok {
 		for _, t := range tgts {
-			if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
-				tgt.healthCancelFn()
-			}
 			delete(sys.arnRemotesMap, t.Arn)
 		}
 		delete(sys.targetsMap, bucket)
@@ -345,10 +462,6 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
 	}
-	cancelFn, err := api.HealthCheck(hcDuration)
-	if err != nil {
-		return nil, err
-	}
 	tc := &TargetClient{
 		Client:              api,
 		healthCheckDuration: hcDuration,
@@ -356,9 +469,10 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 		Bucket:              tcfg.TargetBucket,
 		StorageClass:        tcfg.StorageClass,
 		disableProxy:        tcfg.DisableProxy,
-		healthCancelFn:      cancelFn,
 		ARN:                 tcfg.Arn,
 		ResetID:             tcfg.ResetID,
+		Endpoint:            tcfg.Endpoint,
+		Secure:              tcfg.Secure,
 	}
 	return tc, nil
 }
@@ -432,7 +546,8 @@ type TargetClient struct {
 	replicateSync       bool
 	StorageClass        string // storage class on remote
 	disableProxy        bool
-	healthCancelFn      context.CancelFunc // cancellation function for client healthcheck
-	ARN                 string             // ARN to uniquely identify remote target
+	ARN                 string // ARN to uniquely identify remote target
 	ResetID             string
+	Endpoint            string
+	Secure              bool
 }

cmd/bucket-versioning-handler.go

@@ -143,7 +143,7 @@ func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/common-main.go

@@ -264,7 +264,7 @@ func initConsoleServer() (*restapi.Server, error) {
 		return nil, err
 	}
 
-	api := operations.NewConsoleAPI(swaggerSpec, buildOpenIDConsoleConfig())
+	api := operations.NewConsoleAPI(swaggerSpec)
 
 	if !serverDebugLog {
 		// Disable console logging if server debug log is not enabled
@@ -275,6 +275,12 @@ func initConsoleServer() (*restapi.Server, error) {
 		api.Logger = noLog
 	}
 
+	// Pass in console application config. This needs to happen before the
+	// ConfigureAPI() call.
+	restapi.GlobalMinIOConfig = restapi.MinIOConfig{
+		OpenIDProviders: buildOpenIDConsoleConfig(),
+	}
+
 	server := restapi.NewServer(api)
 	// register all APIs
 	server.ConfigureAPI()
@@ -502,8 +508,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 			Skip: true,
 		}, nil
 	}
-	const envSeparator = "="
-	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
+	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
 	if len(envTokens) != 2 {
 		return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
 	}
@@ -529,7 +534,7 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 // the environment values from a file, in the form "key, value".
 // in a structured form.
 func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
-	f, err := os.Open(envConfigFile)
+	f, err := Open(envConfigFile)
 	if err != nil {
 		return nil, err
 	}
@@ -824,43 +829,56 @@ func handleKMSConfig() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		// Manually load the certificate and private key into memory.
-		// We need to check whether the private key is encrypted, and
-		// if so, decrypt it using the user-provided password.
-		certBytes, err := os.ReadFile(env.Get(config.EnvKESClientCert, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
-		}
-		keyBytes, err := os.ReadFile(env.Get(config.EnvKESClientKey, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client private key as specified by the shell environment")
-		}
-		privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
-		if len(rest) != 0 {
-			logger.Fatal(errors.New("private key contains additional data"), "Unable to load KES client private key as specified by the shell environment")
-		}
-		if x509.IsEncryptedPEMBlock(privateKeyPEM) {
-			keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
-			if err != nil {
-				logger.Fatal(err, "Unable to decrypt KES client private key as specified by the shell environment")
-			}
-			keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
-		}
-		certificate, err := tls.X509KeyPair(certBytes, keyBytes)
-		if err != nil {
-			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
-		}
 		rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
 		if err != nil {
 			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
+		loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
+			// Manually load the certificate and private key into memory.
+			// We need to check whether the private key is encrypted, and
+			// if so, decrypt it using the user-provided password.
+			certBytes, err := os.ReadFile(certFile)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+			}
+			keyBytes, err := os.ReadFile(keyFile)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+			}
+			privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+			if len(rest) != 0 {
+				return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
+			}
+			if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+				keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+				}
+				keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+			}
+			certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+			if err != nil {
+				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+			}
+			return certificate, nil
+		}
+
+		reloadCertEvents := make(chan tls.Certificate, 1)
+		certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
+		if err != nil {
+			logger.Fatal(err, "Failed to load KES client certificate")
+		}
+		certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
+		certificate.Notify(reloadCertEvents)
+
 		defaultKeyID := env.Get(config.EnvKESKeyName, "")
 		KMS, err := kms.NewWithConfig(kms.Config{
-			Endpoints:    endpoints,
-			DefaultKeyID: defaultKeyID,
-			Certificate:  certificate,
-			RootCAs:      rootCAs,
+			Endpoints:        endpoints,
+			DefaultKeyID:     defaultKeyID,
+			Certificate:      certificate,
+			ReloadCertEvents: reloadCertEvents,
+			RootCAs:          rootCAs,
 		})
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
@@ -910,7 +928,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	// Therefore, we read all filenames in the cert directory and check
 	// for each directory whether it contains a public.crt and private.key.
 	// If so, we try to add it to certificate manager.
-	root, err := os.Open(globalCertsDir.Get())
+	root, err := Open(globalCertsDir.Get())
 	if err != nil {
 		return nil, nil, false, err
 	}
@@ -929,7 +947,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 			continue
 		}
 		if file.Mode()&os.ModeSymlink == os.ModeSymlink {
-			file, err = os.Stat(filepath.Join(root.Name(), file.Name()))
+			file, err = Stat(filepath.Join(root.Name(), file.Name()))
 			if err != nil {
 				// not accessible ignore
 				continue

cmd/config-current.go

@@ -396,7 +396,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	}
 
 	if config.NotifySubSystems.Contains(subSys) {
-		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs(), subSys); err != nil {
+		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalEventNotifier.ConfiguredTargetIDs(), subSys); err != nil {
 			return err
 		}
 	}
@@ -471,14 +471,12 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if etcdCfg.Enabled {
-		if globalEtcdClient == nil {
-			globalEtcdClient, err = etcd.New(etcdCfg)
-			if err != nil {
-				if globalIsGateway {
-					logger.FatalIf(err, "Unable to initialize etcd config")
-				} else {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
-				}
+		globalEtcdClient, err = etcd.New(etcdCfg)
+		if err != nil {
+			if globalIsGateway {
+				logger.FatalIf(err, "Unable to initialize etcd config")
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 			}
 		}
 
@@ -486,7 +484,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			if globalDNSConfig != nil {
 				// if global DNS is already configured, indicate with a warning, incase
 				// users are confused.
-				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
+				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
 			} else {
 				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 					dns.DomainNames(globalDomainNames),
@@ -518,16 +516,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
-	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
-	}
-
-	// Initialize remote instance transport once.
-	getRemoteInstanceTransportOnce.Do(func() {
-		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
-	})
-
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
 		if globalIsGateway {
@@ -560,56 +548,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
-	globalOpenIDConfig, err = openid.LookupConfig(s,
-		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
-	}
-
-	globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
-		globalRootCAs)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
-	}
-
-	authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
-	}
-	globalAuthNPlugin = idplugin.New(authNPluginCfg)
-
-	authZPluginCfg, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err))
-	}
-	if authZPluginCfg.URL == nil {
-		opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-			NewGatewayHTTPTransport(), xhttp.DrainBody)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
-		} else {
-			authZPluginCfg.URL = opaCfg.URL
-			authZPluginCfg.AuthToken = opaCfg.AuthToken
-			authZPluginCfg.Transport = opaCfg.Transport
-			authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
-		}
-	}
-
-	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
-
 	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
+	transport := NewGatewayHTTPTransport()
+
+	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, transport, false)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), NewGatewayHTTPTransport(), true)
+	globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), transport, true)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
@@ -636,6 +587,11 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			setDriveCounts = objAPI.SetDriveCounts()
 		}
 		globalAPIConfig.init(apiConfig, setDriveCounts)
+
+		// Initialize remote instance transport once.
+		getRemoteInstanceTransportOnce.Do(func() {
+			getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
+		})
 	case config.CompressionSubSys:
 		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 		if err != nil {
@@ -678,8 +634,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.HTTP[n] = l
 			}
 		}
-		err = logger.UpdateSystemTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateSystemTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
 		}
 	case config.AuditWebhookSubSys:
@@ -697,8 +652,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 		}
 
-		err = logger.UpdateAuditWebhookTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateAuditWebhookTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
 		}
 	case config.AuditKafkaSubSys:
@@ -712,8 +666,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		err = logger.UpdateAuditKafkaTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateAuditKafkaTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
 		}
 	case config.StorageClassSubSys:
@@ -852,7 +805,7 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return saveServerConfig(GlobalContext, objAPI, globalServerConfig)
+	return saveServerConfig(GlobalContext, objAPI, srvCfg)
 }
 
 func getValidConfig(objAPI ObjectLayer) (config.Config, error) {

cmd/config-encrypted.go

@@ -69,6 +69,12 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		return err
 	}
 
+	// If backend doesn't have this file means we have already
+	// attempted then migration
+	if !encrypted {
+		return nil
+	}
+
 	if encrypted && GlobalKMS != nil {
 		stat, err := GlobalKMS.Stat(ctx)
 		if err != nil {
@@ -109,6 +115,8 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 						}
 					}
+				} else {
+					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 				}
 			}
 			data = pdata
@@ -131,6 +139,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 	if encrypted && GlobalKMS != nil {
 		logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
 	}
+
 	return deleteKeyEtcd(ctx, client, backendEncryptedFile)
 }
 
@@ -146,44 +155,56 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 
-	var marker string
-	for {
-		res, err := objAPI.ListObjects(GlobalContext, minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
+	results := make(chan ObjectInfo)
+	if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil {
+		return err
+	}
+
+	for obj := range results {
+		data, err := readConfig(GlobalContext, objAPI, obj.Name)
 		if err != nil {
 			return err
 		}
-		for _, obj := range res.Objects {
-			data, err := readConfig(GlobalContext, objAPI, obj.Name)
+
+		if !utf8.Valid(data) {
+			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+			if err != nil {
+				if GlobalKMS != nil {
+					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+						minioMetaBucket: path.Join(minioMetaBucket, obj.Name),
+					})
+					if err != nil {
+						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+							minioMetaBucket: obj.Name,
+						})
+						if err != nil {
+							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
+						}
+					}
+				} else {
+					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
+				}
+			}
+			data = pdata
+		}
+
+		if GlobalKMS != nil {
+			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
+				obj.Bucket: path.Join(obj.Bucket, obj.Name),
+			})
 			if err != nil {
 				return err
 			}
-
-			if !utf8.Valid(data) {
-				data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-				if err != nil {
-					return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
-				}
-			}
-			if GlobalKMS != nil {
-				data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
-					obj.Bucket: path.Join(obj.Bucket, obj.Name),
-				})
-				if err != nil {
-					return err
-				}
-			}
-			if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
-				return err
-			}
 		}
 
-		if !res.IsTruncated {
-			break
+		if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
+			return err
 		}
-		marker = res.NextMarker
 	}
+
 	if encrypted && GlobalKMS != nil {
 		logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
 	}
+
 	return deleteConfig(GlobalContext, objAPI, backendEncryptedFile)
 }

cmd/config-migrate_test.go

@@ -37,11 +37,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Init Test config failed")
 	}
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 	globalConfigDir = &ConfigDir{path: rootPath}
 
 	globalObjLayerMutex.Lock()
@@ -74,13 +70,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 // Test if all migrate code returns nil when config file does not
 // exist
 func TestServerConfigMigrateInexistentConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	globalConfigDir = &ConfigDir{path: rootPath}
+	globalConfigDir = &ConfigDir{path: t.TempDir()}
 
 	if err := migrateV2ToV3(); err != nil {
 		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@@ -164,11 +154,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 
 // Test if a config migration from v2 to v33 is successfully done
 func TestServerConfigMigrateV2toV33(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 
@@ -234,11 +220,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateFaultyConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
@@ -331,35 +313,31 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	for i := 3; i <= 17; i++ {
 		// Create a corrupted config file
-		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
+		if err := ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
 			0o644); err != nil {
 			t.Fatal("Unexpected error: ", err)
 		}
 
 		// Test different migrate versions and be sure they are returning an error
-		if err = migrateConfig(); err == nil {
+		if err := migrateConfig(); err == nil {
 			t.Fatal("migrateConfig() should fail with a corrupted json")
 		}
 	}
 
 	// Create a corrupted config file for version '2'.
-	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test different migrate versions and be sure they are returning an error
-	if err = migrateConfig(); err == nil {
+	if err := migrateConfig(); err == nil {
 		t.Fatal("migrateConfig() should fail with a corrupted json")
 	}
 }

cmd/data-scanner.go

@@ -64,7 +64,11 @@ const (
 var (
 	globalHealConfig heal.Config
 
-	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	dataScannerLeaderLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
+		timeout:       30 * time.Second,
+		minimum:       10 * time.Second,
+		retryInterval: time.Second,
+	})
 	// Sleeper values are updated when config is loaded.
 	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
 	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
@@ -1225,29 +1229,16 @@ func (i *scannerItem) objectPath() string {
 
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
-	roi := getHealReplicateObjectInfo(oi, i.replication)
-	if !roi.Dsc.ReplicateAny() {
+	if oi.VersionID == "" {
 		return
 	}
-
+	if i.replication.Config == nil {
+		return
+	}
+	roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication)
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
-		// heal delete marker replication failure or versioned delete replication failure
-		if oi.ReplicationStatus == replication.Pending ||
-			oi.ReplicationStatus == replication.Failed ||
-			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, roi)
-			return
-		}
-		// if replication status is Complete on DeleteMarker and existing object resync required
-		if roi.ExistingObjResync.mustResync() && (oi.ReplicationStatus == replication.Completed || oi.ReplicationStatus.Empty()) {
-			i.healReplicationDeletes(ctx, o, roi)
-			return
-		}
 		return
 	}
-	if roi.ExistingObjResync.mustResync() {
-		roi.OpType = replication.ExistingObjectReplicationType
-	}
 
 	if sizeS.replTargetStats == nil && len(roi.TargetStatuses) > 0 {
 		sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
@@ -1277,52 +1268,9 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 	}
 
 	switch oi.ReplicationStatus {
-	case replication.Pending, replication.Failed:
-		roi.EventType = ReplicateHeal
-		globalReplicationPool.queueReplicaTask(roi)
-		return
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
-	if roi.ExistingObjResync.mustResync() {
-		roi.EventType = ReplicateExisting
-		globalReplicationPool.queueReplicaTask(roi)
-	}
-}
-
-// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, roi ReplicateObjectInfo) {
-	// handle soft delete and permanent delete failures here.
-	if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
-		versionID := ""
-		dmVersionID := ""
-		if roi.VersionPurgeStatus.Empty() {
-			dmVersionID = roi.VersionID
-		} else {
-			versionID = roi.VersionID
-		}
-
-		doi := DeletedObjectReplicationInfo{
-			DeletedObject: DeletedObject{
-				ObjectName:            roi.Name,
-				DeleteMarkerVersionID: dmVersionID,
-				VersionID:             versionID,
-				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
-				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-				DeleteMarker:          roi.DeleteMarker,
-			},
-			Bucket:    roi.Bucket,
-			OpType:    replication.HealReplicationType,
-			EventType: ReplicateHealDelete,
-		}
-		if roi.ExistingObjResync.mustResync() {
-			doi.OpType = replication.ExistingObjectReplicationType
-			doi.EventType = ReplicateExistingDelete
-			queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
-			return
-		}
-		globalReplicationPool.queueReplicaDeleteTask(doi)
-	}
 }
 
 type dynamicSleeper struct {

cmd/data-update-tracker.go

@@ -227,7 +227,7 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 	for _, drive := range drives {
 
 		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-		f, err := os.Open(cacheFormatPath)
+		f, err := OpenFile(cacheFormatPath, readMode, 0o666)
 		if err != nil {
 			if osIsNotExist(err) {
 				continue

cmd/data-update-tracker_test.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path"
@@ -100,15 +99,11 @@ func TestDataUpdateTracker(t *testing.T) {
 
 	dut.Current.bf = dut.newBloomFilter()
 
-	tmpDir, err := ioutil.TempDir("", "TestDataUpdateTracker")
+	tmpDir := t.TempDir()
+	err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	dut.start(ctx, tmpDir)
@@ -205,11 +200,13 @@ func TestDataUpdateTracker(t *testing.T) {
 	}
 
 	ctx, cancel = context.WithCancel(context.Background())
-	defer cancel()
-
 	// Reload...
 	dut = newDataUpdateTracker()
 	dut.start(ctx, tmpDir)
+	defer func() {
+		cancel()
+		<-dut.saveExited
+	}()
 
 	if dut.current() != 2 {
 		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())

cmd/data-usage-cache_gen.go

@@ -651,6 +651,7 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
 	zb0001Len := uint32(5)
 	var zb0001Mask uint8 /* 5 bits */
+	_ = zb0001Mask
 	if z.BloomFilter == nil {
 		zb0001Len--
 		zb0001Mask |= 0x10
@@ -724,6 +725,7 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// omitempty: check for empty values
 	zb0001Len := uint32(5)
 	var zb0001Mask uint8 /* 5 bits */
+	_ = zb0001Mask
 	if z.BloomFilter == nil {
 		zb0001Len--
 		zb0001Mask |= 0x10
@@ -1652,6 +1654,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
 	zb0001Len := uint32(8)
 	var zb0001Mask uint8 /* 8 bits */
+	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
 		zb0001Mask |= 0x20
@@ -1782,6 +1785,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	// omitempty: check for empty values
 	zb0001Len := uint32(8)
 	var zb0001Mask uint8 /* 8 bits */
+	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
 		zb0001Mask |= 0x20
@@ -2774,6 +2778,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
 	zb0001Len := uint32(2)
 	var zb0001Mask uint8 /* 2 bits */
+	_ = zb0001Mask
 	if z.Targets == nil {
 		zb0001Len--
 		zb0001Mask |= 0x1
@@ -2835,6 +2840,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	// omitempty: check for empty values
 	zb0001Len := uint32(2)
 	var zb0001Mask uint8 /* 2 bits */
+	_ = zb0001Mask
 	if z.Targets == nil {
 		zb0001Len--
 		zb0001Mask |= 0x1

cmd/data-usage.go

@@ -56,7 +56,8 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 }
 
 // loadPrefixUsageFromBackend returns prefix usages found in passed buckets
-//   e.g.:  /testbucket/prefix => 355601334
+//
+//	e.g.:  /testbucket/prefix => 355601334
 func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) {
 	z, ok := objAPI.(*erasureServerPools)
 	if !ok {

cmd/data-usage_test.go

@@ -35,12 +35,8 @@ type usageTestFile struct {
 }
 
 func TestDataUsageUpdate(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUsageUpdate")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	const bucket = "bucket"
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},
@@ -251,12 +247,8 @@ func TestDataUsageUpdate(t *testing.T) {
 }
 
 func TestDataUsageUpdatePrefix(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUpdateUsagePrefix")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	scannerSleeper.Update(0, 0)
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "bucket/rootfile", size: 10000},
 		{name: "bucket/rootfile2", size: 10000},
@@ -537,12 +529,8 @@ func generateUsageTestFiles(t *testing.T, base, bucket string, nFolders, nFiles,
 }
 
 func TestDataUsageCacheSerialize(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUsageCacheSerialize")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	const bucket = "abucket"
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},

cmd/decommetric_string.go

@@ -0,0 +1,25 @@
+// Code generated by "stringer -type=decomMetric -trimprefix=decomMetric erasure-server-pool-decom.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[decomMetricDecommissionBucket-0]
+	_ = x[decomMetricDecommissionObject-1]
+	_ = x[decomMetricDecommissionRemoveObject-2]
+}
+
+const _decomMetric_name = "DecommissionBucketDecommissionObjectDecommissionRemoveObject"
+
+var _decomMetric_index = [...]uint8{0, 18, 36, 60}
+
+func (i decomMetric) String() string {
+	if i >= decomMetric(len(_decomMetric_index)-1) {
+		return "decomMetric(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _decomMetric_name[_decomMetric_index[i]:_decomMetric_index[i+1]]
+}

cmd/disk-cache-backend.go

@@ -38,6 +38,7 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
+	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/disk"
@@ -132,17 +133,14 @@ func (m *cacheMeta) ToObjectInfo() (o ObjectInfo) {
 	} else {
 		o.StorageClass = globalMinioDefaultStorageClass
 	}
-	var (
-		t time.Time
-		e error
-	)
+
 	if exp, ok := meta["expires"]; ok {
-		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+		if t, e := amztime.ParseHeader(exp); e == nil {
 			o.Expires = t.UTC()
 		}
 	}
 	if mtime, ok := meta["last-modified"]; ok {
-		if t, e = time.Parse(http.TimeFormat, mtime); e == nil {
+		if t, e := amztime.ParseHeader(mtime); e == nil {
 			o.ModTime = t.UTC()
 		}
 	}
@@ -624,7 +622,7 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
 		return err
 	}
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|os.O_CREATE|writeMode, 0o666)
 	if err != nil {
 		return err
 	}
@@ -687,7 +685,7 @@ func (c *diskCache) updateMetadata(ctx context.Context, bucket, object, etag str
 	if err := os.MkdirAll(cachedPath, 0o777); err != nil {
 		return err
 	}
-	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|writeMode, 0o666)
 	if err != nil {
 		return err
 	}
@@ -1275,12 +1273,12 @@ func (c *diskCache) NewMultipartUpload(ctx context.Context, bucket, object, uID
 
 	cachePath := getMultipartCacheSHADir(c.dir, bucket, object)
 	uploadIDDir := path.Join(cachePath, uploadID)
-	if err := os.MkdirAll(uploadIDDir, 0o777); err != nil {
+	if err := mkdirAll(uploadIDDir, 0o777); err != nil {
 		return uploadID, err
 	}
 	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
 
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|os.O_CREATE|writeMode, 0o666)
 	if err != nil {
 		return uploadID, err
 	}
@@ -1386,7 +1384,7 @@ func (c *diskCache) SavePartMetadata(ctx context.Context, bucket, object, upload
 	defer uploadLock.Unlock(ulkctx.Cancel)
 
 	metaPath := pathJoin(uploadDir, cacheMetaJSONFile)
-	f, err := os.OpenFile(metaPath, os.O_RDWR, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|writeMode, 0o666)
 	if err != nil {
 		return err
 	}
@@ -1465,7 +1463,7 @@ func newCachePartEncryptReader(ctx context.Context, bucket, object string, partI
 func (c *diskCache) uploadIDExists(bucket, object, uploadID string) (err error) {
 	mpartCachePath := getMultipartCacheSHADir(c.dir, bucket, object)
 	uploadIDDir := path.Join(mpartCachePath, uploadID)
-	if _, err := os.Stat(uploadIDDir); err != nil {
+	if _, err := Stat(uploadIDDir); err != nil {
 		return err
 	}
 	return nil
@@ -1564,7 +1562,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	uploadMeta.Hits++
 	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
 
-	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0o666)
+	f, err := OpenFile(metaPath, os.O_RDWR|os.O_CREATE|writeMode, 0o666)
 	if err != nil {
 		return oi, err
 	}
@@ -1634,7 +1632,7 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 			readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir), func(shaDir string, typ os.FileMode) error {
 				return readDirFn(pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir), func(uploadIDDir string, typ os.FileMode) error {
 					uploadIDPath := pathJoin(c.dir, minioMetaBucket, cacheMultipartDir, shaDir, uploadIDDir)
-					fi, err := os.Stat(uploadIDPath)
+					fi, err := Stat(uploadIDPath)
 					if err != nil {
 						return nil
 					}
@@ -1649,8 +1647,8 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 			readDirFn(pathJoin(c.dir, minioMetaBucket, cacheWritebackDir), func(shaDir string, typ os.FileMode) error {
 				wbdir := pathJoin(c.dir, minioMetaBucket, cacheWritebackDir, shaDir)
 				cachedir := pathJoin(c.dir, shaDir)
-				if _, err := os.Stat(cachedir); os.IsNotExist(err) {
-					fi, err := os.Stat(wbdir)
+				if _, err := Stat(cachedir); os.IsNotExist(err) {
+					fi, err := Stat(wbdir)
 					if err != nil {
 						return nil
 					}

cmd/disk-cache.go

@@ -87,7 +87,7 @@ type CacheObjectLayer interface {
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	// Multipart operations.
-	NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (res *NewMultipartUploadResult, err error)
 	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -122,7 +122,7 @@ type cacheObjects struct {
 	InnerDeleteObjectFn            func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	InnerPutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	InnerCopyObjectFn              func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
-	InnerNewMultipartUploadFn      func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
+	InnerNewMultipartUploadFn      func(ctx context.Context, bucket, object string, opts ObjectOptions) (res *NewMultipartUploadResult, err error)
 	InnerPutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	InnerAbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error
 	InnerCompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -607,12 +607,12 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 				warningMsg = fmt.Sprintf("Invalid cache dir %s err : %s", dir, err.Error())
 			}
 			if rootDsk {
-				warningMsg = fmt.Sprintf("cache dir cannot be part of root disk: %s", dir)
+				warningMsg = fmt.Sprintf("cache dir cannot be part of root drive: %s", dir)
 			}
 		}
 
 		if err := checkAtimeSupport(dir); err != nil {
-			return nil, false, fmt.Errorf("Atime support required for disk caching, atime check failed with %w", err)
+			return nil, false, fmt.Errorf("Atime support required for drive caching, atime check failed with %w", err)
 		}
 
 		cache, err := newDiskCache(ctx, dir, config)
@@ -622,7 +622,7 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		caches = append(caches, cache)
 	}
 	if warningMsg != "" {
-		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root disk for disk caching is deprecated: %s", warningMsg)))
+		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root drive for drive caching is deprecated: %s", warningMsg)))
 	}
 	return caches, migrating, nil
 }
@@ -866,7 +866,7 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		InnerCopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 		},
-		InnerNewMultipartUploadFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
+		InnerNewMultipartUploadFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (res *NewMultipartUploadResult, err error) {
 			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, opts)
 		},
 		InnerPutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
@@ -961,7 +961,7 @@ func (c *cacheObjects) queuePendingWriteback(ctx context.Context) {
 }
 
 // NewMultipartUpload - Starts a new multipart upload operation to backend - if writethrough mode is enabled, starts caching the multipart.
-func (c *cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
+func (c *cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (res *NewMultipartUploadResult, err error) {
 	newMultipartUploadFn := c.InnerNewMultipartUploadFn
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
@@ -996,9 +996,11 @@ func (c *cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object st
 	}
 
 	// perform multipart upload on backend and cache simultaneously
-	uploadID, err = newMultipartUploadFn(ctx, bucket, object, opts)
-	dcache.NewMultipartUpload(GlobalContext, bucket, object, uploadID, opts)
-	return uploadID, err
+	res, err = newMultipartUploadFn(ctx, bucket, object, opts)
+	if err == nil {
+		dcache.NewMultipartUpload(GlobalContext, bucket, object, res.UploadID, opts)
+	}
+	return res, err
 }
 
 // PutObjectPart streams part to cache concurrently if writethrough mode is enabled. Otherwise redirects the call to remote

cmd/dummy-data-generator_test.go

@@ -45,10 +45,10 @@ type DummyDataGen struct {
 //
 // Given the function:
 //
-// f := func(r io.Reader) string {
-//           b, _ := ioutil.ReadAll(r)
-//           return string(b)
-// }
+//	f := func(r io.Reader) string {
+//	          b, _ := ioutil.ReadAll(r)
+//	          return string(b)
+//	}
 //
 // for example, the following is true:
 //

cmd/dummy-handlers.go

@@ -52,7 +52,7 @@ func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *h
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -84,7 +84,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -117,7 +117,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -151,7 +151,7 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -189,7 +189,7 @@ func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/dynamic-timeouts.go

@@ -34,11 +34,24 @@ const (
 
 // timeouts that are dynamically adapted based on actual usage results
 type dynamicTimeout struct {
-	timeout int64
-	minimum int64
-	entries int64
-	log     [dynamicTimeoutLogSize]time.Duration
-	mutex   sync.Mutex
+	timeout       int64
+	minimum       int64
+	entries       int64
+	log           [dynamicTimeoutLogSize]time.Duration
+	mutex         sync.Mutex
+	retryInterval time.Duration
+}
+
+type dynamicTimeoutOpts struct {
+	timeout       time.Duration
+	minimum       time.Duration
+	retryInterval time.Duration
+}
+
+func newDynamicTimeoutWithOpts(opts dynamicTimeoutOpts) *dynamicTimeout {
+	dt := newDynamicTimeout(opts.timeout, opts.minimum)
+	dt.retryInterval = opts.retryInterval
+	return dt
 }
 
 // newDynamicTimeout returns a new dynamic timeout initialized with timeout value
@@ -57,6 +70,10 @@ func (dt *dynamicTimeout) Timeout() time.Duration {
 	return time.Duration(atomic.LoadInt64(&dt.timeout))
 }
 
+func (dt *dynamicTimeout) RetryInterval() time.Duration {
+	return dt.retryInterval
+}
+
 // LogSuccess logs the duration of a successful action that
 // did not hit the timeout
 func (dt *dynamicTimeout) LogSuccess(duration time.Duration) {

cmd/encryption-v1.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"crypto/hmac"
 	"crypto/rand"
@@ -37,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
+	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
@@ -82,7 +84,7 @@ func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDe
 // metadata, if any. It returns an empty ID if no key ID is
 // present.
 func kmsKeyIDFromMetadata(metadata map[string]string) string {
-	const ARNPrefix = "arn:aws:kms:"
+	const ARNPrefix = crypto.ARNPrefix
 	if len(metadata) == 0 {
 		return ""
 	}
@@ -1052,3 +1054,60 @@ func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
 	mac.Sum(key[:0])
 	return key
 }
+
+type (
+	objectMetaEncryptFn func(baseKey string, data []byte) []byte
+	objectMetaDecryptFn func(baseKey string, data []byte) ([]byte, error)
+)
+
+// metadataEncrypter returns a function that will read data from input,
+// encrypt it using the provided key and return the result.
+// 0 sized inputs are passed through.
+func metadataEncrypter(key crypto.ObjectKey) objectMetaEncryptFn {
+	return func(baseKey string, data []byte) []byte {
+		if len(data) == 0 {
+			return data
+		}
+		var buffer bytes.Buffer
+		mac := hmac.New(sha256.New, key[:])
+		mac.Write([]byte(baseKey))
+		if _, err := sio.Encrypt(&buffer, bytes.NewReader(data), sio.Config{Key: mac.Sum(nil), CipherSuites: fips.DARECiphers()}); err != nil {
+			logger.CriticalIf(context.Background(), errors.New("unable to encrypt using object key"))
+		}
+		return buffer.Bytes()
+	}
+}
+
+// metadataDecrypter reverses metadataEncrypter.
+func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
+	return func(baseKey string, input []byte) ([]byte, error) {
+		if len(input) == 0 {
+			return input, nil
+		}
+
+		key, err := decryptObjectInfo(nil, o.Bucket, o.Name, o.UserDefined)
+		if err != nil {
+			return nil, err
+		}
+		mac := hmac.New(sha256.New, key)
+		mac.Write([]byte(baseKey))
+		return sio.DecryptBuffer(nil, input, sio.Config{Key: mac.Sum(nil), CipherSuites: fips.DARECiphers()})
+	}
+}
+
+// decryptChecksums will attempt to decode checksums and return it/them if set.
+func (o *ObjectInfo) decryptChecksums() map[string]string {
+	data := o.Checksum
+	if len(data) == 0 {
+		return nil
+	}
+	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
+		decrypted, err := o.metadataDecrypter()("object-checksum", data)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			return nil
+		}
+		data = decrypted
+	}
+	return hash.ReadCheckSums(data)
+}

cmd/endpoint-ellipses.go

@@ -156,7 +156,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	setCounts := possibleSetCounts(commonSize)
 	if len(setCounts) == 0 {
-		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of drives %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
 		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 
@@ -183,7 +183,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		setCounts = possibleSetCountsWithSymmetry(setCounts, argPatterns)
 
 		if len(setCounts) == 0 {
-			msg := fmt.Sprintf("No symmetric distribution detected with input endpoints provided %s, disks %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes)
+			msg := fmt.Sprintf("No symmetric distribution detected with input endpoints provided %s, drives %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes)
 			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 		}
 
@@ -193,7 +193,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of drives %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
 		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 

cmd/endpoint.go

@@ -292,6 +292,19 @@ func (l EndpointServerPools) LocalDisksPaths() []string {
 	return disks
 }
 
+// NLocalDisksPathsPerPool returns the disk paths of the local disks per pool
+func (l EndpointServerPools) NLocalDisksPathsPerPool() []int {
+	localDisksCount := make([]int, len(l))
+	for i, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				localDisksCount[i]++
+			}
+		}
+	}
+	return localDisksCount
+}
+
 // FirstLocal returns true if the first endpoint is local.
 func (l EndpointServerPools) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
@@ -302,7 +315,7 @@ func (l EndpointServerPools) HTTPS() bool {
 	return l[0].Endpoints.HTTPS()
 }
 
-// NEndpoints - returns all nodes count
+// NEndpoints - returns number of endpoints
 func (l EndpointServerPools) NEndpoints() (count int) {
 	for _, ep := range l {
 		count += len(ep.Endpoints)
@@ -689,6 +702,17 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		}
 	}
 
+	// Add missing port in all endpoints.
+	for i := range endpoints {
+		_, port, err := net.SplitHostPort(endpoints[i].Host)
+		if err != nil {
+			endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
+		} else if endpoints[i].IsLocal && serverAddrPort != port {
+			// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
+			endpoints[i].IsLocal = false
+		}
+	}
+
 	// All endpoints are pointing to local host
 	if len(endpoints) == localEndpointCount {
 		// If all endpoints have same port number, Just treat it as local erasure setup
@@ -705,17 +729,6 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		// This means it is DistErasure setup.
 	}
 
-	// Add missing port in all endpoints.
-	for i := range endpoints {
-		_, port, err := net.SplitHostPort(endpoints[i].Host)
-		if err != nil {
-			endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
-		} else if endpoints[i].IsLocal && serverAddrPort != port {
-			// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
-			endpoints[i].IsLocal = false
-		}
-	}
-
 	uniqueArgs := set.NewStringSet()
 	for _, endpoint := range endpoints {
 		uniqueArgs.Add(endpoint.Host)

cmd/endpoint_test.go

@@ -252,10 +252,10 @@ func TestCreateEndpoints(t *testing.T) {
 		},
 		// DistErasure Setup with URLEndpointType
 		{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", Endpoints{
-			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d1"}, IsLocal: true},
-			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d2"}, IsLocal: true},
-			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d3"}, IsLocal: true},
-			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d4"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d1"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d2"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d3"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d4"}, IsLocal: true},
 		}, ErasureSetupType, nil},
 		// DistErasure Setup with URLEndpointType having mixed naming to local host.
 		{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},

cmd/erasure-bucket.go

@@ -35,7 +35,7 @@ var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
 // Bucket operations
 
 // MakeBucket - make a bucket.
-func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Verify if bucket is valid.
@@ -94,7 +94,7 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
 }
 
 // getBucketInfo - returns the BucketInfo from one of the load balanced disks.
-func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) {
+func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -108,9 +108,17 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 			}
 			volInfo, err := storageDisks[index].StatVol(ctx, bucketName)
 			if err != nil {
+				if opts.Deleted {
+					dvi, derr := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucketName))
+					if derr != nil {
+						return err
+					}
+					bucketsInfo[index] = BucketInfo{Name: bucketName, Deleted: dvi.Created}
+					return nil
+				}
 				return err
 			}
-			bucketsInfo[index] = BucketInfo(volInfo)
+			bucketsInfo[index] = BucketInfo{Name: volInfo.Name, Created: volInfo.Created}
 			return nil
 		}, index)
 	}
@@ -131,8 +139,8 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
-func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketInfo, err := er.getBucketInfo(ctx, bucket)
+func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
+	bucketInfo, err := er.getBucketInfo(ctx, bucket, opts)
 	if err != nil {
 		return bi, toObjectErr(err, bucket)
 	}
@@ -191,12 +199,58 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts D
 		if purgedDangling {
 			err = nil
 		}
-
+		if opts.SRDeleteOp == MarkDelete {
+			er.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+		}
 	}
 
 	return toObjectErr(err, bucket)
 }
 
+// markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
+// syncs the delete to peers
+func (er erasureObjects) markDelete(ctx context.Context, bucket, prefix string) error {
+	storageDisks := er.getDisks()
+	g := errgroup.WithNErrs(len(storageDisks))
+	// Make a volume entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		if storageDisks[index] == nil {
+			continue
+		}
+		g.Go(func() error {
+			if err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, prefix)); err != nil {
+				if errors.Is(err, errVolumeExists) {
+					return nil
+				}
+				return err
+			}
+			return nil
+		}, index)
+	}
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
+	return toObjectErr(err, bucket)
+}
+
+// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
+// syncs the delete to peers OR on a new MakeBucket call.
+func (er erasureObjects) purgeDelete(ctx context.Context, bucket, prefix string) error {
+	storageDisks := er.getDisks()
+	g := errgroup.WithNErrs(len(storageDisks))
+	// Make a volume entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if storageDisks[index] != nil {
+				return storageDisks[index].DeleteVol(ctx, pathJoin(bucket, prefix), true)
+			}
+			return errDiskNotFound
+		}, index)
+	}
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
+	return toObjectErr(err, bucket)
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (er erasureObjects) IsNotificationSupported() bool {
 	return true

cmd/erasure-decode_test.go

@@ -85,19 +85,17 @@ var erasureDecodeTests = []struct {
 
 func TestErasureDecode(t *testing.T) {
 	for i, test := range erasureDecodeTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		disks := setup.disks
 		data := make([]byte, test.data)
 		if _, err = io.ReadFull(crand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
 
@@ -113,11 +111,9 @@ func TestErasureDecode(t *testing.T) {
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
 		}
 		if n != test.data {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file", i)
 		}
 		for i, w := range writers {
@@ -195,7 +191,6 @@ func TestErasureDecode(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }
 
@@ -210,12 +205,11 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 	dataBlocks := 7
 	parityBlocks := 7
 	blockSize := int64(1 * humanize.MiByte)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	setup, err := newErasureTestSetup(t, dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Error(err)
 		return
 	}
-	defer setup.Remove()
 	disks := setup.disks
 	erasure, err := NewErasure(context.Background(), dataBlocks, parityBlocks, blockSize)
 	if err != nil {
@@ -288,11 +282,10 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 // Benchmarks
 
 func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV2)
+	setup, err := newErasureTestSetup(b, data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
-	defer setup.Remove()
 	disks := setup.disks
 	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2)
 	if err != nil {

cmd/erasure-encode_test.go

@@ -87,21 +87,19 @@ var erasureEncodeTests = []struct {
 
 func TestErasureEncode(t *testing.T) {
 	for i, test := range erasureEncodeTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		disks := setup.disks
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 
 		data := make([]byte, test.data)
 		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
 		writers := make([]io.Writer, len(disks))
@@ -160,18 +158,16 @@ func TestErasureEncode(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }
 
 // Benchmarks
 
 func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV2)
+	setup, err := newErasureTestSetup(b, data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
-	defer setup.Remove()
 	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)

cmd/erasure-errors.go

@@ -20,10 +20,10 @@ package cmd
 import "errors"
 
 // errErasureReadQuorum - did not meet read quorum.
-var errErasureReadQuorum = errors.New("Read failed. Insufficient number of disks online")
+var errErasureReadQuorum = errors.New("Read failed. Insufficient number of drives online")
 
 // errErasureWriteQuorum - did not meet write quorum.
-var errErasureWriteQuorum = errors.New("Write failed. Insufficient number of disks online")
+var errErasureWriteQuorum = errors.New("Write failed. Insufficient number of drives online")
 
 // errNoHealRequired - returned when healing is attempted on a previously healed disks.
 var errNoHealRequired = errors.New("No healing is required")

cmd/erasure-heal_test.go

@@ -66,23 +66,21 @@ func TestErasureHeal(t *testing.T) {
 	for i, test := range erasureHealTests {
 		if test.offDisks < test.badStaleDisks {
 			// test case sanity check
-			t.Fatalf("Test %d: Bad test case - number of stale disks cannot be less than number of badstale disks", i)
+			t.Fatalf("Test %d: Bad test case - number of stale drives cannot be less than number of badstale drives", i)
 		}
 
 		// create some test data
-		setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to setup Erasure environment: %v", i, err)
 		}
 		disks := setup.disks
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		data := make([]byte, test.size)
 		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
@@ -93,7 +91,6 @@ func TestErasureHeal(t *testing.T) {
 		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 
@@ -156,6 +153,5 @@ func TestErasureHeal(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }

cmd/erasure-healing-common.go

@@ -231,8 +231,8 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultPar
 //
 // - disks which have all parts specified in the latest xl.meta.
 //
-// - slice of errors about the state of data files on disk - can have
-//   a not-found error or a hash-mismatch error.
+//   - slice of errors about the state of data files on disk - can have
+//     a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
 	errs []error, latestMeta FileInfo, bucket, object string,
 	scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time,

cmd/erasure-healing-common_test.go

@@ -176,7 +176,7 @@ func TestListOnlineDisks(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -260,7 +260,7 @@ func TestListOnlineDisks(t *testing.T) {
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
-					t.Fatalf("disk (%v) with part.1 missing is not a disk with available data",
+					t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
 						erasureDisks[tamperedIndex])
 				}
 			}
@@ -351,7 +351,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -446,7 +446,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 
 			if test._tamperBackend != noTamper {
 				if tamperedIndex != -1 && availableDisks[tamperedIndex] != nil {
-					t.Fatalf("disk (%v) with part.1 missing is not a disk with available data",
+					t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data",
 						erasureDisks[tamperedIndex])
 				}
 			}
@@ -472,7 +472,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	z := obj.(*erasureServerPools)
 	s := z.serverPools[0].sets[0]
 	erasureDisks := s.getDisks()
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -506,7 +506,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 
 	for diskIndex, disk := range filteredDisks {
@@ -515,7 +515,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		}
 
 		if disk == nil {
-			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+			t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 		}
 	}
 
@@ -528,14 +528,14 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 	for diskIndex, disk := range filteredDisks {
 		if diskIndex == 0 && disk != nil {
-			t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+			t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 		}
 		if diskIndex != 0 && disk == nil {
-			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+			t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 		}
 	}
 	partsMetadata[0] = partsMetadataBackup // Revert before going to the next test
@@ -549,14 +549,14 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 	for diskIndex, disk := range filteredDisks {
 		if diskIndex == 1 && disk != nil {
-			t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+			t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 		}
 		if diskIndex != 1 && disk == nil {
-			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+			t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 		}
 	}
 	partsMetadata[1] = partsMetadataBackup // Revert before going to the next test
@@ -586,23 +586,23 @@ func TestDisksWithAllParts(t *testing.T) {
 		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
-		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+		t.Errorf("Unexpected number of drives: %d", len(filteredDisks))
 	}
 
 	for diskIndex, disk := range filteredDisks {
 		if _, ok := diskFailures[diskIndex]; ok {
 			if disk != nil {
-				t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+				t.Errorf("Drive not filtered as expected, drive: %d", diskIndex)
 			}
 			if errs[diskIndex] == nil {
-				t.Errorf("Expected error not received, diskIndex: %d", diskIndex)
+				t.Errorf("Expected error not received, driveIndex: %d", diskIndex)
 			}
 		} else {
 			if disk == nil {
-				t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+				t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex)
 			}
 			if errs[diskIndex] != nil {
-				t.Errorf("Unexpected error, %s, diskIndex: %d", errs[diskIndex], diskIndex)
+				t.Errorf("Unexpected error, %s, driveIndex: %d", errs[diskIndex], diskIndex)
 			}
 
 		}

cmd/erasure-healing.go

@@ -33,6 +33,15 @@ import (
 
 const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
 
+//go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE
+
+type healingMetric uint8
+
+const (
+	healingMetricBucket healingMetric = iota
+	healingMetricObject
+)
+
 // AcceptableDelta returns 'true' if the fi.DiskMTime is under
 // acceptable delta of "delta" duration with maxTime.
 //
@@ -81,6 +90,13 @@ func (er erasureObjects) healBucket(ctx context.Context, storageDisks []StorageA
 		writeQuorum++
 	}
 
+	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
+		startTime := time.Now()
+		defer func() {
+			healTrace(healingMetricBucket, startTime, bucket, "", "", opts, err, &res)
+		}()
+	}
+
 	// Initialize sync waitgroup.
 	g := errgroup.WithNErrs(len(storageDisks))
 
@@ -243,7 +259,7 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
-func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo) bool {
+func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta FileInfo, doinline bool) bool {
 	switch {
 	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
@@ -256,6 +272,10 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 			// always check first.
 			return true
 		}
+		if doinline {
+			// convert small files to 'inline'
+			return true
+		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
@@ -275,16 +295,18 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
-	if !opts.DryRun {
-		defer NSUpdated(bucket, object)
-	}
-
 	dryRun := opts.DryRun
 	scanMode := opts.ScanMode
 
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
+	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
+		startTime := time.Now()
+		defer func() {
+			healTrace(healingMetricObject, startTime, bucket, object, versionID, opts, err, &result)
+		}()
+	}
 	// Initialize heal result object
 	result = madmin.HealResultItem{
 		Type:      madmin.HealItemObject,
@@ -346,6 +368,26 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
+	var erasure Erasure
+	var recreate bool
+	if !latestMeta.Deleted && !latestMeta.IsRemote() {
+		// Initialize erasure coding
+		erasure, err = NewErasure(ctx, latestMeta.Erasure.DataBlocks,
+			latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
+		if err != nil {
+			return result, err
+		}
+
+		// Is only 'true' if the opts.Recreate is true and
+		// the object shardSize < smallFileThreshold do not
+		// set this to 'true' arbitrarily and must be only
+		// 'true' with caller ask.
+		recreate = (opts.Recreate &&
+			!latestMeta.InlineData() &&
+			len(latestMeta.Parts) == 1 &&
+			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
+	}
+
 	// Loop to find number of disks with valid data, per-drive
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
@@ -372,7 +414,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta, recreate) {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
@@ -426,9 +468,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		return result, nil
 	}
 
-	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// When disk to heal count is greater than parity blocks we should simply error out.
-		err := fmt.Errorf("more disks are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", errs, dataErrs, bucket, object, versionID)
+		err := fmt.Errorf("more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", errs, dataErrs, bucket, object, versionID)
 		logger.LogIf(ctx, err)
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
@@ -482,18 +524,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	}
 
 	var inlineBuffers []*bytes.Buffer
-	if latestMeta.InlineData() {
-		inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
-	}
-
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		// Heal each part. erasureHealFile() will write the healed
-		// part to .minio/tmp/uuid/ which needs to be renamed later to
-		// the final location.
-		erasure, err := NewErasure(ctx, latestMeta.Erasure.DataBlocks,
-			latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
-		if err != nil {
-			return result, err
+		if latestMeta.InlineData() || recreate {
+			inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 		}
 
 		erasureInfo := latestMeta.Erasure
@@ -503,6 +536,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			partModTime := latestMeta.Parts[partIndex].ModTime
 			partNumber := latestMeta.Parts[partIndex].Number
 			partIdx := latestMeta.Parts[partIndex].Index
+			partChecksums := latestMeta.Parts[partIndex].Checksums
 			tillOffset := erasure.ShardFileOffset(0, partSize, partSize)
 			readers := make([]io.ReaderAt, len(latestDisks))
 			checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm
@@ -529,6 +563,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
+
+			// Heal each part. erasure.Heal() will write the healed
+			// part to .minio/tmp/uuid/ which needs to be renamed
+			// later to the final location.
 			err = erasure.Heal(ctx, writers, readers, partSize)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
@@ -552,7 +590,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 
 				partsMetadata[i].DataDir = dstDataDir
-				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize, partModTime, partIdx)
+				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize, partModTime, partIdx, partChecksums)
 				partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 					PartNumber: partNumber,
 					Algorithm:  checksumAlgo,
@@ -560,6 +598,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				})
 				if len(inlineBuffers) > 0 && inlineBuffers[i] != nil {
 					partsMetadata[i].Data = inlineBuffers[i].Bytes()
+					partsMetadata[i].SetInlineData()
 				} else {
 					partsMetadata[i].Data = nil
 				}
@@ -567,7 +606,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 			// If all disks are having errors, we give up.
 			if disksToHealCount == 0 {
-				return result, fmt.Errorf("all disks had write errors, unable to heal %s/%s", bucket, object)
+				return result, fmt.Errorf("all drives had write errors, unable to heal %s/%s", bucket, object)
 			}
 
 		}
@@ -586,13 +625,14 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		partsMetadata[i].Erasure.Index = i + 1
 
 		// Attempt a rename now from healed data to final location.
-		if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
+		if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
 			logger.LogIf(ctx, err)
 			return result, err
 		}
 
-		// Remove any remaining parts from outdated disks from before transition.
-		if partsMetadata[i].IsRemote() {
+		// - Remove any parts from healed disks after its been inlined.
+		// - Remove any remaining parts from outdated disks from before transition.
+		if recreate || partsMetadata[i].IsRemote() {
 			rmDataDir := partsMetadata[i].DataDir
 			disk.DeleteVol(ctx, pathJoin(bucket, encodeDirObject(object), rmDataDir), true)
 		}
@@ -980,3 +1020,25 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	}
 	return hr, toObjectErr(err, bucket, object, versionID)
 }
+
+// healTrace sends healing results to trace output.
+func healTrace(funcName healingMetric, startTime time.Time, bucket, object, versionID string, opts madmin.HealOpts, err error, result *madmin.HealResultItem) {
+	tr := madmin.TraceInfo{
+		TraceType: madmin.TraceHealing,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  "heal." + funcName.String(),
+		Duration:  time.Since(startTime),
+		Message:   fmt.Sprintf("dry:%v, rm:%v, recreate:%v mode:%v", opts.DryRun, opts.Remove, opts.Recreate, opts.ScanMode),
+		Path:      pathJoin(bucket, decodeDirObject(object)),
+	}
+	if versionID != "" && versionID != "null" {
+		tr.Path += " v=" + versionID
+	}
+	if err != nil {
+		tr.Error = err.Error()
+	} else {
+		tr.HealResult = result
+	}
+	globalTrace.Publish(tr)
+}

cmd/erasure-healing_test.go

@@ -236,7 +236,7 @@ func TestHealing(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -359,7 +359,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	object := getRandomObjectName()
 	data := bytes.Repeat([]byte("a"), 128*1024)
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -554,21 +554,21 @@ func TestHealCorrectQuorum(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
 	// Create an object with multiple parts uploaded in decreasing
 	// part number.
-	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	res, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
 	if err != nil {
 		t.Fatalf("Failed to create a multipart upload - %v", err)
 	}
 
 	var uploadedParts []CompletePart
 	for _, partID := range []int{2, 1} {
-		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 		if err1 != nil {
 			t.Fatalf("Failed to upload a part - %v", err1)
 		}
@@ -578,7 +578,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 		})
 	}
 
-	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Failed to complete multipart upload - got: %v", err)
 	}
@@ -685,7 +685,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -694,10 +694,11 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	z := objLayer.(*erasureServerPools)
 	set := z.serverPools[1]
 
-	uploadID, err := set.NewMultipartUpload(ctx, bucket, object, opts)
+	res, err := set.NewMultipartUpload(ctx, bucket, object, opts)
 	if err != nil {
 		t.Fatalf("Failed to create a multipart upload - %v", err)
 	}
+	uploadID := res.UploadID
 
 	var uploadedParts []CompletePart
 	for _, partID := range []int{2, 1} {
@@ -861,21 +862,21 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
 	// Create an object with multiple parts uploaded in decreasing
 	// part number.
-	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	res, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
 	if err != nil {
 		t.Fatalf("Failed to create a multipart upload - %v", err)
 	}
 
 	var uploadedParts []CompletePart
 	for _, partID := range []int{2, 1} {
-		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 		if err1 != nil {
 			t.Fatalf("Failed to upload a part - %v", err1)
 		}
@@ -885,7 +886,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		})
 	}
 
-	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
@@ -1004,21 +1005,21 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
 	// Create an object with multiple parts uploaded in decreasing
 	// part number.
-	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	res, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
 	if err != nil {
 		t.Fatalf("Failed to create a multipart upload - %v", err)
 	}
 
 	var uploadedParts []CompletePart
 	for _, partID := range []int{2, 1} {
-		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 		if err1 != nil {
 			t.Fatalf("Failed to upload a part - %v", err1)
 		}
@@ -1028,7 +1029,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 		})
 	}
 
-	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
@@ -1161,21 +1162,21 @@ func TestHealObjectErasure(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
 	// Create an object with multiple parts uploaded in decreasing
 	// part number.
-	uploadID, err := obj.NewMultipartUpload(ctx, bucket, object, opts)
+	res, err := obj.NewMultipartUpload(ctx, bucket, object, opts)
 	if err != nil {
 		t.Fatalf("Failed to create a multipart upload - %v", err)
 	}
 
 	var uploadedParts []CompletePart
 	for _, partID := range []int{2, 1} {
-		pInfo, err1 := obj.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		pInfo, err1 := obj.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 		if err1 != nil {
 			t.Fatalf("Failed to upload a part - %v", err1)
 		}
@@ -1190,7 +1191,7 @@ func TestHealObjectErasure(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 	firstDisk := er.getDisks()[0]
 
-	_, err = obj.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	_, err = obj.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
@@ -1258,7 +1259,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	object := "empty-dir/"
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1359,7 +1360,7 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 			var opts ObjectOptions
 
-			err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				t.Fatalf("Failed to make a bucket - %v", err)
 			}

cmd/erasure-metadata-utils.go

@@ -27,6 +27,30 @@ import (
 	"github.com/minio/minio/internal/sync/errgroup"
 )
 
+// figure out the most commonVersions across disk that satisfies
+// the 'writeQuorum' this function returns '0' if quorum cannot
+// be achieved and disks have too many inconsistent versions.
+func reduceCommonVersions(diskVersions []uint64, writeQuorum int) (commonVersions uint64) {
+	diskVersionsCount := make(map[uint64]int)
+	for _, versions := range diskVersions {
+		diskVersionsCount[versions]++
+	}
+
+	max := 0
+	for versions, count := range diskVersionsCount {
+		if max < count {
+			max = count
+			commonVersions = versions
+		}
+	}
+
+	if max >= writeQuorum {
+		return commonVersions
+	}
+
+	return 0
+}
+
 // Returns number of errors that occurred the most (incl. nil) and the
 // corresponding error value. NB When there is more than one error value that
 // occurs maximum number of times, the error value returned depends on how
@@ -275,7 +299,7 @@ func shuffleDisks(disks []StorageAPI, distribution []int) (shuffledDisks []Stora
 // the corresponding error in errs slice is not nil
 func evalDisks(disks []StorageAPI, errs []error) []StorageAPI {
 	if len(errs) != len(disks) {
-		logger.LogIf(GlobalContext, errors.New("unexpected disks/errors slice length"))
+		logger.LogIf(GlobalContext, errors.New("unexpected drives/errors slice length"))
 		return nil
 	}
 	newDisks := make([]StorageAPI, len(disks))

cmd/erasure-metadata.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
@@ -127,13 +128,8 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 		SuccessorModTime: fi.SuccessorModTime,
 	}
 
-	// Update expires
-	var (
-		t time.Time
-		e error
-	)
 	if exp, ok := fi.Metadata["expires"]; ok {
-		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+		if t, err := amztime.ParseHeader(exp); err == nil {
 			objInfo.Expires = t.UTC()
 		}
 	}
@@ -187,6 +183,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 			objInfo.RestoreExpires, _ = restoreStatus.Expiry()
 		}
 	}
+	objInfo.Checksum = fi.Checksum
 	// Success.
 	return objInfo
 }
@@ -237,7 +234,7 @@ func objectPartIndex(parts []ObjectPartInfo, partNumber int) int {
 }
 
 // AddObjectPart - add a new object part in order.
-func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, actualSize int64, modTime time.Time, idx []byte) {
+func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, actualSize int64, modTime time.Time, idx []byte, checksums map[string]string) {
 	partInfo := ObjectPartInfo{
 		Number:     partNumber,
 		ETag:       partETag,
@@ -245,6 +242,7 @@ func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, act
 		ActualSize: actualSize,
 		ModTime:    modTime,
 		Index:      idx,
+		Checksums:  checksums,
 	}
 
 	// Update part info if it already exists.

cmd/erasure-metadata_test.go

@@ -58,7 +58,7 @@ func TestAddObjectPart(t *testing.T) {
 	for _, testCase := range testCases {
 		if testCase.expectedIndex > -1 {
 			partNumString := strconv.Itoa(testCase.partNum)
-			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil)
+			fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil, nil)
 		}
 
 		if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex {
@@ -91,7 +91,7 @@ func TestObjectPartIndex(t *testing.T) {
 	// Add some parts for testing.
 	for _, testCase := range testCases {
 		partNumString := strconv.Itoa(testCase.partNum)
-		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil)
+		fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil, nil)
 	}
 
 	// Add failure test case.
@@ -121,7 +121,7 @@ func TestObjectToPartOffset(t *testing.T) {
 	// Total size of all parts is 5,242,899 bytes.
 	for _, partNum := range []int{1, 2, 4, 5, 7} {
 		partNumString := strconv.Itoa(partNum)
-		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize, UTCNow(), nil)
+		fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize, UTCNow(), nil, nil)
 	}
 
 	testCases := []struct {
@@ -160,7 +160,7 @@ func TestObjectToPartOffset(t *testing.T) {
 func TestFindFileInfoInQuorum(t *testing.T) {
 	getNFInfo := func(n int, quorum int, t int64, dataDir string) []FileInfo {
 		fi := newFileInfo("test", 8, 8)
-		fi.AddObjectPart(1, "etag", 100, 100, UTCNow(), nil)
+		fi.AddObjectPart(1, "etag", 100, 100, UTCNow(), nil, nil)
 		fi.ModTime = time.Unix(t, 0)
 		fi.DataDir = dataDir
 		fis := make([]FileInfo, n)

cmd/erasure-multipart.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -47,33 +48,50 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
 }
 
 // checkUploadIDExists - verify if a given uploadID exists and is valid.
-func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string) (err error) {
+func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) {
 	defer func() {
 		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
 			err = errUploadIDNotFound
 		}
 	}()
 
-	disks := er.getDisks()
+	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
+
+	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "", false)
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket,
+		uploadIDPath, "", false)
 
-	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
+	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
-		return err
-	}
-
-	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
-		return reducedErr
+		return fi, nil, err
 	}
 
 	// List all online disks.
-	_, modTime := listOnlineDisks(disks, metaArr, errs)
+	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
-	// Pick latest valid metadata.
-	_, err = pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
-	return err
+	var quorum int
+	if write {
+		reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+		if reducedErr == errErasureWriteQuorum {
+			return fi, nil, reducedErr
+		}
+
+		quorum = writeQuorum
+	} else {
+		if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
+			return fi, nil, reducedErr
+		}
+
+		// Pick one from the first valid metadata.
+		quorum = readQuorum
+	}
+
+	// Pick one from the first valid metadata.
+	fi, err = pickValidFileInfo(ctx, partsMetadata, modTime, quorum)
+
+	return fi, partsMetadata, err
 }
 
 // Removes part.meta given by partName belonging to a mulitpart upload from minioMetaBucket
@@ -325,7 +343,25 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 // '.minio.sys/multipart/bucket/object/uploads.json' on all the
 // disks. `uploads.json` carries metadata regarding on-going multipart
 // operation(s) on the object.
-func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, object string, opts ObjectOptions) (string, error) {
+func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) {
+	if opts.CheckPrecondFn != nil {
+		// Lock the object before reading.
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return nil, err
+		}
+		rctx := lkctx.Context()
+		obj, err := er.getObjectInfo(rctx, bucket, object, opts)
+		lk.RUnlock(lkctx.Cancel)
+		if err != nil {
+			return nil, err
+		}
+		if opts.CheckPrecondFn(obj) {
+			return nil, PreConditionFailed{}
+		}
+	}
+
 	userDefined := cloneMSS(opts.UserDefined)
 
 	onlineDisks := er.getDisks()
@@ -352,7 +388,6 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	if parityOrig != parityDrives {
 		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 	}
-
 	dataDrives := len(onlineDisks) - parityDrives
 
 	// we now know the number of blocks this object needs for data and parity.
@@ -382,6 +417,10 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object))
 	}
 
+	if opts.WantChecksum != nil && opts.WantChecksum.Type.IsSet() {
+		userDefined[hash.MinIOMultipartChecksum] = opts.WantChecksum.Type.String()
+	}
+
 	modTime := opts.MTime
 	if opts.MTime.IsZero() {
 		modTime = UTCNow()
@@ -402,11 +441,12 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 
 	// Write updated `xl.meta` to all disks.
 	if _, err := writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
-		return "", toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+		return nil, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
-
-	// Return success.
-	return uploadID, nil
+	return &NewMultipartUploadResult{
+		UploadID:     uploadID,
+		ChecksumAlgo: userDefined[hash.MinIOMultipartChecksum],
+	}, nil
 }
 
 // NewMultipartUpload - initialize a new multipart upload, returns a
@@ -414,7 +454,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 // subsequent request each UUID is unique.
 //
 // Implements S3 compatible initiate multipart API.
-func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (string, error) {
+func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) {
 	auditObjectErasureSet(ctx, object, &er)
 
 	return er.newMultipartUpload(ctx, bucket, object, opts)
@@ -558,41 +598,26 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return pi, toObjectErr(errInvalidArgument)
 	}
 
-	var partsMetadata []FileInfo
-	var errs []error
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 
 	// Validates if upload ID exists.
-	if err = er.checkUploadIDExists(rctx, bucket, object, uploadID); err != nil {
+	fi, _, err := er.checkUploadIDExists(rctx, bucket, object, uploadID, true)
+	if err != nil {
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	storageDisks := er.getDisks()
+	onlineDisks := er.getDisks()
+	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
 
-	// Read metadata associated with the object from all disks.
-	partsMetadata, errs = readAllFileInfo(rctx, storageDisks, minioMetaMultipartBucket,
-		uploadIDPath, "", false)
-
-	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(pctx, partsMetadata, errs, er.defaultParityCount)
-	if err != nil {
-		return pi, toObjectErr(err, bucket, object)
+	if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
+		if r.ContentCRCType().String() != cs {
+			return pi, InvalidArgument{
+				Bucket: bucket,
+				Object: fi.Name,
+				Err:    fmt.Errorf("checksum missing, want %s, got %s", cs, r.ContentCRCType().String()),
+			}
+		}
 	}
-
-	reducedErr := reduceWriteQuorumErrs(pctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if reducedErr == errErasureWriteQuorum {
-		return pi, toObjectErr(reducedErr, bucket, object)
-	}
-
-	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
-
-	// Pick one from the first valid metadata.
-	fi, err := pickValidFileInfo(pctx, partsMetadata, modTime, writeQuorum)
-	if err != nil {
-		return pi, err
-	}
-
 	onlineDisks = shuffleDisks(onlineDisks, fi.Erasure.Distribution)
 
 	// Need a unique name for the part being written in minioMetaBucket to
@@ -696,33 +721,38 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		index = opts.IndexCB()
 	}
 
-	part := ObjectPartInfo{
+	partInfo := ObjectPartInfo{
 		Number:     partID,
 		ETag:       md5hex,
 		Size:       n,
 		ActualSize: data.ActualSize(),
 		ModTime:    UTCNow(),
 		Index:      index,
+		Checksums:  r.ContentCRC(),
 	}
-
-	partMsg, err := part.MarshalMsg(nil)
+	fi.Parts = []ObjectPartInfo{partInfo}
+	partFI, err := fi.MarshalMsg(nil)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
 	// Write part metadata to all disks.
-	onlineDisks, err = writeAllDisks(ctx, onlineDisks, minioMetaMultipartBucket, partPath+".meta", partMsg, writeQuorum)
+	onlineDisks, err = writeAllDisks(ctx, onlineDisks, minioMetaMultipartBucket, partPath+".meta", partFI, writeQuorum)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
 	// Return success.
 	return PartInfo{
-		PartNumber:   part.Number,
-		ETag:         part.ETag,
-		LastModified: part.ModTime,
-		Size:         part.Size,
-		ActualSize:   part.ActualSize,
+		PartNumber:     partInfo.Number,
+		ETag:           partInfo.ETag,
+		LastModified:   partInfo.ModTime,
+		Size:           partInfo.Size,
+		ActualSize:     partInfo.ActualSize,
+		ChecksumCRC32:  partInfo.Checksums["CRC32"],
+		ChecksumCRC32C: partInfo.Checksums["CRC32C"],
+		ChecksumSHA1:   partInfo.Checksums["SHA1"],
+		ChecksumSHA256: partInfo.Checksums["SHA256"],
 	}, nil
 }
 
@@ -748,36 +778,11 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 	ctx = lkctx.Context()
 	defer uploadIDLock.RUnlock(lkctx.Cancel)
 
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+	if err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-
-	storageDisks := er.getDisks()
-
-	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID, false)
-
-	// get Quorum for this object
-	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
-	if err != nil {
-		return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
-	}
-
-	reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
-	if reducedErr == errErasureReadQuorum {
-		return result, toObjectErr(reducedErr, minioMetaMultipartBucket, uploadIDPath)
-	}
-
-	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
-
-	// Pick one from the first valid metadata.
-	fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, readQuorum)
-	if err != nil {
-		return result, err
-	}
-
 	result.UserDefined = cloneMSS(fi.Metadata)
 	return result, nil
 }
@@ -800,36 +805,14 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	ctx = lkctx.Context()
 	defer uploadIDLock.RUnlock(lkctx.Cancel)
 
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+	if err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
 
+	onlineDisks := er.getDisks()
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 
-	storageDisks := er.getDisks()
-
-	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
-
-	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
-	if err != nil {
-		return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
-	}
-
-	reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if reducedErr == errErasureWriteQuorum {
-		return result, toObjectErr(reducedErr, minioMetaMultipartBucket, uploadIDPath)
-	}
-
-	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
-
-	// Pick one from the first valid metadata.
-	fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, writeQuorum)
-	if err != nil {
-		return result, err
-	}
-
 	if maxParts == 0 {
 		return result, nil
 	}
@@ -860,6 +843,8 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
 	}
 
+	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
+
 	partInfoFiles, err := readMultipleFiles(ctx, onlineDisks, req, writeQuorum)
 	if err != nil {
 		return result, err
@@ -872,33 +857,35 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	result.MaxParts = maxParts
 	result.PartNumberMarker = partNumberMarker
 	result.UserDefined = cloneMSS(fi.Metadata)
+	result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]
 
 	// For empty number of parts or maxParts as zero, return right here.
 	if len(partInfoFiles) == 0 || maxParts == 0 {
 		return result, nil
 	}
 
-	var partI ObjectPartInfo
 	for i, part := range partInfoFiles {
 		partN := i + partNumberMarker + 1
 		if part.Error != "" || !part.Exists {
 			continue
 		}
 
-		_, err := partI.UnmarshalMsg(part.Data)
+		var pfi FileInfo
+		_, err := pfi.UnmarshalMsg(part.Data)
 		if err != nil {
 			// Maybe crash or similar.
 			logger.LogIf(ctx, err)
 			continue
 		}
 
+		partI := pfi.Parts[0]
 		if partN != partI.Number {
 			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", i+1, i+1, partI.Number))
 			continue
 		}
 
 		// Add the current part.
-		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index)
+		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index, partI.Checksums)
 	}
 
 	// Only parts with higher part numbers will be listed.
@@ -906,11 +893,15 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	result.Parts = make([]PartInfo, 0, len(parts))
 	for _, part := range parts {
 		result.Parts = append(result.Parts, PartInfo{
-			PartNumber:   part.Number,
-			ETag:         part.ETag,
-			LastModified: part.ModTime,
-			ActualSize:   part.ActualSize,
-			Size:         part.Size,
+			PartNumber:     part.Number,
+			ETag:           part.ETag,
+			LastModified:   part.ModTime,
+			ActualSize:     part.ActualSize,
+			Size:           part.Size,
+			ChecksumCRC32:  part.Checksums["CRC32"],
+			ChecksumCRC32C: part.Checksums["CRC32C"],
+			ChecksumSHA1:   part.Checksums["SHA1"],
+			ChecksumSHA256: part.Checksums["SHA256"],
 		})
 		if len(result.Parts) >= maxParts {
 			break
@@ -947,35 +938,14 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	wctx := wlkctx.Context()
 	defer uploadIDLock.Unlock(wlkctx.Cancel)
 
-	if err = er.checkUploadIDExists(wctx, bucket, object, uploadID); err != nil {
+	fi, partsMetadata, err := er.checkUploadIDExists(wctx, bucket, object, uploadID, true)
+	if err != nil {
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-
-	storageDisks := er.getDisks()
-
-	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(wctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
-
-	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(wctx, partsMetadata, errs, er.defaultParityCount)
-	if err != nil {
-		return oi, toObjectErr(err, bucket, object)
-	}
-
-	reducedErr := reduceWriteQuorumErrs(wctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if reducedErr == errErasureWriteQuorum {
-		return oi, toObjectErr(reducedErr, bucket, object)
-	}
-
-	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
-
-	// Pick one from the first valid metadata.
-	fi, err := pickValidFileInfo(wctx, partsMetadata, modTime, writeQuorum)
-	if err != nil {
-		return oi, err
-	}
+	onlineDisks := er.getDisks()
+	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
 
 	// Read Part info for all parts
 	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
@@ -1000,7 +970,20 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, toObjectErr(err, bucket, object)
 	}
 
-	var partI ObjectPartInfo
+	// Checksum type set when upload started.
+	var checksumType hash.ChecksumType
+	if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
+		checksumType = hash.NewChecksumType(cs)
+		if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) {
+			return oi, InvalidArgument{
+				Bucket: bucket,
+				Object: fi.Name,
+				Err:    fmt.Errorf("checksum type mismatch"),
+			}
+		}
+	}
+	var checksumCombined []byte
+
 	for i, part := range partInfoFiles {
 		partID := parts[i].PartNumber
 		if part.Error != "" || !part.Exists {
@@ -1009,7 +992,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			}
 		}
 
-		_, err := partI.UnmarshalMsg(part.Data)
+		var pfi FileInfo
+		_, err := pfi.UnmarshalMsg(part.Data)
 		if err != nil {
 			// Maybe crash or similar.
 			logger.LogIf(ctx, err)
@@ -1018,7 +1002,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			}
 		}
 
-		if partID != partI.Number {
+		partI := pfi.Parts[0]
+		partNumber := partI.Number
+		if partID != partNumber {
 			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
 			return oi, InvalidPart{
 				PartNumber: partID,
@@ -1026,7 +1012,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 
 		// Add the current part.
-		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index)
+		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index, partI.Checksums)
 	}
 
 	// Calculate full object size.
@@ -1056,43 +1042,89 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			}
 			return oi, invp
 		}
+		gotPart := currentFI.Parts[partIdx]
 
 		// ensure that part ETag is canonicalized to strip off extraneous quotes
 		part.ETag = canonicalizeETag(part.ETag)
-		if currentFI.Parts[partIdx].ETag != part.ETag {
+		if gotPart.ETag != part.ETag {
 			invp := InvalidPart{
 				PartNumber: part.PartNumber,
-				ExpETag:    currentFI.Parts[partIdx].ETag,
+				ExpETag:    gotPart.ETag,
 				GotETag:    part.ETag,
 			}
 			return oi, invp
 		}
 
+		if checksumType.IsSet() {
+			crc := gotPart.Checksums[checksumType.String()]
+			if crc == "" {
+				return oi, InvalidPart{
+					PartNumber: part.PartNumber,
+				}
+			}
+			wantCS := map[string]string{
+				hash.ChecksumCRC32.String():  part.ChecksumCRC32,
+				hash.ChecksumCRC32C.String(): part.ChecksumCRC32C,
+				hash.ChecksumSHA1.String():   part.ChecksumSHA1,
+				hash.ChecksumSHA256.String(): part.ChecksumSHA256,
+			}
+			if wantCS[checksumType.String()] != crc {
+				return oi, InvalidPart{
+					PartNumber: part.PartNumber,
+					ExpETag:    wantCS[checksumType.String()],
+					GotETag:    crc,
+				}
+			}
+			cs := hash.NewChecksumString(checksumType.String(), crc)
+			if !cs.Valid() {
+				return oi, InvalidPart{
+					PartNumber: part.PartNumber,
+				}
+			}
+			checksumCombined = append(checksumCombined, cs.Raw()...)
+		}
+
 		// All parts except the last part has to be at least 5MB.
 		if (i < len(parts)-1) && !isMinAllowedPartSize(currentFI.Parts[partIdx].ActualSize) {
 			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
-				PartSize:   currentFI.Parts[partIdx].ActualSize,
+				PartSize:   gotPart.ActualSize,
 				PartETag:   part.ETag,
 			}
 		}
 
 		// Save for total object size.
-		objectSize += currentFI.Parts[partIdx].Size
+		objectSize += gotPart.Size
 
 		// Save the consolidated actual size.
-		objectActualSize += currentFI.Parts[partIdx].ActualSize
+		objectActualSize += gotPart.ActualSize
 
 		// Add incoming parts.
 		fi.Parts[i] = ObjectPartInfo{
 			Number:     part.PartNumber,
-			Size:       currentFI.Parts[partIdx].Size,
-			ActualSize: currentFI.Parts[partIdx].ActualSize,
-			ModTime:    currentFI.Parts[partIdx].ModTime,
-			Index:      currentFI.Parts[partIdx].Index,
+			Size:       gotPart.Size,
+			ActualSize: gotPart.ActualSize,
+			ModTime:    gotPart.ModTime,
+			Index:      gotPart.Index,
+			Checksums:  nil, // Not transferred since we do not need it.
 		}
 	}
 
+	if opts.WantChecksum != nil {
+		err := opts.WantChecksum.Matches(checksumCombined)
+		if err != nil {
+			return oi, err
+		}
+	}
+	if checksumType.IsSet() {
+		cs := hash.NewChecksumFromData(checksumType, checksumCombined)
+		fi.Checksum = cs.AppendTo(nil)
+		if opts.EncryptFn != nil {
+			fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
+		}
+	}
+	delete(fi.Metadata, hash.MinIOMultipartChecksum) // Not needed in final object.
+
 	// Save the final object size and modtime.
 	fi.Size = objectSize
 	fi.ModTime = opts.MTime
@@ -1159,6 +1191,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, toObjectErr(err, bucket, object)
 	}
 
+	defer NSUpdated(bucket, object)
+
 	// Check if there is any offline disk and add it to the MRF list
 	for _, disk := range onlineDisks {
 		if disk != nil && disk.IsOnline() {
@@ -1203,7 +1237,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 	defer lk.Unlock(lkctx.Cancel)
 
 	// Validates if upload ID exists.
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false); err != nil {
 		return toObjectErr(err, bucket, object, uploadID)
 	}
 

cmd/erasure-object.go

@@ -42,6 +42,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/mimedb"
+	"github.com/minio/pkg/wildcard"
 	uatomic "go.uber.org/atomic"
 )
 
@@ -448,15 +449,26 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		}
 		defer NSUpdated(bucket, object)
 
-		if opts.VersionID != "" {
-			er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
-				VersionID: opts.VersionID,
-			}, false)
-		} else {
-			er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
-				VersionID: m.VersionID,
-			}, false)
+		fi := FileInfo{
+			VersionID: m.VersionID,
 		}
+		if opts.VersionID != "" {
+			fi.VersionID = opts.VersionID
+		}
+
+		disks := er.getDisks()
+		g := errgroup.WithNErrs(len(disks))
+		for index := range disks {
+			index := index
+			g.Go(func() error {
+				if disks[index] == nil {
+					return errDiskNotFound
+				}
+				return disks[index].DeleteVersion(ctx, bucket, object, fi, false)
+			}, index)
+		}
+
+		g.Wait()
 	}
 	return m, err
 }
@@ -491,17 +503,26 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		}, index)
 	}
 
+	ignoredErrs := []error{
+		errFileNotFound,
+		errVolumeNotFound,
+		errFileVersionNotFound,
+		errDiskNotFound,
+	}
+
 	errs := g.Wait()
 	for index, err := range errs {
 		if err == nil {
 			continue
 		}
-		if !IsErr(err, []error{
-			errFileNotFound,
-			errVolumeNotFound,
-			errFileVersionNotFound,
-			errDiskNotFound,
-		}...) {
+		if bucket == minioMetaBucket {
+			// minioMetaBucket "reads" for .metacache are not written with O_SYNC
+			// so there is a potential for them to not fully committed to stable
+			// storage leading to unexpected EOFs. Allow these failures to
+			// be ignored since the caller already ignores them in streamMetadataParts()
+			ignoredErrs = append(ignoredErrs, io.ErrUnexpectedEOF, io.EOF)
+		}
+		if !IsErr(err, ignoredErrs...) {
 			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], bucket, object, err),
 				disks[index].String())
@@ -534,6 +555,11 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		return metaFileInfos, errs
 	}
 
+	versionID := lfi.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+
 	for index := range metadataArray {
 		if metadataArray[index] == nil {
 			continue
@@ -541,21 +567,15 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 
 		// make sure to preserve this for diskmtime based healing bugfix.
 		diskMTime := metaFileInfos[index].DiskMTime
-		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, "")
+		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID)
 		if errs[index] != nil {
 			continue
 		}
 
-		if metaFileInfos[index].IsValid() && metaFileInfos[index].ModTime.Equal(lfi.ModTime) {
-			versionID := metaFileInfos[index].VersionID
-			if versionID == "" {
-				versionID = nullVersionID
-			}
-			if readData {
-				metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
-			}
-			metaFileInfos[index].DiskMTime = diskMTime
+		if readData {
+			metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
 		}
+		metaFileInfos[index].DiskMTime = diskMTime
 	}
 
 	// Return all the metadata.
@@ -663,10 +683,7 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 		return objInfo, er.defaultWQuorum(), toObjectErr(err, bucket, object)
 	}
 
-	wquorum = fi.Erasure.DataBlocks
-	if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
-		wquorum++
-	}
+	wquorum = fi.WriteQuorum(er.defaultWQuorum())
 
 	objInfo = fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if !fi.VersionPurgeStatus().Empty() && opts.VersionID != "" {
@@ -687,14 +704,14 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
 func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
-	defer NSUpdated(dstBucket, dstEntry)
-
 	g := errgroup.WithNErrs(len(disks))
 
 	fvID := mustGetUUID()
 	for index := range disks {
 		metadata[index].SetTierFreeVersionID(fvID)
 	}
+
+	diskVersions := make([]uint64, len(disks))
 	// Rename file on all underlying storage disks.
 	for index := range disks {
 		index := index
@@ -702,6 +719,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
+
 			// Pick one FileInfo for a disk at index.
 			fi := metadata[index]
 			// Assign index when index is initialized
@@ -709,19 +727,34 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 				fi.Erasure.Index = index + 1
 			}
 
-			if fi.IsValid() {
-				return disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry)
+			if !fi.IsValid() {
+				return errFileCorrupt
 			}
-			return errFileCorrupt
+			sign, err := disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry)
+			if err != nil {
+				return err
+			}
+			diskVersions[index] = sign
+			return nil
 		}, index)
 	}
 
 	// Wait for all renames to finish.
 	errs := g.Wait()
 
-	// We can safely allow RenameData errors up to len(er.getDisks()) - writeQuorum
-	// otherwise return failure. Cleanup successful renames.
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	if err == nil {
+		versions := reduceCommonVersions(diskVersions, writeQuorum)
+		for index, dversions := range diskVersions {
+			if versions != dversions {
+				errs[index] = errFileNeedsHealing
+			}
+		}
+		err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	}
+
+	// We can safely allow RenameData errors up to len(er.getDisks()) - writeQuorum
+	// otherwise return failure.
 	return evalDisks(disks, errs), err
 }
 
@@ -829,7 +862,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		partsMetadata[i].Data = inlineBuffers[i].Bytes()
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, index)
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, index, nil)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -879,6 +912,16 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)
 
+	if opts.CheckPrecondFn != nil {
+		obj, err := er.getObjectInfo(ctx, bucket, object, opts)
+		if err != nil {
+			return objInfo, err
+		}
+		if opts.CheckPrecondFn(obj) {
+			return objInfo, PreConditionFailed{}
+		}
+	}
+
 	data := r.Reader
 
 	userDefined := cloneMSS(opts.UserDefined)
@@ -954,6 +997,10 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	fi.DataDir = mustGetUUID()
+	fi.Checksum = opts.WantChecksum.AppendTo(nil)
+	if opts.EncryptFn != nil {
+		fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
+	}
 	uniqueID := mustGetUUID()
 	tempObj := uniqueID
 
@@ -1097,7 +1144,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		} else {
 			partsMetadata[i].Data = nil
 		}
-		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, compIndex)
+		// No need to add checksum to part. We already have it on the object.
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, compIndex, nil)
 		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
 			PartNumber: 1,
 			Algorithm:  DefaultBitrotAlgorithm,
@@ -1105,11 +1153,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		})
 	}
 
-	if userDefined["etag"] == "" {
-		userDefined["etag"] = r.MD5CurrentHexString()
-		if opts.PreserveETag != "" {
-			userDefined["etag"] = opts.PreserveETag
-		}
+	userDefined["etag"] = r.MD5CurrentHexString()
+	if opts.PreserveETag != "" {
+		userDefined["etag"] = opts.PreserveETag
 	}
 
 	// Guess content-type from the extension if possible.
@@ -1133,14 +1179,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	// Rename the successfully written temporary object to final location.
-	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum); err != nil {
-		if errors.Is(err, errFileNotFound) {
-			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
-		}
-		logger.LogIf(ctx, err)
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
-	}
-
+	onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick
@@ -1164,6 +1203,57 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		}
 	}
 
+	healEntry := func(entry metaCacheEntry) error {
+		if entry.isDir() {
+			return nil
+		}
+		// We might land at .metacache, .trash, .multipart
+		// no need to heal them skip, only when bucket
+		// is '.minio.sys'
+		if bucket == minioMetaBucket {
+			if wildcard.Match("buckets/*/.metacache/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("tmp/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("multipart/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("tmp-old/*", entry.name) {
+				return nil
+			}
+		}
+		fivs, err := entry.fileInfoVersions(bucket)
+		if err != nil {
+			_, err = er.HealObject(ctx, bucket, entry.name, "", madmin.HealOpts{NoLock: true})
+			return err
+		}
+
+		for _, version := range fivs.Versions {
+			_, err = er.HealObject(ctx, bucket, version.Name, version.VersionID, madmin.HealOpts{NoLock: true})
+			if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	if err != nil {
+		if errors.Is(err, errFileNotFound) {
+			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
+		}
+		if errors.Is(err, errFileNeedsHealing) && !opts.Speedtest {
+			listAndHeal(ctx, bucket, object, &er, healEntry)
+		} else {
+			logger.LogIf(ctx, err)
+			return ObjectInfo{}, toObjectErr(err, bucket, object)
+		}
+	}
+
+	defer NSUpdated(bucket, object)
+
 	fi.ReplicationState = opts.PutReplicationState()
 	online = countOnlineDisks(onlineDisks)
 
@@ -1173,8 +1263,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
-func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo, forceDelMarker bool) error {
+func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker bool) error {
 	disks := er.getDisks()
+	// Assume (N/2 + 1) quorum for Delete()
+	// this is a theoretical assumption such that
+	// for delete's we do not need to honor storage
+	// class for objects that have reduced quorum
+	// due to storage class - this only needs to be honored
+	// for Read() requests alone that we already do.
+	writeQuorum := len(disks)/2 + 1
+
 	g := errgroup.WithNErrs(len(disks))
 	for index := range disks {
 		index := index
@@ -1424,69 +1522,49 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	// Acquire a write lock before deleting the object.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
-	if err != nil {
-		return ObjectInfo{}, err
-	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
-
-	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
-	goi, writeQuorum, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
-	if gerr != nil && goi.Name == "" {
-		switch gerr.(type) {
-		case InsufficientReadQuorum:
-			return objInfo, InsufficientWriteQuorum{}
-		}
-		// For delete marker replication, versionID being replicated will not exist on disk
-		if opts.DeleteMarker {
-			versionFound = false
-		} else {
-			return objInfo, gerr
-		}
-	}
-
-	if opts.Expiration.Expire {
-		action := evalActionFromLifecycle(ctx, *lc, rcfg, goi, false)
-		var isErr bool
-		switch action {
-		case lifecycle.NoneAction:
-			isErr = true
-		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			isErr = true
-		}
-		if isErr {
-			if goi.VersionID != "" {
-				return goi, VersionNotFound{
-					Bucket:    bucket,
-					Object:    object,
-					VersionID: goi.VersionID,
-				}
-			}
-			return goi, ObjectNotFound{
-				Bucket: bucket,
-				Object: object,
-			}
-		}
-	}
-
 	defer NSUpdated(bucket, object)
 
 	storageDisks := er.getDisks()
 
-	var markDelete bool
-	// Determine whether to mark object deleted for replication
-	if goi.VersionID != "" {
-		markDelete = true
+	if opts.Expiration.Expire {
+		goi, _, err := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
+		if err == nil {
+			action := evalActionFromLifecycle(ctx, *lc, rcfg, goi, false)
+			var isErr bool
+			switch action {
+			case lifecycle.NoneAction:
+				isErr = true
+			case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+				isErr = true
+			}
+			if isErr {
+				if goi.VersionID != "" {
+					return goi, VersionNotFound{
+						Bucket:    bucket,
+						Object:    object,
+						VersionID: goi.VersionID,
+					}
+				}
+				return goi, ObjectNotFound{
+					Bucket: bucket,
+					Object: object,
+				}
+			}
+		}
 	}
 
-	// Default deleteMarker to true if object is under versioning
-	deleteMarker := opts.Versioned
+	versionFound := !(opts.DeleteMarker && opts.VersionID != "")
 
-	if opts.VersionID != "" {
+	// Determine whether to mark object deleted for replication
+	markDelete := !opts.DeleteMarker && opts.VersionID != ""
+
+	// Default deleteMarker to true if object is under versioning
+	// versioning suspended means we add `null` version as
+	// delete marker, if its not decided already.
+	deleteMarker := (opts.Versioned || opts.VersionSuspended) && opts.VersionID == "" || (opts.DeleteMarker && opts.VersionID != "")
+
+	if markDelete {
 		// case where replica version needs to be deleted on target cluster
 		if versionFound && opts.DeleteMarkerReplicationStatus() == replication.Replica {
 			markDelete = false
@@ -1497,22 +1575,6 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		if opts.VersionPurgeStatus() == Complete {
 			markDelete = false
 		}
-
-		// Version is found but we do not wish to create more delete markers
-		// now, since VersionPurgeStatus() is already set, we can let the
-		// lower layers decide this. This fixes a regression that was introduced
-		// in PR #14555 where !VersionPurgeStatus.Empty() is automatically
-		// considered as Delete marker true to avoid listing such objects by
-		// regular ListObjects() calls. However for delete replication this
-		// ends up being a problem because "upon" a successful delete this
-		// ends up creating a new delete marker that is spurious and unnecessary.
-		if versionFound {
-			if !goi.VersionPurgeStatus.Empty() {
-				deleteMarker = false
-			} else if !goi.DeleteMarker { // implies a versioned delete of object
-				deleteMarker = false
-			}
-		}
 	}
 
 	modTime := opts.MTime
@@ -1520,38 +1582,32 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		modTime = UTCNow()
 	}
 	fvID := mustGetUUID()
-	if markDelete {
-		if opts.Versioned || opts.VersionSuspended {
-			if !deleteMarker {
-				// versioning suspended means we add `null` version as
-				// delete marker, if its not decided already.
-				deleteMarker = opts.VersionSuspended && opts.VersionID == ""
-			}
-			fi := FileInfo{
-				Name:             object,
-				Deleted:          deleteMarker,
-				MarkDeleted:      markDelete,
-				ModTime:          modTime,
-				ReplicationState: opts.DeleteReplication,
-				TransitionStatus: opts.Transition.Status,
-				ExpireRestored:   opts.Transition.ExpireRestored,
-			}
-			fi.SetTierFreeVersionID(fvID)
-			if opts.Versioned {
-				fi.VersionID = mustGetUUID()
-				if opts.VersionID != "" {
-					fi.VersionID = opts.VersionID
-				}
-			}
-			// versioning suspended means we add `null` version as
-			// delete marker. Add delete marker, since we don't have
-			// any version specified explicitly. Or if a particular
-			// version id needs to be replicated.
-			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
-				return objInfo, toObjectErr(err, bucket, object)
-			}
-			return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
+
+	if markDelete && (opts.Versioned || opts.VersionSuspended) {
+		fi := FileInfo{
+			Name:             object,
+			Deleted:          true,
+			MarkDeleted:      markDelete,
+			ModTime:          modTime,
+			ReplicationState: opts.DeleteReplication,
+			TransitionStatus: opts.Transition.Status,
+			ExpireRestored:   opts.Transition.ExpireRestored,
 		}
+		fi.SetTierFreeVersionID(fvID)
+		if opts.Versioned {
+			fi.VersionID = mustGetUUID()
+			if opts.VersionID != "" {
+				fi.VersionID = opts.VersionID
+			}
+		}
+		// versioning suspended means we add `null` version as
+		// delete marker. Add delete marker, since we don't have
+		// any version specified explicitly. Or if a particular
+		// version id needs to be replicated.
+		if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
+			return objInfo, toObjectErr(err, bucket, object)
+		}
+		return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 	}
 
 	// Delete the object version on all disks.
@@ -1566,7 +1622,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		ExpireRestored:   opts.Transition.ExpireRestored,
 	}
 	dfi.SetTierFreeVersionID(fvID)
-	if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, dfi, opts.DeleteMarker); err != nil {
+	if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 
@@ -1578,13 +1634,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		break
 	}
 
-	return ObjectInfo{
-		Bucket:                     bucket,
-		Name:                       object,
-		VersionID:                  opts.VersionID,
-		VersionPurgeStatusInternal: opts.DeleteReplication.VersionPurgeStatusInternal,
-		ReplicationStatusInternal:  opts.DeleteReplication.ReplicationStatusInternal,
-	}, nil
+	return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
 // Send the successful but partial upload/delete, however ignore
@@ -1854,14 +1904,8 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	eventName := event.ObjectTransitionComplete
 
 	storageDisks := er.getDisks()
-	// we now know the number of blocks this object needs for data and parity.
-	// writeQuorum is dataBlocks + 1
-	writeQuorum := fi.Erasure.DataBlocks
-	if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
-		writeQuorum++
-	}
 
-	if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, false); err != nil {
+	if err = er.deleteObjectVersion(ctx, bucket, object, fi, false); err != nil {
 		eventName = event.ObjectTransitionFailed
 	}
 
@@ -1947,7 +1991,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 	}
 
-	uploadID, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
+	res, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
 	if err != nil {
 		return setRestoreHeaderFn(oi, err)
 	}
@@ -1967,7 +2011,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 		if err != nil {
 			return setRestoreHeaderFn(oi, err)
 		}
-		pInfo, err := er.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{})
+		pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{})
 		if err != nil {
 			return setRestoreHeaderFn(oi, err)
 		}
@@ -1979,7 +2023,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 			ETag:       pInfo.ETag,
 		})
 	}
-	_, err = er.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{
+	_, err = er.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{
 		MTime: oi.ModTime,
 	})
 	return setRestoreHeaderFn(oi, err)

cmd/erasure-object_test.go

@@ -31,7 +31,7 @@ import (
 	"strconv"
 	"testing"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/config/storageclass"
 )
 
@@ -53,23 +53,23 @@ func TestRepeatPutObjectPart(t *testing.T) {
 	defer objLayer.Shutdown(context.Background())
 	defer removeRoots(disks)
 
-	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	uploadID, err := objLayer.NewMultipartUpload(ctx, "bucket1", "mpartObj1", opts)
+	res, err := objLayer.NewMultipartUpload(ctx, "bucket1", "mpartObj1", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	fiveMBBytes := bytes.Repeat([]byte("a"), 5*humanize.MiByte)
 	md5Hex := getMD5Hash(fiveMBBytes)
-	_, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts)
+	_, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", res.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	// PutObjectPart should succeed even if part already exists. ref: https://github.com/minio/minio/issues/1930
-	_, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts)
+	_, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", res.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,7 +101,7 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	}
 	defer xl.Shutdown(context.Background())
 
-	err = xl.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = xl.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,7 +155,7 @@ func TestDeleteObjectsVersioned(t *testing.T) {
 		{bucketName, "dir/obj1"},
 	}
 
-	err = obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{
+	err = obj.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{
 		VersioningEnabled: true,
 	})
 	if err != nil {
@@ -244,7 +244,7 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 		{bucketName, "obj_4"},
 	}
 
-	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, BucketOptions{})
+	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -305,7 +305,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -374,7 +374,7 @@ func TestErasureDeleteObjectDiskNotFoundErasure4(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -434,7 +434,7 @@ func TestErasureDeleteObjectDiskNotFoundErr(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +505,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -614,7 +614,7 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -692,7 +692,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -755,7 +755,7 @@ func TestPutObjectNoQuorumSmall(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -824,7 +824,7 @@ func TestPutObjectSmallInlineData(t *testing.T) {
 	object := "object"
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -892,7 +892,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	ctx, cancel := context.WithCancel(GlobalContext)
 	defer cancel()
 
-	err := obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -1073,10 +1073,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	// Create a backend with 4 disks named disk{1...4}, this name convention
 	// because we will unzip some object data from a sample archive.
 	const numDisks = 4
-	path, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
+	path := t.TempDir()
 
 	var fsDirs []string
 	for i := 1; i <= numDisks; i++ {
@@ -1094,7 +1091,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Create a testbucket
-	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1160,7 +1157,7 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
 
 	for i, testCase := range testCases {
 		// Step 1: create a bucket
-		err = z.MakeBucketWithLocation(ctx, testCase.bucket, BucketOptions{VersioningEnabled: testCase.versioned})
+		err = z.MakeBucketWithLocation(ctx, testCase.bucket, MakeBucketOptions{VersioningEnabled: testCase.versioned})
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create a bucket: %v", i+1, err)
 		}
@@ -1186,7 +1183,7 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
 			return disks
 		}
 		sets.erasureDisksMu.Unlock()
-		_, err = z.PutObject(ctx, testCase.bucket, testCase.object, mustGetPutObjReader(t, bytes.NewReader(testCase.content), int64(len(testCase.content)), "", ""),
+		got, err := z.PutObject(ctx, testCase.bucket, testCase.object, mustGetPutObjReader(t, bytes.NewReader(testCase.content), int64(len(testCase.content)), "", ""),
 			ObjectOptions{Versioned: testCase.versioned})
 		if err != nil {
 			t.Fatalf("Test %d: Failed to upload the final object: %v", i+1, err)
@@ -1196,7 +1193,7 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
 		sets.erasureDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI { return origErasureDisks }
 		sets.erasureDisksMu.Unlock()
-		gr, err := z.GetObjectNInfo(ctx, testCase.bucket, testCase.object, nil, nil, readLock, ObjectOptions{})
+		gr, err := z.GetObjectNInfo(ctx, testCase.bucket, testCase.object, nil, nil, readLock, ObjectOptions{VersionID: got.VersionID})
 		if err != nil {
 			t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
 		}

cmd/erasure-server-pool-decom.go

@@ -457,7 +457,7 @@ func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSet
 	now := UTCNow()
 	if now.Sub(p.Pools[idx].LastUpdate) >= duration {
 		if serverDebugLog {
-			console.Debugf("decommission: persisting poolMeta on disk: threshold:%s, poolMeta:%#v\n", now.Sub(p.Pools[idx].LastUpdate), p.Pools[idx])
+			console.Debugf("decommission: persisting poolMeta on drive: threshold:%s, poolMeta:%#v\n", now.Sub(p.Pools[idx].LastUpdate), p.Pools[idx])
 		}
 		p.Pools[idx].LastUpdate = now
 		if err := p.save(ctx, pools); err != nil {
@@ -587,7 +587,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 	}
 
 	if objInfo.isMultipart() {
-		uploadID, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
+		res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:   objInfo.VersionID,
 			MTime:       objInfo.ModTime,
 			UserDefined: objInfo.UserDefined,
@@ -595,14 +595,14 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		if err != nil {
 			return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err)
 		}
-		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, uploadID, ObjectOptions{})
+		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
 			hr, err := hash.NewReader(gr, part.Size, "", "", part.ActualSize)
 			if err != nil {
 				return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 			}
-			pi, err := z.PutObjectPart(ctx, bucket, objInfo.Name, uploadID,
+			pi, err := z.PutObjectPart(ctx, bucket, objInfo.Name, res.UploadID,
 				part.Number,
 				NewPutObjReader(hr),
 				ObjectOptions{
@@ -615,11 +615,15 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 				return fmt.Errorf("decommissionObject: PutObjectPart() %w", err)
 			}
 			parts[i] = CompletePart{
-				ETag:       pi.ETag,
-				PartNumber: pi.PartNumber,
+				ETag:           pi.ETag,
+				PartNumber:     pi.PartNumber,
+				ChecksumCRC32:  pi.ChecksumCRC32,
+				ChecksumCRC32C: pi.ChecksumCRC32C,
+				ChecksumSHA256: pi.ChecksumSHA256,
+				ChecksumSHA1:   pi.ChecksumSHA1,
 			}
 		}
-		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, uploadID, parts, ObjectOptions{
+		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
 			MTime: objInfo.ModTime,
 		})
 		if err != nil {
@@ -652,6 +656,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 }
 
 // versionsSorter sorts FileInfo slices by version.
+//
 //msgp:ignore versionsSorter
 type versionsSorter []FileInfo
 
@@ -677,7 +682,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		set := set
 		disks := set.getOnlineDisks()
 		if len(disks) == 0 {
-			logger.LogIf(GlobalContext, fmt.Errorf("no online disks found for set with endpoints %s",
+			logger.LogIf(GlobalContext, fmt.Errorf("no online drives found for set with endpoints %s",
 				set.getEndpoints()))
 			continue
 		}
@@ -800,11 +805,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						logger.LogIf(ctx, err)
 						continue
 					}
+					stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
+						stopFn(err)
 						failure = true
 						logger.LogIf(ctx, err)
 						continue
 					}
+					stopFn(nil)
 					failure = false
 					break
 				}
@@ -819,6 +827,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 			// if all versions were decommissioned, then we can delete the object versions.
 			if decommissionedCount == len(fivs.Versions) {
+				stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
 				_, err := set.DeleteObject(ctx,
 					bi.Name,
 					encodeDirObject(entry.name),
@@ -826,6 +835,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						DeletePrefix: true, // use prefix delete to delete all versions at once.
 					},
 				)
+				stopFn(err)
 				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
 					logger.LogIf(ctx, err)
@@ -881,6 +891,47 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 	return nil
 }
 
+//msgp:ignore decomMetrics
+type decomMetrics struct{}
+
+var globalDecommissionMetrics decomMetrics
+
+//msgp:ignore decomMetric
+//go:generate stringer -type=decomMetric -trimprefix=decomMetric $GOFILE
+type decomMetric uint8
+
+const (
+	decomMetricDecommissionBucket decomMetric = iota
+	decomMetricDecommissionObject
+	decomMetricDecommissionRemoveObject
+)
+
+func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo {
+	var errStr string
+	if err != nil {
+		errStr = err.Error()
+	}
+	return madmin.TraceInfo{
+		TraceType: madmin.TraceDecommission,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  fmt.Sprintf("decommission.%s (pool-id=%d)", d.String(), poolIdx),
+		Duration:  duration,
+		Path:      path,
+		Error:     errStr,
+	}
+}
+
+func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err error) {
+	startTime := time.Now()
+	return func(err error) {
+		duration := time.Since(startTime)
+		if globalTrace.NumSubscribers(madmin.TraceDecommission) > 0 {
+			globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err))
+		}
+	}
+}
+
 func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
 	pool := z.serverPools[idx]
 	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
@@ -898,9 +949,13 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		if serverDebugLog {
 			console.Debugln("decommission: currently on bucket", bucket.Name)
 		}
+		stopFn := globalDecommissionMetrics.log(decomMetricDecommissionBucket, idx, bucket.Name)
 		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
+			stopFn(err)
 			return err
 		}
+		stopFn(nil)
+
 		z.poolMetaMutex.Lock()
 		z.poolMeta.BucketDone(idx, bucket)
 		z.poolMeta.save(ctx, z.serverPools)
@@ -1119,7 +1174,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		return errInvalidArgument
 	}
 
-	buckets, err := z.ListBuckets(ctx)
+	buckets, err := z.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}
@@ -1157,7 +1212,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		pathJoin(minioMetaBucket, bucketMetaPrefix),
 	} {
 		var bucketExists BucketExists
-		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
+		if err = z.MakeBucketWithLocation(ctx, metaBucket, MakeBucketOptions{}); err != nil {
 			if !errors.As(err, &bucketExists) {
 				return err
 			}