add detection for GOMAXPROCS < NumCPU

fallback quorum should be "strict" globally if config is not loaded (#14589 )
update console to v0.15.4
2026-07-31 07:45:19 +03:00 · 2022-03-21 19:05:10 -07:00 · 2022-03-20 17:39:06 -07:00 · 2022-03-20 15:35:20 -07:00 · 2022-03-20 14:46:43 -07:00 · 2022-03-19 10:15:12 -07:00
302 changed files with 13722 additions and 8307 deletions
@@ -8,8 +8,10 @@ assignees: ''
 ---

 ## NOTE
-If this case is urgent, please subscribe to [Subnet](https://min.io/pricing) so that our 24/7 support team may help you faster.
+All GitHub issues are addressed on a best-effort basis at MinIO's sole discretion. There are no Service Level Agreements (SLA) or Objectives (SLO). Remember our [Code of Conduct](https://github.com/minio/minio/blob/master/code_of_conduct.md) when engaging with MinIO Engineers and the larger community.

+For urgent issues (e.g. production down, etc.), subscribe to [SUBNET](https://min.io/pricing?jmp=github) for direct to engineering support.
+ 
 <!--- Provide a general summary of the issue in the Title above -->

 ## Expected Behavior
@@ -37,10 +37,7 @@ jobs:
        env:
          CGO_ENABLED: 0
          GO111MODULE: on
-          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
-          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
-          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
-          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_SECRET_KEY: "my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY="
          MINIO_KMS_AUTO_ENCRYPTION: on
        run: |
          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
@@ -18,8 +18,7 @@ jobs:
      - uses: dessant/lock-threads@v3
        with:
          github-token: ${{ github.token }}
-          issue-inactive-days: '730'
+          issue-inactive-days: '365'
          exclude-any-issue-labels: 'do-not-close'
          issue-lock-reason: 'resolved'
-          process-only: 'issues'
          log-output: true
@@ -0,0 +1,25 @@
+name: Markdown Linter
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint all docs
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Lint all docs
+      run: |
+        npm install -g markdownlint-cli
+        markdownlint --fix '**/*.md' --disable MD013 MD040
@@ -1,4 +1,5 @@
 # AGPLv3 Compliance
+
 We have designed MinIO as an Open Source software for the Open Source software community. This requires applications to consider whether their usage of MinIO is in compliance with the GNU AGPLv3 [license](https://github.com/minio/minio/blob/master/LICENSE).

 MinIO cannot make the determination as to whether your application's usage of MinIO is in compliance with the AGPLv3 license requirements. You should instead rely on your own legal counsel or licensing specialists to audit and ensure your application is in compliance with the licenses of MinIO and all other open-source projects with which your application integrates or interacts. We understand that AGPLv3 licensing is complex and nuanced. It is for that reason we strongly encourage using experts in licensing to make any such determinations around compliance instead of relying on apocryphal or anecdotal advice.
@@ -7,15 +7,17 @@
 Start by forking the MinIO GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details:

 ### Setup your MinIO GitHub Repository
+
 Fork [MinIO upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your MinIO fork (you will need it for the `git clone` command below).

 ```sh
-$ git clone https://github.com/minio/minio
-$ go install -v
-$ ls /go/bin/minio
+git clone https://github.com/minio/minio
+go install -v
+ls /go/bin/minio
 ```

 ### Set up git remote as ``upstream``
+
 ```sh
 $ cd minio
 $ git remote add upstream https://github.com/minio/minio
@@ -25,13 +27,15 @@ $ git merge upstream/master
 ```

 ### Create your feature branch
+
 Before making code changes, make sure you create a separate branch for these changes

 ```
-$ git checkout -b my-new-feature
+git checkout -b my-new-feature
 ```

 ### Test MinIO server changes
+
 After your code changes, make sure

 - To add test cases for the new code. If you have questions about how to do it, please ask on our [Slack](https://slack.min.io) channel.
@@ -40,29 +44,38 @@ After your code changes, make sure
 - To run `make test` and `make build` completes.

 ### Commit changes
+
 After verification, commit your changes. This is a [great post](https://chris.beams.io/posts/git-commit/) on how to write useful commit messages

 ```
-$ git commit -am 'Add some feature'
+git commit -am 'Add some feature'
 ```

 ### Push to the branch
+
 Push your locally committed changes to the remote origin (your fork)
+
 ```
-$ git push origin my-new-feature
+git push origin my-new-feature
 ```

 ### Create a Pull Request
+
 Pull requests can be created via GitHub. Refer to [this document](https://help.github.com/articles/creating-a-pull-request/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged.

 ## FAQs
+
 ### How does ``MinIO`` manage dependencies?
+
 ``MinIO`` uses `go mod` to manage its dependencies.
+
 - Run `go get foo/bar` in the source folder to add the dependency to `go.mod` file.

 To remove a dependency
+
 - Edit your code and remove the import reference.
 - Run `go mod tidy` in the source folder to remove dependency from `go.mod` file.

 ### What are the coding guidelines for MinIO?
+
 ``MinIO`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](https://slack.min.io).
@@ -1,5 +1,7 @@
 FROM minio/minio:latest

+ENV PATH=/opt/bin:$PATH
+
 COPY ./minio /opt/bin/minio
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

@@ -1,5 +1,7 @@
 FROM minio/minio:latest

+ENV PATH=/opt/bin:$PATH
+
 COPY ./minio /opt/bin/minio
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5

 ARG RELEASE

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5

 ARG TARGETARCH

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.5

 ARG TARGETARCH

@@ -98,19 +98,20 @@ hotfix: hotfix-vars install ## builds minio binary with hotfix tags
 	@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum

 hotfix-push: hotfix
-	@scp -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
-	@scp -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -q -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -q -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"

 docker-hotfix-push: docker-hotfix
-	@docker push $(TAG)
+	@docker push -q $(TAG) && echo "Published new container $(TAG)"

 docker-hotfix: hotfix-push checks ## builds minio docker container with hotfix tags
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) --build-arg RELEASE=$(VERSION) . -f Dockerfile.hotfix
+	@docker build -q --no-cache -t $(TAG) --build-arg RELEASE=$(VERSION) . -f Dockerfile.hotfix

 docker: build checks ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) . -f Dockerfile
+	@docker build -q --no-cache -t $(TAG) . -f Dockerfile

 install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
@@ -1,4 +1,5 @@
 # MinIO Quickstart Guide
+
 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![license](https://img.shields.io/badge/license-AGPL%20V3-blue)](https://github.com/minio/minio/blob/master/LICENSE)

 [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
@@ -7,7 +8,7 @@ MinIO is a High Performance Object Storage released under GNU Affero General Pub

 This README provides quickstart instructions on running MinIO on bare metal hardware, including container-based installations. For Kubernetes environments, use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).

-# Container Installation
+## Container Installation

 Use the following commands to run a standalone MinIO server as a container.

@@ -16,7 +17,7 @@ require distributed deploying MinIO with Erasure Coding. For extended developmen
 with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
 for more complete documentation.

-## Stable
+### Stable

 Run the following command to run the latest stable image of MinIO as a container using an ephemeral data volume:

@@ -26,22 +27,22 @@ podman run -p 9000:9000 -p 9001:9001 \
 ```

 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded
-object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
+object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the
 root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.

 You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
 [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
-see https://docs.min.io/docs/ and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.

 > NOTE: To deploy MinIO on with persistent storage, you must map local persistent directories from the host OS to the container using the `podman -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the container.

-# macOS
+## macOS

 Use the following commands to run a standalone MinIO server on macOS.

 Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.

-## Homebrew (recommended)
+### Homebrew (recommended)

 Run the following command to install the latest stable MinIO package using [Homebrew](https://brew.sh/). Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.

@@ -57,11 +58,11 @@ brew uninstall minio
 brew install minio/stable/minio
 ```

-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.

-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see https://docs.min.io/docs/ and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.

-## Binary Download
+### Binary Download

 Use the following command to download and run a standalone MinIO server on macOS. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.

@@ -71,11 +72,11 @@ chmod +x minio
 ./minio server /data
 ```

-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.

-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see https://docs.min.io/docs/ and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.

-# GNU/Linux
+## GNU/Linux

 Use the following command to run a standalone MinIO server on Linux hosts running 64-bit Intel/AMD architectures. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.

@@ -91,18 +92,18 @@ The following table lists supported architectures. Replace the `wget` URL with t

 | Architecture                   | URL                                                        |
 | --------                       | ------                                                     |
-| 64-bit Intel/AMD               | https://dl.min.io/server/minio/release/linux-amd64/minio   |
-| 64-bit ARM                     | https://dl.min.io/server/minio/release/linux-arm64/minio   |
-| 64-bit PowerPC LE (ppc64le)    | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
-| IBM Z-Series (S390X)           | https://dl.min.io/server/minio/release/linux-s390x/minio   |
+| 64-bit Intel/AMD               | <https://dl.min.io/server/minio/release/linux-amd64/minio>   |
+| 64-bit ARM                     | <https://dl.min.io/server/minio/release/linux-arm64/minio>   |
+| 64-bit PowerPC LE (ppc64le)    | <https://dl.min.io/server/minio/release/linux-ppc64le/minio> |
+| IBM Z-Series (S390X)           | <https://dl.min.io/server/minio/release/linux-s390x/minio>   |

-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.

-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see https://docs.min.io/docs/ and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.

 > NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.

-# Microsoft Windows
+## Microsoft Windows

 To run MinIO on 64-bit Windows hosts, download the MinIO executable from the following URL:

@@ -116,13 +117,13 @@ Use the following command to run a standalone MinIO server on the Windows host.
 minio.exe server D:\
 ```

-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.

-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see https://docs.min.io/docs/ and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.

 > NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.

-# Install from Source
+## Install from Source

 Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)

@@ -130,17 +131,17 @@ Use the following commands to compile and run a standalone MinIO server from sou
 GO111MODULE=on go install github.com/minio/minio@latest
 ```

-The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. Point a web browser running on the host machine to <http://127.0.0.1:9000> and log in with the root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.

-You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see https://docs.min.io/docs/ and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, see <https://docs.min.io/docs/> and click **MinIO SDKs** in the navigation to view MinIO SDKs for supported languages.

 > NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) for more complete documentation.

 MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.

-# Deployment Recommendations
+## Deployment Recommendations

-## Allow port access for Firewalls
+### Allow port access for Firewalls

 By default MinIO uses the port 9000 to listen for incoming connections. If your platform blocks the port by default, you may need to enable access to the port.

@@ -196,18 +197,21 @@ service iptables restart
 ```

 ## Pre-existing data
+
 When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.

 The above statement is also valid for all gateway backends.

-# Test MinIO Connectivity
+## Test MinIO Connectivity

-## Test using MinIO Console
-MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure your server has started successfully.
+### Test using MinIO Console
+
+MinIO Server comes with an embedded web based object browser. Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.

 > NOTE: MinIO runs console on random port by default if you wish choose a specific port use `--console-address` to pick a specific interface and port.

 ### Things to consider
+
 MinIO redirects browser access requests to the configured server port (i.e. `127.0.0.1:9000`) to the configured Console port. MinIO uses the hostname or IP address specified in the request when building the redirect URL. The URL and port *must* be accessible by the client for the redirection to work.

 For deployments behind a load balancer, proxy, or ingress rule where the MinIO host IP address or port is not public, use the `MINIO_BROWSER_REDIRECT_URL` environment variable to specify the external hostname for the redirect. The LB/Proxy must have rules for directing traffic to the Console port specifically.
@@ -218,36 +222,40 @@ Similarly, if your TLS certificates do not have the IP SAN for the MinIO server

 For example: `export MINIO_SERVER_URL="https://minio.example.net"`

-
 | Dashboard                                                                                   | Creating a bucket                                                                           |
 | -------------                                                                               | -------------                                                                               |
 | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic1.png?raw=true) | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic2.png?raw=true) |

 ## Test using MinIO Client `mc`
+
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.

-# Upgrading MinIO
+## Upgrading MinIO
+
 Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all transactions on MinIO are atomic. So upgrading all the servers simultaneously is the recommended way to upgrade MinIO.

-> NOTE: requires internet access to update directly from https://dl.min.io, optionally you can host any mirrors at https://my-artifactory.example.com/minio/
+> NOTE: requires internet access to update directly from <https://dl.min.io>, optionally you can host any mirrors at <https://my-artifactory.example.com/minio/>

 - For deployments that installed the MinIO server binary by hand, use [`mc admin update`](https://docs.min.io/minio/baremetal/reference/minio-mc-admin/mc-admin-update.html)
-```
+
+```sh
 mc admin update <minio alias, e.g., myminio>
 ```

- For deployments without external internet access (e.g. airgapped environments), download the binary from https://dl.min.io and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and do `mc admin service restart alias/`.
+- For deployments without external internet access (e.g. airgapped environments), download the binary from <https://dl.min.io> and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and do `mc admin service restart alias/`.

 - For RPM/DEB installations, upgrade packages **parallelly** on all servers. Once upgraded, perform `systemctl restart minio` across all nodes in **parallel**. RPM/DEB based installations are usually automated using [`ansible`](https://github.com/minio/ansible-minio).

-## Upgrade Checklist
+### Upgrade Checklist
+
 - Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
 - Read the release notes for the targeted MinIO release *before* performing any installation, there is no forced requirement to upgrade to latest releases every week. If it has a bug fix you are looking for then yes, else avoid actively upgrading a running production system.
- Make sure MinIO process has write access to `/opt/bin` if you plan to use `mc admin update`. This is needed for MinIO to download the latest binary from https://dl.min.io and save it locally for upgrades.
+- Make sure MinIO process has write access to `/opt/bin` if you plan to use `mc admin update`. This is needed for MinIO to download the latest binary from <https://dl.min.io> and save it locally for upgrades.
 - `mc admin update` is not supported in kubernetes/container environments, container environments provide their own mechanisms for container updates.
 - **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**

-# Explore Further
+## Explore Further
+
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
 - [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
@@ -255,10 +263,12 @@ mc admin update <minio alias, e.g., myminio>
 - [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
 - [The MinIO documentation website](https://docs.min.io)

-# Contribute to MinIO Project
+## Contribute to MinIO Project
+
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)

-# License
+## License
+
 - MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
 - MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
 - [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)
@@ -18,9 +18,10 @@ you need access credentials for a successful exploit).

 If you have not received a reply to your email within 48 hours or you have not heard from the security team
 for the past five days please contact the security team directly:
-   - Primary security coordinator: aead@min.io
-   - Secondary coordinator: harsha@min.io
-   - If you receive no response: dev@min.io
+
+- Primary security coordinator: aead@min.io
+- Secondary coordinator: harsha@min.io
+- If you receive no response: dev@min.io

 ### Disclosure Process

@@ -32,7 +33,7 @@ MinIO uses the following disclosure process:
   If the report is rejected the response explains why.
 3. Code is audited to find any potential similar problems.
 4. Fixes are prepared for the latest release.
-5. On the date that the fixes are applied a security advisory will be published on https://blog.min.io.
+5. On the date that the fixes are applied a security advisory will be published on <https://blog.min.io>.
   Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing
   the security issue. By default MinIO will **not** publish this information to protect your privacy.

@@ -1,11 +1,11 @@
-## Vulnerability Management Policy
+# Vulnerability Management Policy

 This document formally describes the process of addressing and managing a
 reported vulnerability that has been found in the MinIO server code base,
 any directly connected ecosystem component or a direct / indirect dependency
 of the code base.

-### Scope
+## Scope

 The vulnerability management policy described in this document covers the
 process of investigating, assessing and resolving a vulnerability report
@@ -14,13 +14,13 @@ opened by a MinIO employee or an external third party.
 Therefore, it lists pre-conditions and actions that should be performed to
 resolve and fix a reported vulnerability.

-### Vulnerability Management Process
+## Vulnerability Management Process

 The vulnerability management process requires that the vulnerability report
 contains the following information:

- - The project / component that contains the reported vulnerability.
- - A description of the vulnerability. In particular, the type of the
+- The project / component that contains the reported vulnerability.
+- A description of the vulnerability. In particular, the type of the
   reported vulnerability and how it might be exploited. Alternatively,
   a well-established vulnerability identifier, e.g. CVE number, can be
   used instead.
@@ -28,12 +28,11 @@ contains the following information:
 Based on the description mentioned above, a MinIO engineer or security team
 member investigates:

- - Whether the reported vulnerability exists.
- - The conditions that are required such that the vulnerability can be exploited.
- - The steps required to fix the vulnerability.
+- Whether the reported vulnerability exists.
+- The conditions that are required such that the vulnerability can be exploited.
+- The steps required to fix the vulnerability.

 In general, if the vulnerability exists in one of the MinIO code bases
 itself - not in a code dependency - then MinIO will, if possible, fix
 the vulnerability or implement reasonable countermeasures such that the
 vulnerability cannot be exploited anymore.
-
@@ -1,18 +0,0 @@
-**/*.swp
-cover.out
-*~
-minio
-!*/
-site/
-**/*.test
-**/*.sublime-workspace
-/.idea/
-/Minio.iml
-**/access.log
-build
-vendor/**/*.js
-vendor/**/*.json
-.DS_Store
-*.syso
-coverage.txt
-node_modules
@@ -30,6 +30,7 @@ function start_minio_16drive() {
    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
+    export MINIO_CI_CD=1

    MC_BUILD_DIR="mc-$RANDOM"
    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
@@ -22,6 +22,8 @@ export GO111MODULE=on
 export GOGC=25
 export ENABLE_ADMIN=1

+export MINIO_CI_CD=1
+
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )

@@ -17,6 +17,7 @@ function start_minio_3_node() {
    export MINIO_ROOT_USER=minio
    export MINIO_ROOT_PASSWORD=minio123
    export MINIO_ERASURE_SET_DRIVE_COUNT=6
+    export MINIO_CI_CD=1

    start_port=$2
    args=""
@@ -71,7 +71,12 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}

-	if err = globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, data); err != nil {
+	if quotaConfig.Type == "fifo" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -115,7 +120,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}

-	config, err := globalBucketMetadataSys.GetQuotaConfig(bucket)
+	config, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -246,7 +251,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -340,7 +345,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -182,6 +182,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusConflict,
 			}
+		case errors.Is(err, errTierBackendNotEmpty):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierBackendNotEmpty",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errTierInsufficientCreds):
 			apiErr = APIError{
 				Code:           "XMinioAdminTierInsufficientCreds",
@@ -64,6 +64,12 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}

+	subSys, _, _, err := config.GetSubSys(string(kvBytes))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	cfg, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -74,21 +80,26 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+	if err = validateConfig(cfg, subSys); err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		return
+	}

 	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

-	dynamic := config.SubSystemsDynamic.Contains(string(kvBytes))
+	dynamic := config.SubSystemsDynamic.Contains(subSys)
 	if dynamic {
-		applyDynamic(ctx, objectAPI, cfg, r, w)
+		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
 	}
 }

-func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config, r *http.Request, w http.ResponseWriter) {
+func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config, subSys string,
+	r *http.Request, w http.ResponseWriter) {
 	// Apply dynamic values.
-	if err := applyDynamicConfig(GlobalContext, objectAPI, cfg); err != nil {
+	if err := applyDynamicConfigForSubSys(GlobalContext, objectAPI, cfg, subSys); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -134,7 +145,13 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}

-	if err = validateConfig(cfg); err != nil {
+	subSys, _, _, err := config.GetSubSys(string(kvBytes))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = validateConfig(cfg, subSys); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -152,7 +169,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}

 	if dynamic {
-		applyDynamic(ctx, objectAPI, cfg, r, w)
+		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
 	}
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -256,7 +273,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}

-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, ""); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -367,7 +384,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}

-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, ""); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -253,18 +253,6 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 	}
 }

-// SiteReplicationDisable - PUT /minio/admin/v3/site-replication/disable
-func (a adminAPIHandlers) SiteReplicationDisable(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationDisable")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationDisableAction)
-	if objectAPI == nil {
-		return
-	}
-}
-
 // SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
 func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SiteReplicationInfo")
@@ -313,7 +301,6 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 			Code:  ErrSiteReplicationInvalidRequest,
 		}
 	}
-
 	if encryptionKey != "" {
 		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
 		if err != nil {
@@ -324,7 +311,6 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 			}
 		}
 	}
-
 	return json.Unmarshal(data, v)
 }

@@ -338,8 +324,16 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
 	if objectAPI == nil {
 		return
 	}
-
-	info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI)
+	opts := getSRStatusOptions(r)
+	// default options to all if status options are unset for backward compatibility
+	var dfltOpts madmin.SRStatusOptions
+	if opts == dfltOpts {
+		opts.Buckets = true
+		opts.Users = true
+		opts.Policies = true
+		opts.Groups = true
+	}
+	info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -362,7 +356,8 @@ func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http
 		return
 	}

-	info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, objectAPI)
+	opts := getSRStatusOptions(r)
+	info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, objectAPI, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -428,3 +423,70 @@ func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
+	q := r.Form
+	opts.Buckets = q.Get("buckets") == "true"
+	opts.Policies = q.Get("policies") == "true"
+	opts.Groups = q.Get("groups") == "true"
+	opts.Users = q.Get("users") == "true"
+	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
+	opts.EntityValue = q.Get("entityvalue")
+	return
+}
+
+// SiteReplicationRemove - PUT /minio/admin/v3/site-replication/remove
+func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationRemove")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
+	if objectAPI == nil {
+		return
+	}
+	var rreq madmin.SRRemoveReq
+	err := parseJSONBody(ctx, r.Body, &rreq, "")
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	status, err := globalSiteReplicationSys.RemovePeerCluster(ctx, objectAPI, rreq)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(status)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, body)
+}
+
+// SRPeerRemove - PUT /minio/admin/v3/site-replication/peer/remove
+//
+// used internally to tell current cluster to update endpoint for peer
+func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerRemove")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var req madmin.SRRemoveReq
+	if err := parseJSONBody(ctx, r.Body, &req, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err := globalSiteReplicationSys.InternalRemoveReq(ctx, objectAPI, req); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
@@ -1170,7 +1170,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			}

 			lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)
-			quota, _ := globalBucketQuotaSys.Get(bucket.Name)
+			quota, _ := globalBucketQuotaSys.Get(ctx, bucket.Name)
 			rcfg, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name)
 			tcfg, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)

@@ -21,8 +21,8 @@ import (
 	"context"
 	crand "crypto/rand"
 	"crypto/subtle"
-	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -36,6 +36,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"github.com/dustin/go-humanize"
@@ -283,6 +284,7 @@ type ServerHTTPAPIStats struct {
 // including their average execution time.
 type ServerHTTPStats struct {
 	S3RequestsInQueue      int32              `json:"s3RequestsInQueue"`
+	S3RequestsIncoming     uint64             `json:"s3RequestsIncoming"`
 	CurrentS3Requests      ServerHTTPAPIStats `json:"currentS3Requests"`
 	TotalS3Requests        ServerHTTPAPIStats `json:"totalS3Requests"`
 	TotalS3Errors          ServerHTTPAPIStats `json:"totalS3Errors"`
@@ -933,12 +935,61 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 }

-// SpeedtestHandler - reports maximum speed of a cluster by performing PUT and
+// NetperfHandler - perform mesh style network throughput test
+func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "NetperfHandler")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if !globalIsDistErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf")
+	lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	defer nsLock.Unlock(lkctx.Cancel)
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		duration = globalNetPerfMinDuration
+	}
+
+	if duration < globalNetPerfMinDuration {
+		// We need sample size of minimum 10 secs.
+		duration = globalNetPerfMinDuration
+	}
+
+	duration = duration.Round(time.Second)
+
+	results := globalNotificationSys.Netperf(ctx, duration)
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(madmin.NetperfResult{NodeResults: results}); err != nil {
+		return
+	}
+}
+
+// SpeedtestHandler - Deprecated. See ObjectSpeedtestHandler
+func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
+	a.ObjectSpeedtestHandler(w, r)
+}
+
+// ObjectSpeedtestHandler - reports maximum speed of a cluster by performing PUT and
 // GET operations on the server, supports auto tuning by default by automatically
 // increasing concurrency and stopping when we have reached the limits on the
 // system.
-func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SpeedtestHandler")
+func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ObjectSpeedtestHandler")

 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))

@@ -977,9 +1028,33 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		duration = time.Second * 10
 	}

+	// ignores any errors here.
+	storageInfo, _ := objectAPI.StorageInfo(ctx)
+	capacityNeeded := uint64(concurrent * size)
+	capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))
+
+	if capacity < capacityNeeded {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
+			Code: "XMinioSpeedtestInsufficientCapacity",
+			Message: fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s",
+				humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)),
+			StatusCode: http.StatusInsufficientStorage,
+		}), r.URL)
+		return
+	}
+
+	// Verify if we can employ autotune without running out of capacity,
+	// if we do run out of capacity, make sure to turn-off autotuning
+	// in such situations.
+	newConcurrent := concurrent + (concurrent+1)/2
+	autoTunedCapacityNeeded := uint64(newConcurrent * size)
+	if autotune && capacity < autoTunedCapacityNeeded {
+		// Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity.
+		autotune = false
+	}
+
 	deleteBucket := func() {
-		loc := pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix)
-		objectAPI.DeleteBucket(context.Background(), loc, DeleteBucketOptions{
+		objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaBucket, "speedtest"), DeleteBucketOptions{
 			Force:      true,
 			NoRecreate: true,
 		})
@@ -1019,6 +1094,97 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	}
 }

+// NetSpeedtestHandler - reports maximum network throughput
+func (a adminAPIHandlers) NetSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "NetSpeedtestHandler")
+
+	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+}
+
+// DriveSpeedtestHandler - reports throughput of drives available in the cluster
+func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DriveSpeedtestHandler")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// Freeze all incoming S3 API calls before running speedtest.
+	globalNotificationSys.ServiceFreeze(ctx, true)
+
+	// unfreeze all incoming S3 API calls after speedtest.
+	defer globalNotificationSys.ServiceFreeze(ctx, false)
+
+	serial := r.Form.Get("serial") == "true"
+	blockSizeStr := r.Form.Get("blocksize")
+	fileSizeStr := r.Form.Get("filesize")
+
+	blockSize, err := strconv.ParseUint(blockSizeStr, 10, 64)
+	if err != nil {
+		blockSize = 4 * humanize.MiByte // default value
+	}
+
+	fileSize, err := strconv.ParseUint(fileSizeStr, 10, 64)
+	if err != nil {
+		fileSize = 1 * humanize.GiByte // default value
+	}
+
+	opts := madmin.DriveSpeedTestOpts{
+		Serial:    serial,
+		BlockSize: blockSize,
+		FileSize:  fileSize,
+	}
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	enc := json.NewEncoder(w)
+	ch := globalNotificationSys.DriveSpeedTest(ctx, opts)
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// local driveSpeedTest
+	go func() {
+		defer wg.Done()
+		enc.Encode(driveSpeedTest(ctx, opts))
+		if wf, ok := w.(http.Flusher); ok {
+			wf.Flush()
+		}
+	}()
+
+	for {
+		select {
+		case <-ctx.Done():
+			goto endloop
+		case <-keepAliveTicker.C:
+			// Write a blank entry to prevent client from disconnecting
+			if err := enc.Encode(madmin.DriveSpeedTestResult{}); err != nil {
+				goto endloop
+			}
+			w.(http.Flusher).Flush()
+		case result, ok := <-ch:
+			if !ok {
+				goto endloop
+			}
+			if err := enc.Encode(result); err != nil {
+				goto endloop
+			}
+			w.(http.Flusher).Flush()
+		}
+	}
+endloop:
+	wg.Wait()
+}
+
 // Admin API errors
 const (
 	AdminUpdateUnexpectedFailure = "XMinioAdminUpdateUnexpectedFailure"
@@ -2125,7 +2291,7 @@ func fetchKMSStatus() madmin.KMS {
 func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
 	var auditloggerInfo []madmin.Audit
-	for _, target := range logger.Targets() {
+	for _, target := range logger.SystemTargets() {
 		if target.Endpoint() != "" {
 			tgt := target.String()
 			err := checkConnection(target.Endpoint(), 15*time.Second)
@@ -2165,19 +2331,9 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
 	defer cancel()

-	client := &http.Client{Transport: &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewCustomDialContext(timeout),
-		ResponseHeaderTimeout: 5 * time.Second,
-		TLSHandshakeTimeout:   5 * time.Second,
-		ExpectContinueTimeout: 5 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
-		// Go net/http automatically unzip if content-type is
-		// gzip disable this feature, as we are always interested
-		// in raw stream.
-		DisableCompression: true,
-	}}
-	defer client.CloseIdleConnections()
+	client := &http.Client{
+		Transport: globalProxyTransport,
+	}

 	req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
 	if err != nil {
@@ -2305,7 +2461,9 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		}
 		return nil
 	})
-	logger.LogIf(ctx, err)
+	if !errors.Is(err, errFileNotFound) {
+		logger.LogIf(ctx, err)
+	}
 }

 func createHostAnonymizerForFSMode() map[string]string {
@@ -74,11 +74,11 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro

 	globalEndpoints = mustGetPoolEndpoints(erasureDirs...)

-	newAllSubsystems()
+	initAllSubsystems()

 	initConfigSubsystem(ctx, objLayer)

-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)

 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -700,8 +700,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
+	} else {
+		task.opts.ScanMode = globalHealConfig.ScanMode()
 	}
-	task.opts.ScanMode = globalHealConfig.ScanMode()

 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++
@@ -191,13 +191,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
-
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveTierHandler)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.VerifyTierHandler)))
 			// Tier stats
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))

 			// Cluster Replication APIs
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationRemove)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
@@ -209,6 +210,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
 		}

 		if globalIsDistErasure {
@@ -220,6 +222,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		}

 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))

 		// HTTP Trace
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(http.HandlerFunc(adminAPI.TraceHandler)))
@@ -50,7 +50,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := isServerResolvable(endpoint, 2*time.Second); err == nil {
+				if err := isServerResolvable(endpoint, 5*time.Second); err == nil {
 					network[nodeName] = string(madmin.ItemOnline)
 				} else {
 					network[nodeName] = string(madmin.ItemOffline)
@@ -212,6 +212,7 @@ const (
 	ErrInvalidSSECustomerParameters
 	ErrIncompatibleEncryptionMethod
 	ErrKMSNotConfigured
+	ErrKMSKeyNotFoundException

 	ErrNoAccessKey
 	ErrInvalidToken
@@ -1127,6 +1128,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Server side encryption specified but KMS is not configured",
 		HTTPStatusCode: http.StatusNotImplemented,
 	},
+	ErrKMSKeyNotFoundException: {
+		Code:           "KMS.NotFoundException",
+		Description:    "Invalid keyId",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoAccessKey: {
 		Code:           "AccessDenied",
 		Description:    "No AWSAccessKey was presented",
@@ -1912,6 +1918,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrIncompatibleEncryptionMethod
 	case errKMSNotConfigured:
 		apiErr = ErrKMSNotConfigured
+	case errKMSKeyNotFound:
+		apiErr = ErrKMSKeyNotFoundException
+
 	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
@@ -126,6 +126,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp

 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
+		// Empty values for object lock and retention can be skipped.
+		if v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate) {
+			continue
+		}
+
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			// values to client.
@@ -287,7 +287,7 @@ func registerAPIRouter(router *mux.Router) {
 			collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
 		// GetObject - note gzip compression is *not* added due to Range requests.
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobject", maxClients(httpTraceHdrs(api.GetObjectHandler))))
+			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
 		// CopyObject
 		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
 			collectAPIStats("copyobject", maxClients(gz(httpTraceAll(api.CopyObjectHandler)))))
@@ -343,6 +343,9 @@ func registerAPIRouter(router *mux.Router) {
 		// ListenNotification
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listennotification", maxClients(gz(httpTraceAll(api.ListenNotificationHandler))))).Queries("events", "{events:.*}")
+		// ResetBucketReplicationStatus - MinIO extension API
+		router.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "")

 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
@@ -417,9 +420,10 @@ func registerAPIRouter(router *mux.Router) {
 		// PutBucketNotification
 		router.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putbucketnotification", maxClients(gz(httpTraceAll(api.PutBucketNotificationHandler))))).Queries("notification", "")
-		// ResetBucketReplicationState - MinIO extension API
+		// ResetBucketReplicationStart - MinIO extension API
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("resetbucketreplicationstate", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStateHandler))))).Queries("replication-reset", "")
+			collectAPIStats("resetbucketreplicationstart", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStartHandler))))).Queries("replication-reset", "")
+
 		// PutBucket
 		router.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putbucket", maxClients(gz(httpTraceAll(api.PutBucketHandler)))))
@@ -197,13 +197,7 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 	return claims
 }

-// Fetch claims in the security token returned by the client.
-func getClaimsFromToken(token string) (map[string]interface{}, error) {
-	if token == "" {
-		claims := xjwt.NewMapClaims()
-		return claims.Map(), nil
-	}
-
+func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
 	// JWT token for x-amz-security-token is signed with admin
 	// secret key, temporary credentials become invalid if
 	// server admin credentials change. This is done to ensure
@@ -212,9 +206,15 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	// hijacking the policies. We need to make sure that this is
 	// based an admin credential such that token cannot be decoded
 	// on the client side and is treated like an opaque value.
-	claims, err := auth.ExtractClaims(token, globalActiveCred.SecretKey)
+	claims, err := auth.ExtractClaims(token, secret)
 	if err != nil {
-		return nil, errAuthentication
+		if subtle.ConstantTimeCompare([]byte(secret), []byte(globalActiveCred.SecretKey)) == 1 {
+			return nil, errAuthentication
+		}
+		claims, err = auth.ExtractClaims(token, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, errAuthentication
+		}
 	}

 	// If OPA is set, return without any further checks.
@@ -241,23 +241,50 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	return claims.Map(), nil
 }

+// Fetch claims in the security token returned by the client.
+func getClaimsFromToken(token string) (map[string]interface{}, error) {
+	return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+}
+
 // Fetch claims in the security token returned by the client and validate the token.
 func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
 	token := getSessionToken(r)
 	if token != "" && cred.AccessKey == "" {
+		// x-amz-security-token is not allowed for anonymous access.
 		return nil, ErrNoAccessKey
 	}
-	if cred.IsServiceAccount() && token == "" {
-		token = cred.SessionToken
-	}
-	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+
+	if token == "" && cred.IsTemp() {
+		// Temporary credentials should always have x-amz-security-token
 		return nil, ErrInvalidToken
 	}
-	claims, err := getClaimsFromToken(token)
-	if err != nil {
-		return nil, toAPIErrorCode(r.Context(), err)
+
+	if token != "" && !cred.IsTemp() {
+		// x-amz-security-token should not present for static credentials.
+		return nil, ErrInvalidToken
 	}
-	return claims, ErrNone
+
+	if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+		// validate token for temporary credentials only.
+		return nil, ErrInvalidToken
+	}
+
+	secret := globalActiveCred.SecretKey
+	if cred.IsServiceAccount() {
+		token = cred.SessionToken
+		secret = cred.SecretKey
+	}
+
+	if token != "" {
+		claims, err := getClaimsFromTokenWithSecret(token, secret)
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		return claims, ErrNone
+	}
+
+	claims := xjwt.NewMapClaims()
+	return claims.Map(), ErrNone
 }

 // Check request auth type verifies the incoming http request
@@ -362,14 +362,14 @@ func TestIsReqAuthenticated(t *testing.T) {
 		t.Fatalf("unable initialize config file, %s", err)
 	}

-	newAllSubsystems()
+	initAllSubsystems()

 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

 	initConfigSubsystem(ctx, objLayer)

-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)

 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -455,11 +455,11 @@ func TestValidateAdminSignature(t *testing.T) {
 		t.Fatalf("unable initialize config file, %s", err)
 	}

-	newAllSubsystems()
+	initAllSubsystems()

 	initConfigSubsystem(ctx, objLayer)

-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)

 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {
@@ -263,7 +263,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)

 	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
-		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content - use 'mc admin heal alias/ --verbose' to check the status",
 			drivesToHeal, defaultMonitorNewDiskInterval))

 		// Heal any disk format and metadata early, if possible.
@@ -277,14 +277,6 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}

-	if err := bgSeq.healDiskMeta(objAPI); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(bgSeq.ctx, err)
-		}
-	}
-
 	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }

@@ -333,7 +325,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 				// Ensure that reformatting disks is finished
 				bgSeq.queueHealTask(healSource{bucket: nopHeal}, madmin.HealItemMetadata)

-				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal - 'mc admin heal alias/ --verbose' to check the status.",
 					len(healDisks)))

 				erasureSetInPoolDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
@@ -404,13 +396,15 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 					go func(setIndex int, disks []StorageAPI) {
 						defer wg.Done()
 						for _, disk := range disks {
-							logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
+							if serverDebugLog {
+								logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
+							}

 							// So someone changed the drives underneath, healing tracker missing.
 							tracker, err := loadHealingTracker(ctx, disk)
 							if err != nil {
-								logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool",
-									disk, humanize.Ordinal(i+1))
+								logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
+									disk, humanize.Ordinal(i+1), err))
 								tracker = newHealingTracker(disk)
 							}

@@ -438,12 +432,14 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 								continue
 							}

-							logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
-								humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
-							logger.Info("Summary:\n")
-							tracker.printTo(os.Stdout)
+							if serverDebugLog {
+								logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
+									humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
+								logger.Info("Summary:\n")
+								tracker.printTo(os.Stdout)
+								logger.Info("\n")
+							}
 							logger.LogIf(ctx, tracker.delete(ctx))
-							logger.Info("\n")

 							// Only upon success pop the healed disk.
 							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())
@@ -205,11 +205,11 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
-				if isNetworkError(err) {
-					offlineEndpoints = append(offlineEndpoints, clnt.String())
-					continue
+				if !isNetworkError(err) {
+					logger.LogIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
 				}
-				return fmt.Errorf("%s as has incorrect configuration: %w", clnt.String(), err)
+				offlineEndpoints = append(offlineEndpoints, clnt.String())
+				continue
 			}
 			onlineServers++
 		}
@@ -20,12 +20,15 @@ package cmd
 import (
 	"encoding/base64"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"

 	"github.com/gorilla/mux"
+	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -84,6 +87,19 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
+	kmsKey := encConfig.KeyID()
+	if kmsKey != "" {
+		kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
+		_, err := GlobalKMS.GenerateKey(kmsKey, kmsContext)
+		if err != nil {
+			if errors.Is(err, kes.ErrKeyNotFound) {
+				writeErrorResponse(ctx, w, toAPIError(ctx, errKMSKeyNotFound), r.URL)
+				return
+			}
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}

 	configData, err := xml.Marshal(encConfig)
 	if err != nil {
@@ -92,7 +108,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}

 	// Store the bucket encryption configuration in the object layer
-	if err = globalBucketMetadataSys.Update(bucket, bucketSSEConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -186,10 +202,19 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}

 	// Delete bucket encryption config from object layer
-	if err = globalBucketMetadataSys.Update(bucket, bucketSSEConfig, nil); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	// Call site replication hook.
+	//
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:      madmin.SRBucketMetaTypeSSEConfig,
+		Bucket:    bucket,
+		SSEConfig: nil,
+	}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
 	writeSuccessNoContent(w)
 }
@@ -951,6 +951,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// explicit permissions for the user.
 	if !globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
 		Action:          iampolicy.PutObjectAction,
 		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
 		BucketName:      bucket,
@@ -1269,6 +1270,17 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 				return
 			}
+			rcfg, err := getReplicationConfig(ctx, bucket)
+			switch {
+			case err != nil:
+				if _, ok := err.(BucketReplicationConfigNotFound); !ok {
+					writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+					return
+				}
+			case rcfg.HasActiveRules("", true):
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+				return
+			}
 		}
 	}

@@ -1300,7 +1312,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}

 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
-
+	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
 	// Call site replication hook.
 	if err := globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1367,7 +1379,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}

-	if err = globalBucketMetadataSys.Update(bucket, objectLockConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1472,7 +1484,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}

-	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1554,7 +1566,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}

-	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, nil); err != nil {
+	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1628,7 +1640,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1707,7 +1719,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
 		return
 	}
-	if err := globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, nil); err != nil {
+	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1759,12 +1771,13 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	}
 }

-// ResetBucketReplicationStateHandler - starts a replication reset for all objects in a bucket which
+// ResetBucketReplicationStartHandler - starts a replication reset for all objects in a bucket which
 // qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is
 // enabled for the qualifying rule. This API is a MinIO only extension provided for situations where
-// remote target is entirely lost,and previously replicated objects need to be re-synced.
-func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ResetBucketReplicationState")
+// remote target is entirely lost,and previously replicated objects need to be re-synced. If resync is
+// already in progress it returns an error
+func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ResetBucketReplicationStart")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))

 	vars := mux.Vars(r)
@@ -1788,6 +1801,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 			}), r.URL)
 		}
 	}
+	resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))

 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1824,7 +1838,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 	if len(tgtArns) == 0 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
 			Bucket: bucket,
-			Err:    fmt.Errorf("Remote target ARN %s missing/not eligible for replication resync", arn),
+			Err:    fmt.Errorf("Remote target ARN %s missing or ineligible for replication resync", arn),
 		}), r.URL)
 		return
 	}
@@ -1848,22 +1862,90 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		}
+	}
+	if err := startReplicationResync(ctx, bucket, arn, resetID, resetBeforeDate, objectAPI); err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    err,
+		}), r.URL)
 		return
 	}
-	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	tgtBytes, err := json.Marshal(&targets)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
-		return
-	}
-	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+
+	data, err := json.Marshal(rinfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}
+
+// ResetBucketReplicationStatusHandler - returns the status of replication reset.
+// This API is a MinIO only extension
+func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ResetBucketReplicationStatus")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	arn := r.URL.Query().Get("arn")
+	var err error
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	globalReplicationPool.resyncState.RLock()
+	brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
+	if !ok {
+		brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
+		if err != nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+				Bucket: bucket,
+				Err:    fmt.Errorf("No replication resync status available for %s", arn),
+			}), r.URL)
+		}
+		return
+	}
+
+	var rinfo ResyncTargetsInfo
+	for tarn, st := range brs.TargetsMap {
+		if arn != "" && tarn != arn {
+			continue
+		}
+		rinfo.Targets = append(rinfo.Targets, ResyncTarget{
+			Arn:             tarn,
+			ResetID:         st.ResyncID,
+			StartTime:       st.StartTime,
+			EndTime:         st.EndTime,
+			ResyncStatus:    st.ResyncStatus.String(),
+			ReplicatedSize:  st.ReplicatedSize,
+			ReplicatedCount: st.ReplicatedCount,
+			FailedSize:      st.FailedSize,
+			FailedCount:     st.FailedCount,
+			Bucket:          st.Bucket,
+			Object:          st.Object,
+		})
+	}
+	globalReplicationPool.resyncState.RUnlock()
 	data, err := json.Marshal(rinfo)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -91,7 +91,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}

-	if err = globalBucketMetadataSys.Update(bucket, bucketLifecycleConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -168,7 +168,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 		return
 	}

-	if err := globalBucketMetadataSys.Update(bucket, bucketLifecycleConfig, nil); err != nil {
+	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -221,17 +221,18 @@ func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
 			var err error
 			if tier, err = transitionObject(ctx, objectAPI, oi); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
+			} else {
+				ts := tierStats{
+					TotalSize:   uint64(oi.Size),
+					NumVersions: 1,
+				}
+				if oi.IsLatest {
+					ts.NumObjects = 1
+				}
+				t.addLastDayStats(tier, ts)
 			}
 			atomic.AddInt32(&t.activeTasks, -1)

-			ts := tierStats{
-				TotalSize:   uint64(oi.Size),
-				NumVersions: 1,
-			}
-			if oi.IsLatest {
-				ts.NumObjects = 1
-			}
-			t.addLastDayStats(tier, ts)
 		}
 	}
 }
@@ -693,9 +694,9 @@ func completedRestoreObj(expiry time.Time) restoreObjStatus {
 // String returns x-amz-restore compatible representation of r.
 func (r restoreObjStatus) String() string {
 	if r.Ongoing() {
-		return "ongoing-request=true"
+		return `ongoing-request="true"`
 	}
-	return fmt.Sprintf("ongoing-request=false, expiry-date=%s", r.expiry.Format(http.TimeFormat))
+	return fmt.Sprintf(`ongoing-request="false", expiry-date="%s"`, r.expiry.Format(http.TimeFormat))
 }

 // Expiry returns expiry of restored object and true if restore-object has completed.
@@ -739,12 +740,11 @@ func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) {
 	}

 	switch progressTokens[1] {
-	case "true":
+	case "true", `"true"`: // true without double quotes is deprecated in Feb 2022
 		if len(tokens) == 1 {
 			return ongoingRestoreObj(), nil
 		}
-
-	case "false":
+	case "false", `"false"`: // false without double quotes is deprecated in Feb 2022
 		if len(tokens) != 2 {
 			return restoreObjStatus{}, errRestoreHDRMalformed
 		}
@@ -755,8 +755,7 @@ func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) {
 		if strings.TrimSpace(expiryTokens[0]) != "expiry-date" {
 			return restoreObjStatus{}, errRestoreHDRMalformed
 		}
-
-		expiry, err := time.Parse(http.TimeFormat, expiryTokens[1])
+		expiry, err := time.Parse(http.TimeFormat, strings.Trim(expiryTokens[1], `"`))
 		if err != nil {
 			return restoreObjStatus{}, errRestoreHDRMalformed
 		}
@@ -36,7 +36,7 @@ func TestParseRestoreObjStatus(t *testing.T) {
 	}{
 		{
 			// valid: represents a restored object, 'pending' expiry.
-			restoreHdr: "ongoing-request=false, expiry-date=Fri, 21 Dec 2012 00:00:00 GMT",
+			restoreHdr: `ongoing-request="false", expiry-date="Fri, 21 Dec 2012 00:00:00 GMT"`,
 			expectedStatus: restoreObjStatus{
 				ongoing: false,
 				expiry:  time.Date(2012, 12, 21, 0, 0, 0, 0, time.UTC),
@@ -45,7 +45,7 @@ func TestParseRestoreObjStatus(t *testing.T) {
 		},
 		{
 			// valid: represents an ongoing restore object request.
-			restoreHdr: "ongoing-request=true",
+			restoreHdr: `ongoing-request="true"`,
 			expectedStatus: restoreObjStatus{
 				ongoing: true,
 			},
@@ -53,13 +53,13 @@ func TestParseRestoreObjStatus(t *testing.T) {
 		},
 		{
 			// invalid; ongoing restore object request can't have expiry set on it.
-			restoreHdr:     "ongoing-request=true, expiry-date=Fri, 21 Dec 2012 00:00:00 GMT",
+			restoreHdr:     `ongoing-request="true", expiry-date="Fri, 21 Dec 2012 00:00:00 GMT"`,
 			expectedStatus: restoreObjStatus{},
 			expectedErr:    errRestoreHDRMalformed,
 		},
 		{
 			// invalid; completed restore object request must have expiry set on it.
-			restoreHdr:     "ongoing-request=false",
+			restoreHdr:     `ongoing-request="false"`,
 			expectedStatus: restoreObjStatus{},
 			expectedErr:    errRestoreHDRMalformed,
 		},
@@ -74,7 +74,7 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {

 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
-func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
+func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) error {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
@@ -83,13 +83,13 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 	if globalIsGateway && globalGatewayName != NASBackendGateway {
 		if configFile == bucketPolicyConfig {
 			if configData == nil {
-				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
+				return objAPI.DeleteBucketPolicy(ctx, bucket)
 			}
 			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
 			if err != nil {
 				return err
 			}
-			return objAPI.SetBucketPolicy(GlobalContext, bucket, config)
+			return objAPI.SetBucketPolicy(ctx, bucket, config)
 		}
 		return NotImplemented{}
 	}
@@ -98,7 +98,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		return errInvalidArgument
 	}

-	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
 		if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
 			// Only single drive mode needs this fallback.
@@ -148,12 +148,12 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}

-	if err := meta.Save(GlobalContext, objAPI); err != nil {
+	if err := meta.Save(ctx, objAPI); err != nil {
 		return err
 	}

 	sys.Set(bucket, meta)
-	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
+	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket) // Do not use caller context here

 	return nil
 }
@@ -187,7 +187,7 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 // GetVersioningConfig returns configured versioning config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, error) {
-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -197,7 +197,7 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {
-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketTaggingNotFound{Bucket: bucket}
@@ -213,7 +213,7 @@ func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error
 // GetObjectLockConfig returns configured object lock config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) {
-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
@@ -245,7 +245,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 		return meta.lifecycleConfig, nil
 	}

-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketLifecycleNotFound{Bucket: bucket}
@@ -274,7 +274,7 @@ func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Confi
 		return meta.notificationConfig, nil
 	}

-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -284,7 +284,7 @@ func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Confi
 // GetSSEConfig returns configured SSE config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) {
-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketSSEConfigNotFound{Bucket: bucket}
@@ -308,7 +308,7 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, er
 		return objAPI.GetBucketPolicy(GlobalContext, bucket)
 	}

-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketPolicyNotFound{Bucket: bucket}
@@ -323,8 +323,8 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, er

 // GetQuotaConfig returns configured bucket quota
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota, error) {
-	meta, err := sys.GetConfig(bucket)
+func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, error) {
+	meta, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -334,7 +334,7 @@ func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota
 // GetReplicationConfig returns configured bucket replication config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, error) {
-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketReplicationConfigNotFound{Bucket: bucket}
@@ -351,7 +351,7 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 // GetBucketTargetsConfig returns configured bucket targets for this bucket
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
-	meta, err := sys.GetConfig(bucket)
+	meta, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -377,7 +377,7 @@ func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin

 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
+func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (BucketMetadata, error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return newBucketMetadata(bucket), errServerNotInitialized
@@ -397,7 +397,7 @@ func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
 	if ok {
 		return meta, nil
 	}
-	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
 		return meta, err
 	}
@@ -434,6 +434,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
 				// Ensure heal opts for bucket metadata be deep healed all the time.
 				ScanMode: madmin.HealDeepScan,
+				Recreate: true,
 			})
 			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
 			if err != nil {
@@ -375,6 +375,7 @@ func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string)
 	metadataFiles := []string{
 		dataUsageCacheName,
 		bucketMetadataFile,
+		path.Join(replicationDir, resyncFileName),
 	}
 	for _, metaFile := range metadataFiles {
 		configFile := path.Join(bucketMetaPrefix, bucket, metaFile)
@@ -160,7 +160,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}

-	if err = globalBucketMetadataSys.Update(bucketName, bucketNotificationConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -103,7 +103,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}

-	if err = globalBucketMetadataSys.Update(bucket, bucketPolicyConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -148,7 +148,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}

-	if err := globalBucketMetadataSys.Update(bucket, bucketPolicyConfig, nil); err != nil {
+	if err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -20,11 +20,11 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"time"

 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
 )

@@ -34,7 +34,7 @@ type BucketQuotaSys struct {
 }

 // Get - Get quota configuration.
-func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) {
+func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerFn()
 		if objAPI == nil {
@@ -43,7 +43,7 @@ func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) {
 		return &madmin.BucketQuota{}, nil
 	}

-	return globalBucketMetadataSys.GetQuotaConfig(bucketName)
+	return globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
 }

 // NewBucketQuotaSys returns initialized BucketQuotaSys
@@ -51,6 +51,35 @@ func NewBucketQuotaSys() *BucketQuotaSys {
 	return &BucketQuotaSys{}
 }

+// Init initialize bucket quota.
+func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
+	sys.bucketStorageCache.Once.Do(func() {
+		sys.bucketStorageCache.TTL = 1 * time.Second
+		sys.bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
+			defer done()
+
+			return loadDataUsageFromBackend(ctx, objAPI)
+		}
+	})
+}
+
+// GetBucketUsageInfo return bucket usage info for a given bucket
+func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
+	v, err := sys.bucketStorageCache.Get()
+	if err != nil {
+		return BucketUsageInfo{}, err
+	}
+
+	dui, ok := v.(DataUsageInfo)
+	if !ok {
+		return BucketUsageInfo{}, fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
+	}
+
+	bui := dui.BucketsUsage[bucket]
+	return bui, nil
+}
+
 // parseBucketQuota parses BucketQuota from json
 func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) {
 	quotaCfg = &madmin.BucketQuota{}
@@ -58,50 +87,32 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 		return quotaCfg, err
 	}
 	if !quotaCfg.IsValid() {
+		if quotaCfg.Type == "fifo" {
+			logger.LogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"))
+			return quotaCfg, nil
+		}
 		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
 	}
 	return
 }

-func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return errServerNotInitialized
+func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string, size int64) error {
+	if size < 0 {
+		return nil
 	}

-	sys.bucketStorageCache.Once.Do(func() {
-		sys.bucketStorageCache.TTL = 1 * time.Second
-		sys.bucketStorageCache.Update = func() (interface{}, error) {
-			ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
-			defer done()
-			return loadDataUsageFromBackend(ctx, objAPI)
-		}
-	})
-
-	q, err := sys.Get(bucket)
+	q, err := sys.Get(ctx, bucket)
 	if err != nil {
 		return err
 	}

 	if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
-		v, err := sys.bucketStorageCache.Get()
+		bui, err := sys.GetBucketUsageInfo(bucket)
 		if err != nil {
 			return err
 		}

-		dui, ok := v.(DataUsageInfo)
-		if !ok {
-			return fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
-		}
-
-		bui, ok := dui.BucketsUsage[bucket]
-		if !ok {
-			// bucket not found, cannot enforce quota
-			// call will fail anyways later.
-			return nil
-		}
-
-		if (bui.Size + uint64(size)) >= q.Quota {
+		if bui.Size > 0 && ((bui.Size + uint64(size)) >= q.Quota) {
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 	}
@@ -109,118 +120,9 @@ func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64)
 	return nil
 }

-func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
-	if size < 0 {
+func enforceBucketQuotaHard(ctx context.Context, bucket string, size int64) error {
+	if globalBucketQuotaSys == nil {
 		return nil
 	}
-
-	return globalBucketQuotaSys.check(ctx, bucket, size)
-}
-
-// enforceFIFOQuota deletes objects in FIFO order until sufficient objects
-// have been deleted so as to bring bucket usage within quota.
-func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui BucketUsageInfo) {
-	// Check if the current bucket has quota restrictions, if not skip it
-	cfg, err := globalBucketQuotaSys.Get(bucket)
-	if err != nil {
-		return
-	}
-
-	if cfg.Type != madmin.FIFOQuota {
-		return
-	}
-
-	var toFree uint64
-	if bui.Size > cfg.Quota && cfg.Quota > 0 {
-		toFree = bui.Size - cfg.Quota
-	}
-
-	if toFree <= 0 {
-		return
-	}
-
-	// Allocate new results channel to receive ObjectInfo.
-	objInfoCh := make(chan ObjectInfo)
-
-	versioned := globalBucketVersioningSys.Enabled(bucket)
-
-	// Walk through all objects
-	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
-
-	// reuse the fileScorer used by disk cache to score entries by
-	// ModTime to find the oldest objects in bucket to delete. In
-	// the context of bucket quota enforcement - number of hits are
-	// irrelevant.
-	scorer, err := newFileScorer(toFree, UTCNow().Unix(), 1)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
-
-	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-	for obj := range objInfoCh {
-		// skip objects currently under retention
-		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
-			continue
-		}
-		scorer.addFileWithObjInfo(obj, 1)
-	}
-
-	// If we saw less than quota we are good.
-	if scorer.seenBytes <= cfg.Quota {
-		return
-	}
-
-	// Calculate how much we want to delete now.
-	toFreeNow := scorer.seenBytes - cfg.Quota
-	// We were less over quota than we thought. Adjust so we delete less.
-	// If we are more over, leave it for the next run to pick up.
-	if toFreeNow < toFree {
-		if !scorer.adjustSaveBytes(int64(toFreeNow) - int64(toFree)) {
-			// We got below or at quota.
-			return
-		}
-	}
-
-	var objects []ObjectToDelete
-	numKeys := len(scorer.fileObjInfos())
-	for i, obj := range scorer.fileObjInfos() {
-		objects = append(objects, ObjectToDelete{
-			ObjectV: ObjectV{
-				ObjectName: obj.Name,
-				VersionID:  obj.VersionID,
-			},
-		})
-		if len(objects) < maxDeleteList && (i < numKeys-1) {
-			// skip deletion until maxDeleteList or end of slice
-			continue
-		}
-
-		if len(objects) == 0 {
-			break
-		}
-
-		// Deletes a list of objects.
-		_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
-			Versioned: versioned,
-		})
-		for j := range deleteErrs {
-			if deleteErrs[j] != nil {
-				logger.LogIf(ctx, deleteErrs[j])
-				continue
-			}
-
-			// Notify object deleted event.
-			sendEvent(eventArgs{
-				EventName:  event.ObjectRemovedDelete,
-				BucketName: bucket,
-				Object:     obj,
-				Host:       "Internal: [FIFO-QUOTA-EXPIRY]",
-			})
-		}
-		objects = nil
-	}
+	return globalBucketQuotaSys.enforceQuotaHard(ctx, bucket, size)
 }
@@ -20,12 +20,15 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
 )

 //go:generate msgp -file=$GOFILE
@@ -576,8 +579,23 @@ type ResyncTargetsInfo struct {

 // ResyncTarget is a struct representing the Target reset ID where target is identified by its Arn
 type ResyncTarget struct {
-	Arn     string `json:"arn"`
-	ResetID string `json:"resetid"`
+	Arn       string    `json:"arn"`
+	ResetID   string    `json:"resetid"`
+	StartTime time.Time `json:"startTime"`
+	EndTime   time.Time `json:"endTime"`
+	// Status of resync operation
+	ResyncStatus string `json:"resyncStatus,omitempty"`
+	// Completed size in bytes
+	ReplicatedSize int64 `json:"completedReplicationSize"`
+	// Failed size in bytes
+	FailedSize int64 `json:"failedReplicationSize"`
+	// Total number of failed operations
+	FailedCount int64 `json:"failedReplicationCount"`
+	// Total number of failed operations
+	ReplicatedCount int64 `json:"replicationCount"`
+	// Last bucket/object replicated.
+	Bucket string `json:"bucket,omitempty"`
+	Object string `json:"object,omitempty"`
 }

 // VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication
@@ -603,3 +621,106 @@ func (v VersionPurgeStatusType) Empty() bool {
 func (v VersionPurgeStatusType) Pending() bool {
 	return v == Pending || v == Failed
 }
+
+type replicationResyncState struct {
+	// map of bucket to their resync status
+	statusMap map[string]BucketReplicationResyncStatus
+	sync.RWMutex
+}
+
+const (
+	replicationDir      = "replication"
+	resyncFileName      = "resync.bin"
+	resyncMetaFormat    = 1
+	resyncMetaVersionV1 = 1
+	resyncMetaVersion   = resyncMetaVersionV1
+)
+
+// ResyncStatusType status of resync operation
+type ResyncStatusType int
+
+const (
+	// NoResync - no resync in progress
+	NoResync ResyncStatusType = iota
+	// ResyncStarted -  resync in progress
+	ResyncStarted
+	// ResyncCompleted -  resync finished
+	ResyncCompleted
+	// ResyncFailed -  resync failed
+	ResyncFailed
+)
+
+func (rt ResyncStatusType) String() string {
+	switch rt {
+	case ResyncStarted:
+		return "Ongoing"
+	case ResyncCompleted:
+		return "Completed"
+	case ResyncFailed:
+		return "Failed"
+	default:
+		return ""
+	}
+}
+
+// TargetReplicationResyncStatus status of resync of bucket for a specific target
+type TargetReplicationResyncStatus struct {
+	StartTime time.Time `json:"startTime" msg:"st"`
+	EndTime   time.Time `json:"endTime" msg:"et"`
+	// Resync ID assigned to this reset
+	ResyncID string `json:"resyncID" msg:"id"`
+	// ResyncBeforeDate - resync all objects created prior to this date
+	ResyncBeforeDate time.Time `json:"resyncBeforeDate" msg:"rdt"`
+	// Status of resync operation
+	ResyncStatus ResyncStatusType `json:"resyncStatus" msg:"rst"`
+	// Failed size in bytes
+	FailedSize int64 `json:"failedReplicationSize"  msg:"fs"`
+	// Total number of failed operations
+	FailedCount int64 `json:"failedReplicationCount"  msg:"frc"`
+	// Completed size in bytes
+	ReplicatedSize int64 `json:"completedReplicationSize"  msg:"rs"`
+	// Total number of failed operations
+	ReplicatedCount int64 `json:"replicationCount"  msg:"rrc"`
+	// Last bucket/object replicated.
+	Bucket string `json:"-" msg:"bkt"`
+	Object string `json:"-" msg:"obj"`
+}
+
+// BucketReplicationResyncStatus captures current replication resync status
+type BucketReplicationResyncStatus struct {
+	Version int `json:"version" msg:"v"`
+	// map of remote arn to their resync status for a bucket
+	TargetsMap map[string]TargetReplicationResyncStatus `json:"resyncMap,omitempty" msg:"brs"`
+	ID         int                                      `json:"id" msg:"id"`
+	LastUpdate time.Time                                `json:"lastUpdate" msg:"lu"`
+}
+
+func newBucketResyncStatus(bucket string) BucketReplicationResyncStatus {
+	return BucketReplicationResyncStatus{
+		TargetsMap: make(map[string]TargetReplicationResyncStatus),
+		Version:    resyncMetaVersion,
+	}
+}
+
+var contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)
+
+// parse size from content-range header
+func parseSizeFromContentRange(h http.Header) (sz int64, err error) {
+	cr := h.Get(xhttp.ContentRange)
+	if cr == "" {
+		return sz, fmt.Errorf("Content-Range not set")
+	}
+	parts := contentRangeRegexp.FindStringSubmatch(cr)
+	if len(parts) != 4 {
+		return sz, fmt.Errorf("invalid Content-Range header %s", cr)
+	}
+	if parts[3] == "*" {
+		return -1, nil
+	}
+	var usz uint64
+	usz, err = strconv.ParseUint(parts[3], 10, 64)
+	if err != nil {
+		return sz, err
+	}
+	return int64(usz), nil
+}
@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )

+func TestMarshalUnmarshalBucketReplicationResyncStatus(t *testing.T) {
+	v := BucketReplicationResyncStatus{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBucketReplicationResyncStatus(b *testing.B) {
+	v := BucketReplicationResyncStatus{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBucketReplicationResyncStatus(b *testing.B) {
+	v := BucketReplicationResyncStatus{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBucketReplicationResyncStatus(b *testing.B) {
+	v := BucketReplicationResyncStatus{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBucketReplicationResyncStatus(t *testing.T) {
+	v := BucketReplicationResyncStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBucketReplicationResyncStatus Msgsize() is inaccurate")
+	}
+
+	vn := BucketReplicationResyncStatus{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBucketReplicationResyncStatus(b *testing.B) {
+	v := BucketReplicationResyncStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBucketReplicationResyncStatus(b *testing.B) {
+	v := BucketReplicationResyncStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
 	v := ReplicateDecision{}
 	bts, err := v.MarshalMsg(nil)
@@ -686,3 +799,116 @@ func BenchmarkDecodeResyncTargetsInfo(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalTargetReplicationResyncStatus(t *testing.T) {
+	v := TargetReplicationResyncStatus{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgTargetReplicationResyncStatus(b *testing.B) {
+	v := TargetReplicationResyncStatus{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgTargetReplicationResyncStatus(b *testing.B) {
+	v := TargetReplicationResyncStatus{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalTargetReplicationResyncStatus(b *testing.B) {
+	v := TargetReplicationResyncStatus{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeTargetReplicationResyncStatus(t *testing.T) {
+	v := TargetReplicationResyncStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeTargetReplicationResyncStatus Msgsize() is inaccurate")
+	}
+
+	vn := TargetReplicationResyncStatus{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeTargetReplicationResyncStatus(b *testing.B) {
+	v := TargetReplicationResyncStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeTargetReplicationResyncStatus(b *testing.B) {
+	v := TargetReplicationResyncStatus{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
@@ -19,10 +19,13 @@ package cmd

 import (
 	"context"
+	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"math"
 	"net/http"
+	"path"
 	"reflect"
 	"strings"
 	"sync"
@@ -422,7 +425,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		wg.Add(1)
 		go func(index int, tgt *TargetClient) {
 			defer wg.Done()
-			rinfo := replicateDeleteToTarget(ctx, dobj, objectAPI, tgt)
+			rinfo := replicateDeleteToTarget(ctx, dobj, tgt)
 			rinfos.Targets[index] = rinfo
 		}(idx, tgt)
 	}
@@ -479,7 +482,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	}
 }

-func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID
@@ -998,6 +1001,7 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 		OpType:                ri.OpType,
 		ReplicationAction:     rAction,
 	}
+
 	if ri.ObjectInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
 		rinfo.ReplicationStatus = replication.Completed
 		rinfo.ReplicationResynced = true
@@ -1057,6 +1061,13 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 		})
 		return rinfo
 	}
+	defer func() {
+		if rinfo.ReplicationStatus == replication.Completed && ri.OpType == replication.ExistingObjectReplicationType && tgt.ResetID != "" {
+			rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID)
+			rinfo.ReplicationResynced = true
+		}
+		rinfo.Duration = time.Since(startTime)
+	}()

 	rAction = replicateAll
 	oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, miniogo.StatObjectOptions{
@@ -1089,13 +1100,8 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 				// Note: Replication Stats would have been updated despite metadata update failure.
 				gr.Close()
 				closeOnDefer = false
-				return replicatedTargetInfo{
-					ReplicationStatus:     replication.Completed,
-					Size:                  sz,
-					Arn:                   tgt.ARN,
-					ReplicationAction:     rAction,
-					PrevReplicationStatus: objInfo.TargetReplicationStatus(tgt.ARN),
-				}
+				rinfo.ReplicationAction = rAction
+				rinfo.ReplicationStatus = replication.Completed
 			}
 			return
 		}
@@ -1103,13 +1109,6 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 	rinfo.ReplicationStatus = replication.Completed
 	rinfo.Size = size
 	rinfo.ReplicationAction = rAction
-	defer func() {
-		if rinfo.ReplicationStatus == replication.Completed && ri.OpType == replication.ExistingObjectReplicationType && tgt.ResetID != "" {
-			rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID)
-			rinfo.ReplicationResynced = true
-		}
-		rinfo.Duration = time.Since(startTime)
-	}()
 	// use core client to avoid doing multipart on PUT
 	c := &miniogo.Core{Client: tgt.Client}
 	if rAction != replicateAll {
@@ -1308,6 +1307,7 @@ type ReplicationPool struct {
 	existingReplicaDeleteCh chan DeletedObjectReplicationInfo
 	workerSize              int
 	mrfWorkerSize           int
+	resyncState             replicationResyncState
 	workerWg                sync.WaitGroup
 	mrfWorkerWg             sync.WaitGroup
 	once                    sync.Once
@@ -1324,6 +1324,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		mrfWorkerKillCh:         make(chan struct{}, opts.FailedWorkers),
 		existingReplicaCh:       make(chan ReplicateObjectInfo, 100000),
 		existingReplicaDeleteCh: make(chan DeletedObjectReplicationInfo, 100000),
+		resyncState:             replicationResyncState{statusMap: make(map[string]BucketReplicationResyncStatus)},
 		ctx:                     ctx,
 		objLayer:                o,
 	}
@@ -1331,6 +1332,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	pool.ResizeWorkers(opts.Workers)
 	pool.ResizeFailedWorkers(opts.FailedWorkers)
 	go pool.AddExistingObjectReplicateWorker()
+	go pool.periodicResyncMetaSave(ctx, o)
 	return pool
 }

@@ -1529,16 +1531,21 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 	go globalReplicationStats.loadInitialReplicationMetrics(ctx)
 }

+type proxyResult struct {
+	Proxy bool
+	Err   error
+}
+
 // get Reader from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy bool) {
-	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
-	if !proxy {
-		return nil, false
+func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy proxyResult, err error) {
+	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
+	if !proxy.Proxy {
+		return nil, proxy, nil
 	}
-	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	fn, _, _, err := NewGetObjectReader(nil, oi, opts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	gopts := miniogo.GetObjectOptions{
 		VersionID:            opts.VersionID,
@@ -1546,30 +1553,40 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		Internal: miniogo.AdvancedGetOptions{
 			ReplicationProxyRequest: "true",
 		},
+		PartNumber: opts.PartNumber,
 	}
 	// get correct offsets for encrypted object
-	if off >= 0 && length >= 0 {
-		if err := gopts.SetRange(off, off+length-1); err != nil {
-			return nil, false
+	if rs != nil {
+		h, err := rs.ToHeader()
+		if err != nil {
+			return nil, proxy, err
 		}
+		gopts.Set(xhttp.Range, h)
 	}
 	// Make sure to match ETag when proxying.
 	if err = gopts.SetMatchETag(oi.ETag); err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	c := miniogo.Core{Client: tgt.Client}
-	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
+	obj, _, h, err := c.GetObject(ctx, bucket, object, gopts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	closeReader := func() { obj.Close() }
-
 	reader, err := fn(obj, h, closeReader)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	reader.ObjInfo = oi.Clone()
-	return reader, true
+	if rs != nil {
+		contentSize, err := parseSizeFromContentRange(h)
+		if err != nil {
+			return nil, proxy, err
+		}
+		reader.ObjInfo.Size = contentSize
+	}
+
+	return reader, proxyResult{Proxy: true}, nil
 }

 func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions) (tgts *madmin.BucketTargets) {
@@ -1588,11 +1605,11 @@ func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	return tgts
 }

-func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy bool) {
+func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy proxyResult) {
 	// this option is set when active-active replication is in place between site A -> B,
 	// and site B does not have the object yet.
 	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
-		return nil, oi, false
+		return nil, oi, proxy
 	}
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, t.Arn)
@@ -1610,9 +1627,22 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 			Internal: miniogo.AdvancedGetOptions{
 				ReplicationProxyRequest: "true",
 			},
+			PartNumber: opts.PartNumber,
 		}
+		if rs != nil {
+			h, err := rs.ToHeader()
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				continue
+			}
+			gopts.Set(xhttp.Range, h)
+		}
+
 		objInfo, err := tgt.StatObject(ctx, t.TargetBucket, object, gopts)
 		if err != nil {
+			if isErrInvalidRange(ErrorRespToObjectError(err, bucket, object)) {
+				return nil, oi, proxyResult{Err: err}
+			}
 			continue
 		}

@@ -1643,15 +1673,15 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 		if ok {
 			oi.ContentEncoding = ce
 		}
-		return tgt, oi, true
+		return tgt, oi, proxyResult{Proxy: true}
 	}
-	return nil, oi, false
+	return nil, oi, proxy
 }

 // get object info from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy bool) {
-	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
+func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy proxyResult) {
+	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
 	return oi, proxy
 }

@@ -1870,3 +1900,333 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 	s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
 	return s
 }
+
+const resyncTimeInterval = time.Minute * 10
+
+// periodicResyncMetaSave saves in-memory resync meta stats to disk in periodic intervals
+func (p *ReplicationPool) periodicResyncMetaSave(ctx context.Context, objectAPI ObjectLayer) {
+	resyncTimer := time.NewTimer(resyncTimeInterval)
+	defer resyncTimer.Stop()
+
+	for {
+		select {
+		case <-resyncTimer.C:
+			resyncTimer.Reset(resyncTimeInterval)
+			now := UTCNow()
+			p.resyncState.RLock()
+			for bucket, brs := range p.resyncState.statusMap {
+				var updt bool
+				for _, st := range brs.TargetsMap {
+					// if resync in progress or just ended, needs to save to disk
+					if st.EndTime.Equal(timeSentinel) || now.Sub(st.EndTime) <= resyncTimeInterval {
+						updt = true
+						break
+					}
+				}
+				if updt {
+					brs.LastUpdate = now
+					if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil {
+						logger.LogIf(ctx, fmt.Errorf("Could not save resync metadata to disk for %s - %w", bucket, err))
+						continue
+					}
+				}
+			}
+			p.resyncState.RUnlock()
+		case <-ctx.Done():
+			// server could be restarting - need
+			// to exit immediately
+			return
+		}
+	}
+}
+
+// resyncBucket resyncs all qualifying objects as per replication rules for the target
+// ARN
+func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI ObjectLayer) {
+	resyncStatus := ResyncFailed
+	defer func() {
+		globalReplicationPool.resyncState.Lock()
+		m := globalReplicationPool.resyncState.statusMap[bucket]
+		st := m.TargetsMap[arn]
+		st.EndTime = UTCNow()
+		st.ResyncStatus = resyncStatus
+		m.TargetsMap[arn] = st
+		globalReplicationPool.resyncState.Unlock()
+	}()
+	// Allocate new results channel to receive ObjectInfo.
+	objInfoCh := make(chan ObjectInfo)
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Replication resync of %s for arn %s failed with %w", bucket, arn, err))
+		return
+	}
+	tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Replication resync of %s for arn %s failed  %w", bucket, arn, err))
+		return
+	}
+	rcfg := replicationConfig{
+		Config:  cfg,
+		remotes: tgts,
+	}
+	tgtArns := cfg.FilterTargetArns(
+		replication.ObjectOpts{
+			OpType:    replication.ResyncReplicationType,
+			TargetArn: arn,
+		})
+	if len(tgtArns) != 1 {
+		logger.LogIf(ctx, fmt.Errorf("Replication resync failed for %s - arn specified %s is missing in the replication config", bucket, arn))
+		return
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn)
+	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("Replication resync failed for %s - target could not be created for arn %s", bucket, arn))
+		return
+	}
+
+	// Walk through all object versions - note ascending order of walk needed to ensure delete marker replicated to
+	// target after object version is first created.
+	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkAscending: true}); err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+
+	globalReplicationPool.resyncState.RLock()
+	m := globalReplicationPool.resyncState.statusMap[bucket]
+	st := m.TargetsMap[arn]
+	globalReplicationPool.resyncState.RUnlock()
+	var lastCheckpoint string
+	if st.ResyncStatus == ResyncStarted || st.ResyncStatus == ResyncFailed {
+		lastCheckpoint = st.Object
+	}
+	for obj := range objInfoCh {
+		if heal && lastCheckpoint != "" && lastCheckpoint != obj.Name {
+			continue
+		}
+		lastCheckpoint = ""
+
+		roi := getHealReplicateObjectInfo(obj, rcfg)
+		if !roi.ExistingObjResync.mustResync() {
+			continue
+		}
+
+		if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
+
+			versionID := ""
+			dmVersionID := ""
+			if roi.VersionPurgeStatus.Empty() {
+				dmVersionID = roi.VersionID
+			} else {
+				versionID = roi.VersionID
+			}
+
+			doi := DeletedObjectReplicationInfo{
+				DeletedObject: DeletedObject{
+					ObjectName:            roi.Name,
+					DeleteMarkerVersionID: dmVersionID,
+					VersionID:             versionID,
+					ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+					DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+					DeleteMarker:          roi.DeleteMarker,
+				},
+				Bucket: roi.Bucket,
+				OpType: replication.ExistingObjectReplicationType,
+			}
+			replicateDelete(ctx, doi, objectAPI, ReplicateDelete)
+		} else {
+			roi.OpType = replication.ExistingObjectReplicationType
+			replicateObject(ctx, roi, objectAPI, ReplicateExisting)
+		}
+		_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, miniogo.StatObjectOptions{
+			VersionID: roi.VersionID,
+			Internal: miniogo.AdvancedGetOptions{
+				ReplicationProxyRequest: "false",
+			},
+		})
+		globalReplicationPool.resyncState.Lock()
+		m = globalReplicationPool.resyncState.statusMap[bucket]
+		st = m.TargetsMap[arn]
+		st.Object = roi.Name
+		if err != nil {
+			if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, bucket, roi.Name)) {
+				st.ReplicatedCount++
+			} else {
+				st.FailedCount++
+			}
+		} else {
+			st.ReplicatedCount++
+			st.ReplicatedSize += roi.Size
+		}
+		m.TargetsMap[arn] = st
+		globalReplicationPool.resyncState.Unlock()
+	}
+	resyncStatus = ResyncCompleted
+}
+
+// start replication resync for the remote target ARN specified
+func startReplicationResync(ctx context.Context, bucket, arn, resyncID string, resyncBeforeDate time.Time, objAPI ObjectLayer) error {
+	if bucket == "" {
+		return fmt.Errorf("bucket name is empty")
+	}
+	if arn == "" {
+		return fmt.Errorf("target ARN specified for resync is empty")
+	}
+	// Check if the current bucket has quota restrictions, if not skip it
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	tgtArns := cfg.FilterTargetArns(
+		replication.ObjectOpts{
+			OpType:    replication.ResyncReplicationType,
+			TargetArn: arn,
+		})
+
+	if len(tgtArns) == 0 {
+		return fmt.Errorf("arn %s specified for resync not found in replication config", arn)
+	}
+
+	data, err := loadBucketResyncMetadata(ctx, bucket, objAPI)
+	if err != nil {
+		return err
+	}
+	// validate if resync is in progress for this arn
+	for tArn, st := range data.TargetsMap {
+		if arn == tArn && st.ResyncStatus == ResyncStarted {
+			return fmt.Errorf("Resync of bucket %s is already in progress for remote bucket %s", bucket, arn)
+		}
+	}
+
+	status := TargetReplicationResyncStatus{
+		ResyncID:         resyncID,
+		ResyncBeforeDate: resyncBeforeDate,
+		StartTime:        UTCNow(),
+		ResyncStatus:     ResyncStarted,
+		Bucket:           bucket,
+	}
+	data.TargetsMap[arn] = status
+	if err = saveResyncStatus(ctx, bucket, data, objAPI); err != nil {
+		return err
+	}
+	globalReplicationPool.resyncState.Lock()
+	defer globalReplicationPool.resyncState.Unlock()
+	brs, ok := globalReplicationPool.resyncState.statusMap[bucket]
+	if !ok {
+		brs = BucketReplicationResyncStatus{
+			Version:    resyncMetaVersion,
+			TargetsMap: make(map[string]TargetReplicationResyncStatus),
+		}
+	}
+	brs.TargetsMap[arn] = status
+	globalReplicationPool.resyncState.statusMap[bucket] = brs
+	go resyncBucket(GlobalContext, bucket, arn, false, objAPI)
+	return nil
+}
+
+// delete resync metadata from replication resync state in memory
+func (p *ReplicationPool) deleteResyncMetadata(ctx context.Context, bucket string) {
+	if p == nil {
+		return
+	}
+	p.resyncState.Lock()
+	delete(p.resyncState.statusMap, bucket)
+	defer p.resyncState.Unlock()
+}
+
+// initResync - initializes bucket replication resync for all buckets.
+func (p *ReplicationPool) initResync(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+	// replication applies only to erasure coded setups
+	if !globalIsErasure {
+		return nil
+	}
+	// Load bucket metadata sys in background
+	go p.loadResync(ctx, buckets, objAPI)
+	return nil
+}
+
+// Loads bucket replication resync statuses into memory.
+func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+	for index := range buckets {
+		meta, err := loadBucketResyncMetadata(ctx, buckets[index].Name, objAPI)
+		if err != nil {
+			if errors.Is(err, errVolumeNotFound) {
+				meta = newBucketResyncStatus(buckets[index].Name)
+			} else {
+				logger.LogIf(ctx, err)
+				continue
+			}
+		}
+		p.resyncState.statusMap[buckets[index].Name] = meta
+	}
+	for index := range buckets {
+		bucket := buckets[index].Name
+		m, ok := p.resyncState.statusMap[bucket]
+		if ok {
+			for arn, st := range m.TargetsMap {
+				if st.ResyncStatus == ResyncFailed || st.ResyncStatus == ResyncStarted {
+					go resyncBucket(ctx, bucket, arn, true, objAPI)
+				}
+			}
+		}
+	}
+}
+
+// load bucket resync metadata from disk
+func loadBucketResyncMetadata(ctx context.Context, bucket string, objAPI ObjectLayer) (brs BucketReplicationResyncStatus, e error) {
+	brs = newBucketResyncStatus(bucket)
+
+	resyncDirPath := path.Join(bucketMetaPrefix, bucket, replicationDir)
+	data, err := readConfig(GlobalContext, objAPI, pathJoin(resyncDirPath, resyncFileName))
+	if err != nil && err != errConfigNotFound {
+		return brs, err
+	}
+	if len(data) == 0 {
+		// Seems to be empty.
+		return brs, nil
+	}
+	if len(data) <= 4 {
+		return brs, fmt.Errorf("replication resync: no data")
+	}
+	// Read resync meta header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case resyncMetaFormat:
+	default:
+		return brs, fmt.Errorf("resyncMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case resyncMetaVersion:
+	default:
+		return brs, fmt.Errorf("resyncMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+	// OK, parse data.
+	if _, err = brs.UnmarshalMsg(data[4:]); err != nil {
+		return brs, err
+	}
+
+	switch brs.Version {
+	case resyncMetaVersionV1:
+	default:
+		return brs, fmt.Errorf("unexpected resync meta version: %d", brs.Version)
+	}
+	return brs, nil
+}
+
+// save resync status to resync.bin
+func saveResyncStatus(ctx context.Context, bucket string, brs BucketReplicationResyncStatus, objectAPI ObjectLayer) error {
+	data := make([]byte, 4, brs.Msgsize()+4)
+
+	// Initialize the resync meta header.
+	binary.LittleEndian.PutUint16(data[0:2], resyncMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], resyncMetaVersion)
+
+	buf, err := brs.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	configFile := path.Join(bucketMetaPrefix, bucket, replicationDir, resyncFileName)
+	return saveConfig(ctx, objectAPI, configFile, buf)
+}
@@ -95,7 +95,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}

-	if err = globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -100,12 +100,16 @@ func init() {
 		PersistOnFailure: false,
 	}

+	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
+
+	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile()
+
 	// Call to refresh will refresh names in cache. If you pass true, it will also
 	// remove cached names not looked up since the last call to Refresh. It is a good idea
 	// to call this method on a regular interval.
 	go func() {
 		var t *time.Ticker
-		if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
+		if containers {
 			t = time.NewTicker(1 * time.Minute)
 		} else {
 			t = time.NewTicker(10 * time.Minute)
@@ -213,6 +217,9 @@ func minioConfigToConsoleFeatures() {
 	if globalSubnetConfig.APIKey != "" {
 		os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
 	}
+	if globalSubnetConfig.Proxy != "" {
+		os.Setenv("CONSOLE_SUBNET_PROXY", globalSubnetConfig.Proxy)
+	}
 }

 func initConsoleServer() (*restapi.Server, error) {
@@ -324,7 +331,7 @@ func checkUpdate(mode string) {
 		return
 	}

-	logStartupMessage(prepareUpdateMessage("\nRun `mc admin update`", lrTime.Sub(crTime)))
+	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
 }

 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -661,8 +668,6 @@ func handleCommonEnvVars() {
 		globalRootDiskThreshold = size
 	}

-	globalIsCICD = env.Get("MINIO_CI_CD", "") != ""
-
 	domains := env.Get(config.EnvDomain, "")
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
@@ -759,7 +764,7 @@ func handleCommonEnvVars() {
 				"         Please use %s and %s",
 				config.EnvAccessKey, config.EnvSecretKey,
 				config.EnvRootUser, config.EnvRootPassword)
-			logStartupMessage(color.RedBold(msg))
+			logger.Info(color.RedBold(msg))
 		}
 		globalActiveCred = cred
 	}
@@ -824,13 +829,6 @@ func handleCommonEnvVars() {
 	}
 }

-func logStartupMessage(msg string) {
-	if globalConsoleSys != nil {
-		globalConsoleSys.Send(msg, string(logger.All))
-	}
-	logger.StartupMessage(msg)
-}
-
 func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, false, nil
@@ -907,6 +905,9 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	}
 	secureConn = true

+	// Certs might be symlinks, reload them every 10 seconds.
+	manager.UpdateReloadDuration(10 * time.Second)
+
 	// syscall.SIGHUP to reload the certs.
 	manager.ReloadOnSignal(syscall.SIGHUP)

@@ -44,8 +44,6 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/target/http"
-	"github.com/minio/minio/internal/logger/target/kafka"
 	"github.com/minio/pkg/env"
 )

@@ -255,65 +253,55 @@ var (
 	globalServerConfigMu sync.RWMutex
 )

-func validateConfig(s config.Config) error {
-	objAPI := newObjectLayerFn()
-
-	// We must have a global lock for this so nobody else modifies env while we do.
-	defer env.LockSetEnv()()
-
-	// Disable merging env values with config for validation.
-	env.SetEnvOff()
-
-	// Enable env values to validate KMS.
-	defer env.SetEnvOn()
-
-	if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil {
-		return err
-	}
-
-	if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
-		return err
-	}
-
-	if _, err := api.LookupConfig(s[config.APISubSys][config.Default]); err != nil {
-		return err
-	}
-
-	if globalIsErasure {
-		if objAPI == nil {
-			return errServerNotInitialized
+func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) error {
+	switch subSys {
+	case config.CredentialsSubSys:
+		if _, err := config.LookupCreds(s[config.CredentialsSubSys][config.Default]); err != nil {
+			return err
 		}
-		for _, setDriveCount := range objAPI.SetDriveCounts() {
-			if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
-				return err
+	case config.SiteSubSys:
+		if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
+			return err
+		}
+	case config.APISubSys:
+		if _, err := api.LookupConfig(s[config.APISubSys][config.Default]); err != nil {
+			return err
+		}
+	case config.StorageClassSubSys:
+		if globalIsErasure {
+			if objAPI == nil {
+				return errServerNotInitialized
+			}
+			for _, setDriveCount := range objAPI.SetDriveCounts() {
+				if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
+					return err
+				}
 			}
 		}
-	}
-
-	if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
-		return err
-	}
-
-	compCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
-	if err != nil {
-		return err
-	}
-
-	if objAPI != nil {
-		if compCfg.Enabled && !objAPI.IsCompressionSupported() {
-			return fmt.Errorf("Backend does not support compression")
+	case config.CacheSubSys:
+		if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
+			return err
+		}
+	case config.CompressionSubSys:
+		compCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+		if err != nil {
+			return err
 		}
-	}

-	if _, err = heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
-		return err
-	}
-
-	if _, err = scanner.LookupConfig(s[config.ScannerSubSys][config.Default]); err != nil {
-		return err
-	}
-
-	{
+		if objAPI != nil {
+			if compCfg.Enabled && !objAPI.IsCompressionSupported() {
+				return fmt.Errorf("Backend does not support compression")
+			}
+		}
+	case config.HealSubSys:
+		if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
+			return err
+		}
+	case config.ScannerSubSys:
+		if _, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default]); err != nil {
+			return err
+		}
+	case config.EtcdSubSys:
 		etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 		if err != nil {
 			return err
@@ -325,15 +313,13 @@ func validateConfig(s config.Config) error {
 			}
 			etcdClnt.Close()
 		}
-	}
-	if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
-		return err
-	}
-
-	{
-		cfg, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
-			globalRootCAs)
+	case config.IdentityOpenIDSubSys:
+		if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
+			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
+			return err
+		}
+	case config.IdentityLDAPSubSys:
+		cfg, err := xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default], globalRootCAs)
 		if err != nil {
 			return err
 		}
@@ -344,24 +330,58 @@ func validateConfig(s config.Config) error {
 			}
 			conn.Close()
 		}
+	case config.IdentityTLSSubSys:
+		if _, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default]); err != nil {
+			return err
+		}
+	case config.SubnetSubSys:
+		if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default]); err != nil {
+			return err
+		}
+	case config.PolicyOPASubSys:
+		if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+			NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+			return err
+		}
+	default:
+		if config.LoggerSubSystems.Contains(subSys) {
+			if err := logger.ValidateSubSysConfig(s, subSys); err != nil {
+				return err
+			}
+		}
 	}
-	{
-		_, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
-		if err != nil {
+
+	if config.NotifySubSystems.Contains(subSys) {
+		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs(), subSys); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func validateConfig(s config.Config, subSys string) error {
+	objAPI := newObjectLayerFn()
+
+	// We must have a global lock for this so nobody else modifies env while we do.
+	defer env.LockSetEnv()()
+
+	// Disable merging env values with config for validation.
+	env.SetEnvOff()
+
+	// Enable env values to validate KMS.
+	defer env.SetEnvOn()
+	if subSys != "" {
+		return validateSubSysConfig(s, subSys, objAPI)
+	}
+
+	// No sub-system passed. Validate all of them.
+	for _, ss := range config.SubSystems.ToSlice() {
+		if err := validateSubSysConfig(s, ss, objAPI); err != nil {
 			return err
 		}
 	}

-	if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
-		return err
-	}
-
-	if _, err := logger.LookupConfig(s); err != nil {
-		return err
-	}
-
-	return notify.TestNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs())
+	return nil
 }

 func lookupConfigs(s config.Config, objAPI ObjectLayer) {
@@ -509,7 +529,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}

 	if globalSTSTLSConfig.InsecureSkipVerify {
-		logger.Info("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify)
+		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}

 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
@@ -538,48 +558,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 	}

-	// Load logger targets based on user's configuration
-	loggerUserAgent := getUserAgent(getMinioMode())
-
-	loggerCfg, err := logger.LookupConfig(s)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger/audit targets: %w", err))
-	}
-
-	for _, l := range loggerCfg.HTTP {
-		if l.Enabled {
-			l.LogOnce = logger.LogOnceIf
-			l.UserAgent = loggerUserAgent
-			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
-			// Enable http logging
-			if err = logger.AddTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server logger HTTP target: %w", err))
-			}
-		}
-	}
-
-	for _, l := range loggerCfg.AuditWebhook {
-		if l.Enabled {
-			l.LogOnce = logger.LogOnceIf
-			l.UserAgent = loggerUserAgent
-			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
-			// Enable http audit logging
-			if err = logger.AddAuditTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit HTTP target: %w", err))
-			}
-		}
-	}
-
-	for _, l := range loggerCfg.AuditKafka {
-		if l.Enabled {
-			l.LogOnce = logger.LogOnceIf
-			// Enable Kafka audit logging
-			if err = logger.AddAuditTarget(kafka.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit Kafka target: %w", err))
-			}
-		}
-	}
-
 	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
@@ -600,65 +578,114 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 }

-// applyDynamicConfig will apply dynamic config values.
-// Dynamic systems should be in config.SubSystemsDynamic as well.
-func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {
-	// Read all dynamic configs.
-	// API
-	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
-	}
+func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s config.Config, subSys string) error {
+	switch subSys {
+	case config.APISubSys:
+		apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
+		}
+		var setDriveCounts []int
+		if objAPI != nil {
+			setDriveCounts = objAPI.SetDriveCounts()
+		}
+		globalAPIConfig.init(apiConfig, setDriveCounts)
+	case config.CompressionSubSys:
+		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
+		if err != nil {
+			return fmt.Errorf("Unable to setup Compression: %w", err)
+		}
+		// Validate if the object layer supports compression.
+		if objAPI != nil {
+			if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+				return fmt.Errorf("Backend does not support compression")
+			}
+		}
+		globalCompressConfigMu.Lock()
+		globalCompressConfig = cmpCfg
+		globalCompressConfigMu.Unlock()
+	case config.HealSubSys:
+		healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
+		if err != nil {
+			return fmt.Errorf("Unable to apply heal config: %w", err)
+		}
+		globalHealConfig.Update(healCfg)
+	case config.ScannerSubSys:
+		scannerCfg, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default])
+		if err != nil {
+			return fmt.Errorf("Unable to apply scanner config: %w", err)
+		}
+		// update dynamic scanner values.
+		scannerCycle.Update(scannerCfg.Cycle)
+		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
+	case config.LoggerWebhookSubSys:
+		loggerCfg, err := logger.LookupConfigForSubSys(s, config.LoggerWebhookSubSys)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err))
+		}
+		userAgent := getUserAgent(getMinioMode())
+		for n, l := range loggerCfg.HTTP {
+			if l.Enabled {
+				l.LogOnce = logger.LogOnceIf
+				l.UserAgent = userAgent
+				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
+				loggerCfg.HTTP[n] = l
+			}
+		}
+		err = logger.UpdateSystemTargets(loggerCfg)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
+		}
+	case config.AuditWebhookSubSys:
+		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditWebhookSubSys)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err))
+		}
+		userAgent := getUserAgent(getMinioMode())
+		for n, l := range loggerCfg.AuditWebhook {
+			if l.Enabled {
+				l.LogOnce = logger.LogOnceIf
+				l.UserAgent = userAgent
+				l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
+				loggerCfg.AuditWebhook[n] = l
+			}
+		}

-	// Compression
-	cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
-	if err != nil {
-		return fmt.Errorf("Unable to setup Compression: %w", err)
-	}
-
-	// Validate if the object layer supports compression.
-	if objAPI != nil {
-		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
-			return fmt.Errorf("Backend does not support compression")
+		err = logger.UpdateAuditWebhookTargets(loggerCfg)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
+		}
+	case config.AuditKafkaSubSys:
+		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditKafkaSubSys)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err))
+		}
+		for n, l := range loggerCfg.AuditKafka {
+			if l.Enabled {
+				l.LogOnce = logger.LogOnceIf
+				loggerCfg.AuditKafka[n] = l
+			}
+		}
+		err = logger.UpdateAuditKafkaTargets(loggerCfg)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
 		}
 	}
-
-	// Heal
-	healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
-	if err != nil {
-		return fmt.Errorf("Unable to apply heal config: %w", err)
-	}
-
-	// Scanner
-	scannerCfg, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default])
-	if err != nil {
-		return fmt.Errorf("Unable to apply scanner config: %w", err)
-	}
-
-	// Apply configurations.
-	// We should not fail after this.
-	var setDriveCounts []int
-	if objAPI != nil {
-		setDriveCounts = objAPI.SetDriveCounts()
-	}
-	globalAPIConfig.init(apiConfig, setDriveCounts)
-
-	globalCompressConfigMu.Lock()
-	globalCompressConfig = cmpCfg
-	globalCompressConfigMu.Unlock()
-
-	globalHealConfig.Update(healCfg)
-
-	// update dynamic scanner values.
-	scannerCycle.Update(scannerCfg.Cycle)
-	logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
-
-	// Update all dynamic config values in memory.
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()
 	if globalServerConfig != nil {
-		for k := range config.SubSystemsDynamic {
-			globalServerConfig[k] = s[k]
+		globalServerConfig[subSys] = s[subSys]
+	}
+	return nil
+}
+
+// applyDynamicConfig will apply dynamic config values.
+// Dynamic systems should be in config.SubSystemsDynamic as well.
+func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {
+	for subSys := range config.SubSystemsDynamic {
+		err := applyDynamicConfigForSubSys(ctx, objAPI, s, subSys)
+		if err != nil {
+			return err
 		}
 	}
 	return nil
@@ -74,7 +74,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		if err != nil {
 			return err
 		}
-		logger.Info("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name)
+		logger.Info(fmt.Sprintf("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name))
 	}

 	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
@@ -143,7 +143,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		if err != nil {
 			return err
 		}
-		logger.Info("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name)
+		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
 	}

 	var marker string
@@ -25,6 +25,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/logger/target/console"
+	"github.com/minio/minio/internal/logger/target/types"
 	"github.com/minio/minio/internal/pubsub"
 	xnet "github.com/minio/pkg/net"
 )
@@ -77,7 +78,7 @@ func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
 func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
-		logger.AddTarget(sys)
+		logger.AddSystemTarget(sys)
 	}

 	cnt := 0
@@ -129,7 +130,7 @@ func (sys *HTTPConsoleLoggerSys) Endpoint() string {

 // String - stringer function for interface compatibility
 func (sys *HTTPConsoleLoggerSys) String() string {
-	return "console+http"
+	return logger.ConsoleLoggerTgt
 }

 // Content returns the console stdout log
@@ -150,6 +151,15 @@ func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
 	return
 }

+// Cancel - cancels the target
+func (sys *HTTPConsoleLoggerSys) Cancel() {
+}
+
+// Type - returns type of the target
+func (sys *HTTPConsoleLoggerSys) Type() types.TargetType {
+	return types.TargetConsole
+}
+
 // Send log message 'e' to console and publish to console
 // log pubsub system
 func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {
@@ -289,7 +289,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 			if poolIdx < len(objAPI.serverPools) && setIdx < len(objAPI.serverPools[poolIdx].sets) {
 				// Pass the disks belonging to the set.
 				s.disks = objAPI.serverPools[poolIdx].sets[setIdx].getDisks()
-				s.disksQuorum = objAPI.serverPools[poolIdx].sets[setIdx].defaultRQuorum()
+				s.disksQuorum = len(s.disks) / 2
 			} else {
 				logger.LogIf(ctx, fmt.Errorf("Matching pool %s, set %s not found", humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1)))
 			}
@@ -403,7 +403,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		if filter != nil && ok && existing.Compacted {
 			// If folder isn't in filter and we have data, skip it completely.
 			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				if f.healObjectSelect == 0 || !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+				if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					if f.dataUsageScannerDebug {
@@ -482,7 +482,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				debug:       f.dataUsageScannerDebug,
 				lifeCycle:   activeLifeCycle,
 				replication: replicationCfg,
-				heal:        thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
+				heal:        thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
 			// if the drive belongs to an erasure set
 			// that is already being healed, skip the
@@ -609,13 +609,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if f.healObjectSelect == 0 || !h.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+					if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 						// Transfer and add as child...
 						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
 						into.addChild(h)
 						continue
 					}
-					folder.objectHealProbDiv = dataUsageUpdateDirCycles
+					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 			scanFolder(folder)
@@ -31,6 +31,7 @@ import (

 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/log"
+	"github.com/minio/minio/internal/logger/target/types"
 )

 type testLoggerI interface {
@@ -55,6 +56,13 @@ func (t *testingLogger) Init() error {
 	return nil
 }

+func (t *testingLogger) Cancel() {
+}
+
+func (t *testingLogger) Type() types.TargetType {
+	return types.TargetHTTP
+}
+
 func (t *testingLogger) Send(entry interface{}, errKind string) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
@@ -73,7 +81,7 @@ func (t *testingLogger) Send(entry interface{}, errKind string) error {

 func addTestingLogging(t testLoggerI) func() {
 	tl := &testingLogger{t: t}
-	logger.AddTarget(tl)
+	logger.AddSystemTarget(tl)
 	return func() {
 		tl.mu.Lock()
 		defer tl.mu.Unlock()
@@ -369,6 +369,17 @@ func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
 	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }

+// modAlt returns true if the hash mod cycles == cycle.
+// This is out of sync with mod.
+// If cycles is 0 false is always returned.
+// If cycles is 1 true is always returned (as expected).
+func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool {
+	if cycles <= 1 {
+		return cycles == 1
+	}
+	return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles
+}
+
 // addChild will add a child based on its hash.
 // If it already exists it will not be added again.
 func (e *dataUsageEntry) addChild(hash dataUsageHash) {
@@ -798,35 +809,6 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 	return dst
 }

-// bucketUsageInfo returns the buckets usage info.
-// If not found all values returned are zero values.
-func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
-	e := d.find(bucket)
-	if e == nil {
-		return BucketUsageInfo{}
-	}
-	flat := d.flatten(*e)
-	bui := BucketUsageInfo{
-		Size:                 uint64(flat.Size),
-		ObjectsCount:         flat.Objects,
-		ObjectSizesHistogram: flat.ObjSizes.toMap(),
-	}
-	if flat.ReplicationStats != nil {
-		bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
-		bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
-		for arn, stat := range flat.ReplicationStats.Targets {
-			bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
-				ReplicationPendingSize:  stat.PendingSize,
-				ReplicatedSize:          stat.ReplicatedSize,
-				ReplicationFailedSize:   stat.FailedSize,
-				ReplicationPendingCount: stat.PendingCount,
-				ReplicationFailedCount:  stat.FailedCount,
-			}
-		}
-	}
-	return bui
-}
-
 // sizeRecursive returns the path as a flattened entry.
 func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
 	root := d.find(path)
@@ -92,32 +92,37 @@ type DataUsageInfo struct {
 }

 func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
-	if globalTierConfigMgr.Empty() {
+	if dui.TierStats == nil {
 		return nil
 	}

-	ts := make(map[string]madmin.TierStats)
-	// Add configured remote tiers
-	for tier := range globalTierConfigMgr.Tiers {
-		ts[tier] = madmin.TierStats{}
+	cfgs := globalTierConfigMgr.ListTiers()
+	if len(cfgs) == 0 {
+		return nil
 	}
+
+	ts := make(map[string]madmin.TierStats, len(cfgs)+1)
+	infos := make([]madmin.TierInfo, 0, len(ts))
+
 	// Add STANDARD (hot-tier)
 	ts[minioHotTier] = madmin.TierStats{}
+	infos = append(infos, madmin.TierInfo{
+		Name: minioHotTier,
+		Type: "internal",
+	})
+	// Add configured remote tiers
+	for _, cfg := range cfgs {
+		ts[cfg.Name] = madmin.TierStats{}
+		infos = append(infos, madmin.TierInfo{
+			Name: cfg.Name,
+			Type: cfg.Type.String(),
+		})
+	}

 	ts = dui.TierStats.adminStats(ts)
-	infos := make([]madmin.TierInfo, 0, len(ts))
-	for tier, st := range ts {
-		var tierType string
-		if tier == minioHotTier {
-			tierType = "internal"
-		} else {
-			tierType = globalTierConfigMgr.Tiers[tier].Type.String()
-		}
-		infos = append(infos, madmin.TierInfo{
-			Name:  tier,
-			Type:  tierType,
-			Stats: st,
-		})
+	for i := range infos {
+		info := infos[i]
+		infos[i].Stats = ts[info.Name]
 	}

 	sort.Slice(infos, func(i, j int) bool {
@@ -131,3 +136,31 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 	})
 	return infos
 }
+
+func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
+	if dui.TierStats == nil {
+		return nil
+	}
+	// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
+	//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
+	//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
+	for tier, st := range dui.TierStats.Tiers {
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedBytesMD(),
+			Value:          float64(st.TotalSize),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedObjectsMD(),
+			Value:          float64(st.NumObjects),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedVersionsMD(),
+			Value:          float64(st.NumVersions),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+	}
+
+	return metrics
+}
@@ -24,6 +24,7 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -252,16 +253,17 @@ func (c *diskCache) diskUsageLow() bool {
 // Returns if the disk usage reaches  or exceeds configured cache quota when size is added.
 // If current usage without size exceeds high watermark a GC is automatically queued.
 func (c *diskCache) diskSpaceAvailable(size int64) bool {
+	reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+
 	gcTriggerPct := c.quotaPct * c.highWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err)
 		return false
 	}
 	if di.Total == 0 {
-		logger.Info("diskCache: Received 0 total disk size")
+		logger.LogIf(ctx, errors.New("diskCache: Received 0 total disk size"))
 		return false
 	}
 	usedPercent := float64(di.Used) * 100 / float64(di.Total)
@@ -477,22 +477,6 @@ func (f *fileScorer) trimQueue() {
 	}
 }

-// fileObjInfos returns all queued file object infos
-func (f *fileScorer) fileObjInfos() []ObjectInfo {
-	res := make([]ObjectInfo, 0, f.queue.Len())
-	e := f.queue.Front()
-	for e != nil {
-		qfile := e.Value.(queuedFile)
-		res = append(res, ObjectInfo{
-			Name:      qfile.name,
-			Size:      int64(qfile.size),
-			VersionID: qfile.versionID,
-		})
-		e = e.Next()
-	}
-	return res
-}
-
 func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
 	e := f.queue.Front()
 	for e != nil {
@@ -19,7 +19,6 @@ package cmd

 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -32,6 +31,7 @@ import (
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -595,14 +595,25 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	if err != nil {
 		return nil, false, err
 	}
+	var warningMsg string
 	for i, dir := range config.Drives {
 		// skip diskCache creation for cache drives missing a format.json
 		if formats[i] == nil {
 			caches = append(caches, nil)
 			continue
 		}
+		if !globalIsCICD && len(warningMsg) == 0 {
+			rootDsk, err := disk.IsRootDisk(dir, "/")
+			if err != nil {
+				warningMsg = fmt.Sprintf("Invalid cache dir %s err : %s", dir, err.Error())
+			}
+			if rootDsk {
+				warningMsg = fmt.Sprintf("cache dir cannot be part of root disk: %s", dir)
+			}
+		}
+
 		if err := checkAtimeSupport(dir); err != nil {
-			return nil, false, errors.New("Atime support required for disk caching")
+			return nil, false, fmt.Errorf("Atime support required for disk caching, atime check failed with %w", err)
 		}

 		cache, err := newDiskCache(ctx, dir, config)
@@ -611,11 +622,14 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		}
 		caches = append(caches, cache)
 	}
+	if warningMsg != "" {
+		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root disk for disk caching is deprecated: %s", warningMsg)))
+	}
 	return caches, migrating, nil
 }

 func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
-	logStartupMessage(color.Blue("Cache migration initiated ...."))
+	logger.Info(color.Blue("Cache migration initiated ...."))

 	g := errgroup.WithNErrs(len(c.cache))
 	for index, dc := range c.cache {
@@ -644,7 +658,7 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {

 	// update migration status
 	c.migrating = false
-	logStartupMessage(color.Blue("Cache migration completed successfully."))
+	logger.Info(color.Blue("Cache migration completed successfully."))
 }

 // PutObject - caches the uploaded object for single Put operations
@@ -33,6 +33,7 @@ import (
 	"strconv"
 	"strings"

+	"github.com/minio/kes"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/fips"
 	xhttp "github.com/minio/minio/internal/http"
@@ -46,6 +47,7 @@ var (
 	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
 	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
 	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
+	errKMSKeyNotFound       = errors.New("Invalid KMS keyId")
 	// Additional MinIO errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
 	// error returned when invalid encryption parameters are specified
@@ -69,11 +71,19 @@ const (
 )

 // KMSKeyID returns in AWS compatible KMS KeyID() format.
-func (o ObjectInfo) KMSKeyID() string {
-	if len(o.UserDefined) == 0 {
+func (o *ObjectInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) }
+
+// KMSKeyID returns in AWS compatible KMS KeyID() format.
+func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) }
+
+// kmsKeyIDFromMetadata returns any AWS S3 KMS key ID in the
+// metadata, if any. It returns an empty ID if no key ID is
+// present.
+func kmsKeyIDFromMetadata(metadata map[string]string) string {
+	if len(metadata) == 0 {
 		return ""
 	}
-	kmsID, ok := o.UserDefined[crypto.MetaKeyID]
+	kmsID, ok := metadata[crypto.MetaKeyID]
 	if !ok {
 		return ""
 	}
@@ -262,6 +272,9 @@ func newEncryptMetadata(kind crypto.Type, keyID string, key []byte, bucket, obje
 		}
 		key, err := GlobalKMS.GenerateKey(keyID, kmsCtx)
 		if err != nil {
+			if errors.Is(err, kes.ErrKeyNotFound) {
+				return crypto.ObjectKey{}, errKMSKeyNotFound
+			}
 			return crypto.ObjectKey{}, err
 		}

@@ -39,8 +39,10 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 	defer NSUpdated(bucket, slashSeparator)

 	// Verify if bucket is valid.
-	if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
-		return BucketNameInvalid{Bucket: bucket}
+	if !isMinioMetaBucketName(bucket) {
+		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
+			return BucketNameInvalid{Bucket: bucket}
+		}
 	}

 	storageDisks := er.getDisks()
@@ -59,9 +59,7 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	// Based on the random shuffling return back randomized disks.
 	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
 		if disks[i-1] != nil && disks[i-1].IsLocal() {
-			if disks[i-1].Healing() == nil && disks[i-1].IsOnline() {
-				newDisks = append(newDisks, disks[i-1])
-			}
+			newDisks = append(newDisks, disks[i-1])
 		}
 	}
 	return newDisks
@@ -83,7 +83,10 @@ func commonTime(modTimes []time.Time) (modTime time.Time) {
 }

 // Beginning of unix time is treated as sentinel value here.
-var timeSentinel = time.Unix(0, 0).UTC()
+var (
+	timeSentinel     = time.Unix(0, 0).UTC()
+	timeSentinel1970 = time.Unix(0, 1).UTC() // 1970 used for special cases when xlmeta.version == 0
+)

 // Boot modTimes up to disk count, setting the value to time sentinel.
 func bootModtimes(diskCount int) []time.Time {
@@ -808,52 +808,20 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 ) {
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
-	// Check if the object is dangling, if yes and user requested
-	// remove we simply delete it from namespace.
-	m, ok := isObjectDangling(metaArr, errs, dataErrs)
-	if ok {
-		parityBlocks := er.defaultParityCount
-		dataBlocks := len(storageDisks) - parityBlocks
-		if m.IsValid() {
-			parityBlocks = m.Erasure.ParityBlocks
-			dataBlocks = m.Erasure.DataBlocks
-		}

-		writeQuorum := dataBlocks
-		if dataBlocks == parityBlocks {
-			writeQuorum++
-		}
-
-		var err error
-		if opts.Remove {
-			err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{
-				VersionID: versionID,
-			}, false)
-
-			// If Delete was successful, make sure to return the appropriate error
-			// and heal result appropriate with delete's error messages
-			errs = make([]error, len(errs))
-			for i := range errs {
-				errs[i] = err
-			}
-			if err == nil {
-				// Dangling object successfully purged, size is '0'
-				m.Size = 0
-			}
-
-			return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
-				errs, bucket, object, versionID), nil
-		}
-
-		return er.defaultHealResult(m, storageDisks, storageEndpoints,
-			errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID)
+	m, err := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, dataErrs, ObjectOptions{
+		VersionID: versionID,
+	})
+	errs = make([]error, len(errs))
+	for i := range errs {
+		errs[i] = err
 	}
-
-	readQuorum := len(storageDisks) - er.defaultParityCount
-
-	err := toObjectErr(reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum),
-		bucket, object, versionID)
-	return er.defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID), err
+	if err == nil {
+		// Dangling object successfully purged, size is '0'
+		m.Size = 0
+	}
+	return er.defaultHealResult(m, storageDisks, storageEndpoints,
+		errs, bucket, object, versionID), nil
 }

 // Object is considered dangling/corrupted if any only
@@ -963,5 +931,12 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	}

 	// Heal the object.
-	return er.healObject(healCtx, bucket, object, versionID, opts)
+	hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	if errors.Is(err, errFileCorrupt) && opts.ScanMode != madmin.HealDeepScan {
+		// Instead of returning an error when a bitrot error is detected
+		// during a normal heal scan, heal again with bitrot flag enabled.
+		opts.ScanMode = madmin.HealDeepScan
+		hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	}
+	return hr, err
 }
@@ -368,7 +368,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	orgDisks := append([]StorageAPI{}, disks...)

 	// Enable versioning.
-	globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`))
+	globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`))

 	_, err = objLayer.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{
 		Versioned: true,
@@ -382,6 +382,13 @@ func TestHealingDanglingObject(t *testing.T) {
 		copy(disks, newDisks)
 		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
 	}
+	getDisk := func(n int) StorageAPI {
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock()
+		disk := disks[n]
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
+		return disk
+	}
+
 	// Remove 4 disks.
 	setDisks(nil, nil, nil, nil)

@@ -440,8 +447,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}

 	setDisks(orgDisks[:4]...)
-
-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk := getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -458,7 +465,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}

-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -488,7 +496,8 @@ func TestHealingDanglingObject(t *testing.T) {

 	setDisks(orgDisks[:4]...)

-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +514,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}

-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -807,7 +817,140 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 }

-func TestHealObjectCorrupted(t *testing.T) {
+func TestHealObjectCorruptedXLMeta(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	z := objLayer.(*erasureServerPools)
+	er := z.serverPools[0].sets[0]
+	erasureDisks := er.getDisks()
+	firstDisk := erasureDisks[0]
+
+	// Test 1: Remove the object backend files from the first disk.
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	if err != nil {
+		t.Fatalf("Failed to delete a file - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Fatalf("Failed to heal object - %v", err)
+	}
+
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi1, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi1) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 2: Test with a corrupted xl.meta
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), []byte("abcd"))
+	if err != nil {
+		t.Errorf("Failure during creating part.1 - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Errorf("Expected nil but received %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi2, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi2) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 3: checks if HealObject returns an error when xl.meta is not found
+	// in more than read quorum number of disks, to create a corrupted situation.
+	for i := 0; i <= nfi2.Erasure.DataBlocks; i++ {
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	}
+
+	// Try healing now, expect to receive errFileNotFound.
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err != nil {
+		if _, ok := err.(ObjectNotFound); !ok {
+			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+		}
+	}
+
+	// since majority of xl.meta's are not available, object should be successfully deleted.
+	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if _, ok := err.(ObjectNotFound); !ok {
+		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	}
+}
+
+func TestHealObjectCorruptedParts(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

@@ -867,7 +1010,26 @@ func TestHealObjectCorrupted(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	secondDisk := erasureDisks[1]
+
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	part1Disk1Origin, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	part1Disk2Origin, err := secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	// Test 1, remove part.1
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -877,85 +1039,67 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}

-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	part1Replaced, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}

-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
 	}

-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	// Test 2, Corrupt part.1
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}

-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}

-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	part1Replaced, err = firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}

-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	// Test 3, Corrupt one part and remove data in another disk
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}

-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = secondDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to delete a file - %v", err)
 	}

-	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}

-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	partReconstructed, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}

-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	partReconstructed, err = secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}

-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	// Test 4: checks if HealObject returns an error when xl.meta is not found
-	// in more than read quorum number of disks, to create a corrupted situation.
-	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-	}
-
-	// Try healing now, expect to receive errFileNotFound.
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
-	if err != nil {
-		if _, ok := err.(ObjectNotFound); !ok {
-			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
-		}
-	}
-
-	// since majority of xl.meta's are not available, object should be successfully deleted.
-	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	if _, ok := err.(ObjectNotFound); !ok {
-		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	if !reflect.DeepEqual(part1Disk2Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
 	}
 }

@@ -1017,7 +1161,8 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}

-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	// Delete the whole object folder
+	err = firstDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -1043,10 +1188,14 @@ func TestHealObjectErasure(t *testing.T) {
 	z.serverPools[0].erasureDisksMu.Unlock()

 	// Try healing now, expect to receive errDiskNotFound.
-	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
-	// since majority of xl.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
-	if _, ok := err.(InsufficientReadQuorum); !ok {
-		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)
+	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{
+		ScanMode: madmin.HealDeepScan,
+	})
+	// since majority of xl.meta's are not available, object quorum
+	// can't be read properly will be deleted automatically and
+	// err is nil
+	if err != nil {
+		t.Fatal(err)
 	}
 }

@@ -67,6 +67,9 @@ func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error)
 // values of maximally occurring errors validated against a generic
 // quorum number that can be read or write quorum depending on usage.
 func reduceQuorumErrs(ctx context.Context, errs []error, ignoredErrs []error, quorum int, quorumErr error) error {
+	if contextCanceled(ctx) {
+		return context.Canceled
+	}
 	maxCount, maxErr := reduceErrs(errs, ignoredErrs)
 	if maxCount >= quorum {
 		return maxErr
@@ -406,11 +406,27 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 		return 0, 0, err
 	}

+	if latestFileInfo.Deleted {
+		// For delete markers do not use 'defaultParityCount' as it is not expected to be the case.
+		// Use maximum allowed read quorum instead, writeQuorum+1 is returned for compatibility sake
+		// but there are no callers that shall be using this.
+		readQuorum := len(partsMetaData) / 2
+		return readQuorum, readQuorum + 1, nil
+	}
+
 	parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass])
 	if parityBlocks <= 0 {
 		parityBlocks = defaultParityCount
 	}

+	// For erasure code upgraded objects choose the parity
+	// blocks saved internally, instead of 'defaultParityCount'
+	if _, ok := latestFileInfo.Metadata[minIOErasureUpgraded]; ok {
+		if latestFileInfo.Erasure.ParityBlocks != 0 {
+			parityBlocks = latestFileInfo.Erasure.ParityBlocks
+		}
+	}
+
 	dataBlocks := latestFileInfo.Erasure.DataBlocks
 	if dataBlocks == 0 {
 		dataBlocks = len(partsMetaData) - parityBlocks
@@ -461,6 +477,18 @@ func (fi *FileInfo) TierFreeVersion() bool {
 	return ok
 }

+// IsRestoreObjReq returns true if fi corresponds to a RestoreObject request.
+func (fi *FileInfo) IsRestoreObjReq() bool {
+	if restoreHdr, ok := fi.Metadata[xhttp.AmzRestore]; ok {
+		if restoreStatus, err := parseRestoreObjStatus(restoreHdr); err == nil {
+			if !restoreStatus.Ongoing() {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // VersionPurgeStatus returns overall version purge status for this object version across targets
 func (fi *FileInfo) VersionPurgeStatus() VersionPurgeStatusType {
 	return fi.ReplicationState.CompositeVersionPurgeStatus()
@@ -343,10 +343,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 				switch {
 				case errors.Is(err, errFileNotFound):
 					scan = madmin.HealNormalScan
-					logger.Info("Healing required, triggering async heal missing shards for %s", pathJoin(bucket, object, fi.VersionID))
 				case errors.Is(err, errFileCorrupt):
 					scan = madmin.HealDeepScan
-					logger.Info("Healing required, triggering async heal bitrot for %s", pathJoin(bucket, object, fi.VersionID))
 				}
 				switch scan {
 				case madmin.HealNormalScan, madmin.HealDeepScan:
@@ -397,6 +395,29 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }

+func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
+	var err error
+	m, ok := isObjectDangling(metaArr, errs, dataErrs)
+	if ok {
+		err = errFileNotFound
+		if opts.VersionID != "" {
+			err = errFileVersionNotFound
+		}
+		defer NSUpdated(bucket, object)
+
+		if opts.VersionID != "" {
+			er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
+				VersionID: opts.VersionID,
+			}, false)
+		} else {
+			er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
+				VersionID: m.VersionID,
+			}, false)
+		}
+	}
+	return m, err
+}
+
 func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
 	disks := er.getDisks()

@@ -405,27 +426,20 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s

 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
-		return fi, nil, nil, err
+		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
+			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			if derr != nil {
+				err = derr
+			}
+		}
+		return fi, nil, nil, toObjectErr(err, bucket, object)
 	}

 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 		if errors.Is(reducedErr, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
-			// Skip buckets that live at `.minio.sys` bucket.
-			if _, ok := isObjectDangling(metaArr, errs, nil); ok {
-				reducedErr = errFileNotFound
-				if opts.VersionID != "" {
-					reducedErr = errFileVersionNotFound
-				}
-				// Remove the dangling object only when:
-				//  - This is a non versioned bucket
-				//  - This is a versioned bucket and the version ID is passed, the reason
-				//    is that we cannot fetch the ID of the latest version when we don't trust xl.meta
-				if !opts.Versioned || opts.VersionID != "" {
-					er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
-						Name:      object,
-						VersionID: opts.VersionID,
-					}, false)
-				}
+			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			if derr != nil {
+				err = derr
 			}
 		}
 		return fi, nil, nil, toObjectErr(reducedErr, bucket, object)
@@ -479,11 +493,6 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 	objInfo = fi.ToObjectInfo(bucket, object)
-	if opts.VersionID != "" && !fi.VersionPurgeStatus().Empty() {
-		// Make sure to return object info to provide extra information.
-		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
-	}
-
 	if fi.Deleted {
 		if opts.VersionID == "" || opts.DeleteMarker {
 			return objInfo, toObjectErr(errFileNotFound, bucket, object)
@@ -558,7 +567,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 	// Wait for all renames to finish.
 	errs := g.Wait()

-	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
+	// We can safely allow RenameData errors up to len(er.getDisks()) - writeQuorum
 	// otherwise return failure. Cleanup successful renames.
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	return evalDisks(disks, errs), err
@@ -985,14 +994,18 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		}
 	}

-	// Whether a disk was initially or becomes offline
-	// during this upload, send it to the MRF list.
-	for i := 0; i < len(onlineDisks); i++ {
-		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
-			continue
+	// For speedtest objects do not attempt to heal them.
+	if !opts.Speedtest {
+		// Whether a disk was initially or becomes offline
+		// during this upload, send it to the MRF list.
+		for i := 0; i < len(onlineDisks); i++ {
+			if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
+				continue
+			}
+
+			er.addPartial(bucket, object, fi.VersionID, fi.Size)
+			break
 		}
-		er.addPartial(bucket, object, fi.VersionID, fi.Size)
-		break
 	}

 	fi.ReplicationState = opts.PutReplicationState()
@@ -1037,7 +1050,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		// class for objects which have reduced quorum
 		// storage class only needs to be honored for
 		// Read() requests alone which we already do.
-		writeQuorums[i] = er.defaultWQuorum()
+		writeQuorums[i] = len(storageDisks)/2 + 1
 	}

 	versionsMap := make(map[string]FileInfoVersions, len(objects))
@@ -1153,9 +1166,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 			errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
 		}

-		if errs[objIndex] == nil {
-			defer NSUpdated(bucket, objects[objIndex].ObjectName)
-		}
+		defer NSUpdated(bucket, objects[objIndex].ObjectName)
 	}

 	// Check failed deletes across multiple objects
@@ -128,6 +128,86 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	removeRoots(fsDirs)
 }

+func TestDeleteObjectsVersioned(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	obj, fsDirs, err := prepareErasure(ctx, 16)
+	if err != nil {
+		t.Fatal("Unable to initialize 'Erasure' object layer.", err)
+	}
+	// Remove all dirs.
+	for _, dir := range fsDirs {
+		defer os.RemoveAll(dir)
+	}
+
+	type testCaseType struct {
+		bucket string
+		object string
+	}
+
+	bucketName := "bucket"
+	testCases := []testCaseType{
+		{bucketName, "dir/obj1"},
+		{bucketName, "dir/obj1"},
+	}
+
+	err = obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{
+		VersioningEnabled: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	names := make([]ObjectToDelete, len(testCases))
+	for i, testCase := range testCases {
+		objInfo, err := obj.PutObject(ctx, testCase.bucket, testCase.object,
+			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{
+				Versioned: true,
+			})
+		if err != nil {
+			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)
+		}
+		names[i] = ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: objInfo.Name,
+				VersionID:  objInfo.VersionID,
+			},
+		}
+
+	}
+	names = append(names, ObjectToDelete{
+		ObjectV: ObjectV{
+			ObjectName: "dir/obj1",
+			VersionID:  mustGetUUID(), // add a non-existent UUID.
+		},
+	})
+
+	_, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{
+		Versioned: true,
+	})
+	for i := range delErrs {
+		if delErrs[i] != nil {
+			t.Errorf("Failed to remove object `%v` with the error: `%v`", names[i], delErrs[i])
+		}
+	}
+
+	for i, test := range testCases {
+		_, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{
+			VersionID: names[i].ObjectV.VersionID,
+		})
+		switch statErr.(type) {
+		case VersionNotFound:
+		default:
+			t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object)
+		}
+	}
+
+	if _, err = ioutil.ReadFile(pathJoin(fsDirs[0], bucketName, "dir/obj1", "xl.meta")); err == nil {
+		t.Fatalf("xl.meta still present after removal")
+	}
+}
+
 func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -472,13 +472,16 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 		// '-1' as argument to decommission multiple pools at a time
 		// but this is not a priority at the moment.
 		for _, pool := range meta.returnResumablePools(1) {
-			err := z.Decommission(ctx, pool.ID)
-			switch err {
-			case errDecommissionAlreadyRunning:
-				fallthrough
-			case nil:
-				go z.doDecommissionInRoutine(ctx, pool.ID)
-			}
+			go func(pool PoolStatus) {
+				switch err := z.Decommission(ctx, pool.ID); err {
+				case errDecommissionAlreadyRunning:
+					fallthrough
+				case nil:
+					z.doDecommissionInRoutine(ctx, pool.ID)
+				default:
+					logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+				}
+			}(pool)
 		}
 		z.poolMeta = meta

@@ -593,7 +596,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				return
 			}

-			// We need a reversed order for Decommissioning,
+			// We need a reversed order for decommissioning,
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()

@@ -682,8 +685,8 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool

 		// How to resolve partial results.
 		resolver := metadataResolutionParams{
-			dirQuorum: set.defaultRQuorum(),
-			objQuorum: set.defaultRQuorum(),
+			dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+			objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
 			bucket:    bName,
 		}

@@ -692,7 +695,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			bucket:         bName,
 			recursive:      true,
 			forwardTo:      forwardTo,
-			minDisks:       len(disks),
+			minDisks:       len(disks) / 2, // to capture all quorum ratios
 			reportNotFound: false,
 			agreed:         decommissionEntry,
 			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
@@ -955,6 +958,21 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		}
 	}

+	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
+	// this code is present to avoid any missing meta buckets on other
+	// pools.
+	for _, metaBucket := range []string{
+		pathJoin(minioMetaBucket, minioConfigPrefix),
+		pathJoin(minioMetaBucket, bucketMetaPrefix),
+	} {
+		var bucketExists BucketExists
+		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
+			if !errors.As(err, &bucketExists) {
+				return err
+			}
+		}
+	}
+
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
 	buckets = append(buckets, BucketInfo{
@@ -128,7 +128,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			if !configRetriableErrors(err) {
 				logger.Fatal(err, "Unable to initialize backend")
 			}
-			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
+			retry := time.Duration(r.Float64() * float64(5*time.Second))
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
+			time.Sleep(retry)
 			continue
 		}
 		break
@@ -335,7 +337,9 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 			pinfo := poolObjInfo{
 				PoolIndex: i,
 			}
-			opts.VersionID = "" // no need to check for specific versionId
+			// do not remove this check as it can lead to inconsistencies
+			// for all callers of bucket replication.
+			opts.VersionID = ""
 			pinfo.ObjInfo, pinfo.Err = pool.GetObjectInfo(ctx, bucket, object, opts)
 			poolObjInfos[i] = pinfo
 		}(i, pool, poolOpts[i])
@@ -353,15 +357,15 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 	})

 	for _, pinfo := range poolObjInfos {
-		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
-			return -1, pinfo.Err
-		}
-
 		// skip all objects from suspended pools for mutating calls.
 		if z.IsSuspended(pinfo.PoolIndex) && opts.Mutate {
 			continue
 		}

+		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
+			return -1, pinfo.Err
+		}
+
 		if isErrObjectNotFound(pinfo.Err) {
 			// No object exists or its a delete marker,
 			// check objInfo to confirm.
@@ -619,12 +623,6 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 				return
 			case v := <-updateCloser:
 				update()
-				// Enforce quotas when all is done.
-				if firstErr == nil {
-					for _, b := range allBuckets {
-						enforceFIFOQuotaBucket(ctx, z, b.Name, allMerged.bucketUsageInfo(b.Name))
-					}
-				}
 				close(v)
 				return
 			case <-updateTicker.C:
@@ -665,6 +663,9 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].MakeBucketWithLocation(ctx, bucket, opts)
 		}, index)
 	}
@@ -675,7 +676,10 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 		if err != nil {
 			if _, ok := err.(BucketExists); !ok {
 				// Delete created buckets, ignoring errors.
-				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: true, NoRecreate: true})
+				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+					Force:      false,
+					NoRecreate: true,
+				})
 			}
 			return err
 		}
@@ -1480,6 +1484,9 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}
@@ -1672,7 +1679,13 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 							cancel()
 							return
 						}
-
+						if opts.WalkAscending {
+							for i := len(fivs.Versions) - 1; i >= 0; i-- {
+								version := fivs.Versions[i]
+								results <- version.ToObjectInfo(bucket, version.Name)
+							}
+							return
+						}
 						for _, version := range fivs.Versions {
 							results <- version.ToObjectInfo(bucket, version.Name)
 						}
@@ -1728,14 +1741,13 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string) error

-func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error, errCh chan<- error) {
+func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()

 	disks, _ := set.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		errCh <- errors.New("listAndHeal: No non-healing disks found")
-		return
+		return errors.New("listAndHeal: No non-healing disks found")
 	}

 	// How to resolve partial results.
@@ -1761,8 +1773,8 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
 			if err := healEntry(entry); err != nil {
-				errCh <- err
-				return
+				logger.LogIf(ctx, err)
+				cancel()
 			}
 		},
 		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
@@ -1774,7 +1786,8 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 			}

 			if err := healEntry(*entry); err != nil {
-				errCh <- err
+				logger.LogIf(ctx, err)
+				cancel()
 				return
 			}
 		},
@@ -1782,13 +1795,13 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 	}

 	if err := listPathRaw(ctx, lopts); err != nil {
-		errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
-		return
+		return fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
 	}
+
+	return nil
 }

 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
-	errCh := make(chan error)
 	healEntry := func(entry metaCacheEntry) error {
 		if entry.isDir() {
 			return nil
@@ -1827,26 +1840,33 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()

-	go func() {
-		defer close(errCh)
-
+	var poolErrs [][]error
+	for idx, erasureSet := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
+		errs := make([]error, len(erasureSet.sets))
 		var wg sync.WaitGroup
-		for idx, erasureSet := range z.serverPools {
-			if z.IsSuspended(idx) {
-				continue
-			}
-			for _, set := range erasureSet.sets {
-				wg.Add(1)
-				go func(set *erasureObjects) {
-					defer wg.Done()
+		for idx, set := range erasureSet.sets {
+			wg.Add(1)
+			go func(idx int, set *erasureObjects) {
+				defer wg.Done()

-					listAndHeal(ctx, bucket, prefix, set, healEntry, errCh)
-				}(set)
-			}
+				errs[idx] = listAndHeal(ctx, bucket, prefix, set, healEntry)
+			}(idx, set)
 		}
 		wg.Wait()
-	}()
-	return <-errCh
+		poolErrs = append(poolErrs, errs)
+	}
+	for _, errs := range poolErrs {
+		for _, err := range errs {
+			if err == nil {
+				continue
+			}
+			return err
+		}
+	}
+	return nil
 }

 func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
@@ -97,21 +97,6 @@ type erasureSets struct {
 	lastConnectDisksOpTime time.Time
 }

-// Return false if endpoint is not connected or has been reconnected after last check
-func isEndpointConnectionStable(diskMap map[Endpoint]StorageAPI, endpoint Endpoint, lastCheck time.Time) bool {
-	disk := diskMap[endpoint]
-	if disk == nil {
-		return false
-	}
-	if !disk.IsOnline() {
-		return false
-	}
-	if !lastCheck.IsZero() && disk.LastConn().After(lastCheck) {
-		return false
-	}
-	return true
-}
-
 func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 	diskMap := make(map[Endpoint]StorageAPI)

@@ -146,7 +131,7 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 		if errors.Is(err, errUnformattedDisk) {
 			info, derr := disk.DiskInfo(context.TODO())
 			if derr != nil && info.RootDisk {
-				return nil, nil, fmt.Errorf("Disk: %s returned %w", disk, derr) // make sure to '%w' to wrap the error
+				return nil, nil, fmt.Errorf("Disk: %s is a root disk", disk)
 			}
 		}
 		return nil, nil, fmt.Errorf("Disk: %s returned %w", disk, err) // make sure to '%w' to wrap the error
@@ -209,12 +194,26 @@ func (s *erasureSets) connectDisks() {
 	}()

 	var wg sync.WaitGroup
-	setsJustConnected := make([]bool, s.setCount)
 	diskMap := s.getDiskMap()
+	setsJustConnected := make([]bool, s.setCount)
 	for _, endpoint := range s.endpoints.Endpoints {
-		if isEndpointConnectionStable(diskMap, endpoint, s.lastConnectDisksOpTime) {
-			continue
+		cdisk := diskMap[endpoint]
+		if cdisk != nil && cdisk.IsOnline() {
+			if s.lastConnectDisksOpTime.IsZero() {
+				continue
+			}
+
+			// An online-disk means its a valid disk but it may be a re-connected disk
+			// we verify that here based on LastConn(), however we make sure to avoid
+			// putting it back into the s.erasureDisks by re-placing the disk again.
+			_, setIndex, _ := cdisk.GetDiskLoc()
+			if setIndex != -1 {
+				// Recently disconnected disks must go to MRF
+				setsJustConnected[setIndex] = cdisk.LastConn().After(s.lastConnectDisksOpTime)
+				continue
+			}
 		}
+
 		wg.Add(1)
 		go func(endpoint Endpoint) {
 			defer wg.Done()
@@ -222,7 +221,6 @@ func (s *erasureSets) connectDisks() {
 			if err != nil {
 				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
 					globalBackgroundHealState.pushHealLocalDisks(endpoint)
-					logger.Info(fmt.Sprintf("Found unformatted drive %s, attempting to heal...", endpoint))
 				} else {
 					printEndpointError(endpoint, err, true)
 				}
@@ -230,7 +228,6 @@ func (s *erasureSets) connectDisks() {
 			}
 			if disk.IsLocal() && disk.Healing() != nil {
 				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
-				logger.Info(fmt.Sprintf("Found the drive %s that needs healing, attempting to heal...", disk))
 			}
 			s.erasureDisksMu.RLock()
 			setIndex, diskIndex, err := findDiskIndex(s.format, format)
@@ -268,7 +265,7 @@ func (s *erasureSets) connectDisks() {
 				s.erasureDisks[setIndex][diskIndex] = disk
 			}
 			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
-			setsJustConnected[setIndex] = true
+			setsJustConnected[setIndex] = true // disk just went online we treat it is as MRF event
 			s.erasureDisksMu.Unlock()
 		}(endpoint)
 	}
@@ -280,7 +277,7 @@ func (s *erasureSets) connectDisks() {
 			if !justConnected {
 				continue
 			}
-			globalMRFState.newSetReconnected(setIndex, s.poolIndex)
+			globalMRFState.newSetReconnected(s.poolIndex, setIndex)
 		}
 	}()
 }
@@ -508,9 +505,18 @@ func (s *erasureSets) cleanupDeletedObjects(ctx context.Context) {
 			// Reset for the next interval
 			timer.Reset(globalAPIConfig.getDeleteCleanupInterval())

+			var wg sync.WaitGroup
 			for _, set := range s.sets {
-				set.cleanupDeletedObjects(ctx)
+				wg.Add(1)
+				go func(set *erasureObjects) {
+					defer wg.Done()
+					if set == nil {
+						return
+					}
+					set.cleanupDeletedObjects(ctx)
+				}(set)
 			}
+			wg.Wait()
 		}
 	}
 }
@@ -527,9 +533,18 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 			// Reset for the next interval
 			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())

+			var wg sync.WaitGroup
 			for _, set := range s.sets {
-				set.cleanupStaleUploads(ctx, globalAPIConfig.getStaleUploadsExpiry())
+				wg.Add(1)
+				go func(set *erasureObjects) {
+					defer wg.Done()
+					if set == nil {
+						return
+					}
+					set.cleanupStaleUploads(ctx, globalAPIConfig.getStaleUploadsExpiry())
+				}(set)
 			}
+			wg.Wait()
 		}
 	}
 }
@@ -1208,21 +1223,6 @@ func formatsToDrivesInfo(endpoints Endpoints, formats []*formatErasureV3, sErrs
 	return beforeDrives
 }

-// If it is a single node Erasure and all disks are root disks, it is most likely a test setup, else it is a production setup.
-// On a test setup we allow creation of format.json on root disks to help with dev/testing.
-func isTestSetup(infos []DiskInfo, errs []error) bool {
-	rootDiskCount := 0
-	for i := range errs {
-		if errs[i] == nil || errs[i] == errUnformattedDisk {
-			if infos[i].RootDisk {
-				rootDiskCount++
-			}
-		}
-	}
-	// It is a test setup if all disks are root disks in quorum.
-	return rootDiskCount >= len(infos)/2+1
-}
-
 func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []error) {
 	infos := make([]DiskInfo, len(storageDisks))
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -1245,18 +1245,17 @@ func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []er

 // Mark root disks as down so as not to heal them.
 func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
-	var infos []DiskInfo
-	infos, errs = getHealDiskInfos(storageDisks, errs)
-	if !isTestSetup(infos, errs) {
-		for i := range storageDisks {
-			if storageDisks[i] != nil && infos[i].RootDisk {
-				// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
-				// defective drive we should not heal a path on the root disk.
-				logger.Info("Disk `%s` the same as the system root disk.\n"+
-					"Disk will not be used. Please supply a separate disk and restart the server.",
-					storageDisks[i].String())
-				storageDisks[i] = nil
-			}
+	if globalIsCICD {
+		// Do nothing
+		return
+	}
+	infos, _ := getHealDiskInfos(storageDisks, errs)
+	for i := range storageDisks {
+		if storageDisks[i] != nil && infos[i].RootDisk {
+			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
+			// defective drive we should not heal a path on the root disk.
+			logger.LogIf(GlobalContext, fmt.Errorf("Disk `%s` is part of root disk, will not be used", storageDisks[i]))
+			storageDisks[i] = nil
 		}
 	}
 }
@@ -29,7 +29,6 @@ import (

 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bpool"
-	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -353,7 +352,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 	// Collect disks we can use.
 	disks, healing := er.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		logger.Info(color.Green("data-scanner:") + " all disks are offline or being healed, skipping scanner")
+		logger.LogIf(ctx, errors.New("data-scanner: all disks are offline or being healed, skipping scanner cycle"))
 		return nil
 	}

@@ -19,8 +19,6 @@ package cmd

 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -124,6 +122,13 @@ type formatErasureV3 struct {
 	} `json:"xl"`
 }

+func (f *formatErasureV3) Drives() (drives int) {
+	for _, set := range f.Erasure.Sets {
+		drives += len(set)
+	}
+	return drives
+}
+
 func (f *formatErasureV3) Clone() *formatErasureV3 {
 	b, err := json.Marshal(f)
 	if err != nil {
@@ -545,43 +550,36 @@ func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []Stora

 // Get backend Erasure format in quorum `format.json`.
 func getFormatErasureInQuorum(formats []*formatErasureV3) (*formatErasureV3, error) {
-	formatHashes := make([]string, len(formats))
-	for i, format := range formats {
+	formatCountMap := make(map[int]int, len(formats))
+	for _, format := range formats {
 		if format == nil {
 			continue
 		}
-		h := sha256.New()
-		for _, set := range format.Erasure.Sets {
-			for _, diskID := range set {
-				h.Write([]byte(diskID))
-			}
-		}
-		formatHashes[i] = hex.EncodeToString(h.Sum(nil))
+		formatCountMap[format.Drives()]++
 	}

-	formatCountMap := make(map[string]int)
-	for _, hash := range formatHashes {
-		if hash == "" {
-			continue
-		}
-		formatCountMap[hash]++
-	}
-
-	maxHash := ""
+	maxDrives := 0
 	maxCount := 0
-	for hash, count := range formatCountMap {
+	for drives, count := range formatCountMap {
 		if count > maxCount {
 			maxCount = count
-			maxHash = hash
+			maxDrives = drives
 		}
 	}

+	if maxDrives == 0 {
+		return nil, errErasureReadQuorum
+	}
+
 	if maxCount < len(formats)/2 {
 		return nil, errErasureReadQuorum
 	}

-	for i, hash := range formatHashes {
-		if hash == maxHash {
+	for i, format := range formats {
+		if format == nil {
+			continue
+		}
+		if format.Drives() == maxDrives {
 			format := formats[i].Clone()
 			format.Erasure.This = ""
 			return format, nil
@@ -624,43 +622,6 @@ func formatErasureV3Check(reference *formatErasureV3, format *formatErasureV3) e
 	return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.Erasure.Sets)
 }

-// Initializes meta volume only on local storage disks.
-func initErasureMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*formatErasureV3) error {
-	// Compute the local disks eligible for meta volumes (re)initialization
-	disksToInit := make([]StorageAPI, 0, len(storageDisks))
-	for index := range storageDisks {
-		if formats[index] == nil || storageDisks[index] == nil || !storageDisks[index].IsLocal() {
-			// Ignore create meta volume on disks which are not found or not local.
-			continue
-		}
-		disksToInit = append(disksToInit, storageDisks[index])
-	}
-
-	// Initialize errs to collect errors inside go-routine.
-	g := errgroup.WithNErrs(len(disksToInit))
-
-	// Initialize all disks in parallel.
-	for index := range disksToInit {
-		// Initialize a new index variable in each loop so each
-		// goroutine will return its own instance of index variable.
-		index := index
-		g.Go(func() error {
-			return makeFormatErasureMetaVolumes(disksToInit[index])
-		}, index)
-	}
-
-	// Return upon first error.
-	for _, err := range g.Wait() {
-		if err == nil {
-			continue
-		}
-		return toObjectErr(err, minioMetaBucket)
-	}
-
-	// Return success here.
-	return nil
-}
-
 // saveFormatErasureAll - populates `format.json` on disks in its order.
 func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatErasureV3) error {
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -18,6 +18,8 @@
 package cmd

 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"io/ioutil"
@@ -54,10 +56,6 @@ func TestFixFormatV3(t *testing.T) {
 		formats[j] = newFormat
 	}

-	if err = initErasureMetaVolumesInLocalDisks(storageDisks, formats); err != nil {
-		t.Fatal(err)
-	}
-
 	formats[1] = nil
 	expThis := formats[2].Erasure.This
 	formats[2].Erasure.This = ""
@@ -342,6 +340,102 @@ func TestGetFormatErasureInQuorumCheck(t *testing.T) {
 	}
 }

+// Get backend Erasure format in quorum `format.json`.
+func getFormatErasureInQuorumOld(formats []*formatErasureV3) (*formatErasureV3, error) {
+	formatHashes := make([]string, len(formats))
+	for i, format := range formats {
+		if format == nil {
+			continue
+		}
+		h := sha256.New()
+		for _, set := range format.Erasure.Sets {
+			for _, diskID := range set {
+				h.Write([]byte(diskID))
+			}
+		}
+		formatHashes[i] = hex.EncodeToString(h.Sum(nil))
+	}
+
+	formatCountMap := make(map[string]int)
+	for _, hash := range formatHashes {
+		if hash == "" {
+			continue
+		}
+		formatCountMap[hash]++
+	}
+
+	maxHash := ""
+	maxCount := 0
+	for hash, count := range formatCountMap {
+		if count > maxCount {
+			maxCount = count
+			maxHash = hash
+		}
+	}
+
+	if maxCount < len(formats)/2 {
+		return nil, errErasureReadQuorum
+	}
+
+	for i, hash := range formatHashes {
+		if hash == maxHash {
+			format := formats[i].Clone()
+			format.Erasure.This = ""
+			return format, nil
+		}
+	}
+
+	return nil, errErasureReadQuorum
+}
+
+func BenchmarkGetFormatErasureInQuorumOld(b *testing.B) {
+	setCount := 200
+	setDriveCount := 15
+
+	format := newFormatErasureV3(setCount, setDriveCount)
+	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
+	formats := make([]*formatErasureV3, 15*200)
+
+	for i := 0; i < setCount; i++ {
+		for j := 0; j < setDriveCount; j++ {
+			newFormat := format.Clone()
+			newFormat.Erasure.This = format.Erasure.Sets[i][j]
+			formats[i*setDriveCount+j] = newFormat
+		}
+	}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = getFormatErasureInQuorumOld(formats)
+	}
+}
+
+func BenchmarkGetFormatErasureInQuorum(b *testing.B) {
+	setCount := 200
+	setDriveCount := 15
+
+	format := newFormatErasureV3(setCount, setDriveCount)
+	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
+	formats := make([]*formatErasureV3, 15*200)
+
+	for i := 0; i < setCount; i++ {
+		for j := 0; j < setDriveCount; j++ {
+			newFormat := format.Clone()
+			newFormat.Erasure.This = format.Erasure.Sets[i][j]
+			formats[i*setDriveCount+j] = newFormat
+		}
+	}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = getFormatErasureInQuorum(formats)
+	}
+}
+
 // Tests formatErasureGetDeploymentID()
 func TestGetErasureID(t *testing.T) {
 	setCount := 2
@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -35,6 +36,11 @@ import (
 	"github.com/minio/pkg/trie"
 )

+const (
+	bgAppendsDirName         = "bg-appends"
+	bgAppendsCleanupInterval = 10 * time.Minute
+)
+
 // Returns EXPORT/.minio.sys/multipart/SHA256/UPLOADID
 func (fs *FSObjects) getUploadIDDir(bucket, object, uploadID string) string {
 	return pathJoin(fs.fsPath, minioMetaMultipartBucket, getSHA256Hash([]byte(pathJoin(bucket, object))), uploadID)
@@ -74,7 +80,7 @@ func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploa
 	file := fs.appendFileMap[uploadID]
 	if file == nil {
 		file = &fsAppendFile{
-			filePath: pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, fmt.Sprintf("%s.%s", uploadID, mustGetUUID())),
+			filePath: pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, bgAppendsDirName, fmt.Sprintf("%s.%s", uploadID, mustGetUUID())),
 		}
 		fs.appendFileMap[uploadID] = file
 	}
@@ -643,7 +649,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}

 	appendFallback := true // In case background-append did not append the required parts.
-	appendFilePath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, fmt.Sprintf("%s.%s", uploadID, mustGetUUID()))
+	appendFilePath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, "bg-appends", fmt.Sprintf("%s.%s", uploadID, mustGetUUID()))

 	// Most of the times appendFile would already be fully appended by now. We call fs.backgroundAppend()
 	// to take care of the following corner case:
@@ -843,58 +849,99 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 	return nil
 }

+// Return all uploads IDs with full path of each upload-id directory.
+// Do not return an error as this is a lazy operation
+func (fs *FSObjects) getAllUploadIDs(ctx context.Context) (result map[string]string) {
+	result = make(map[string]string)
+
+	entries, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket))
+	if err != nil {
+		return
+	}
+	for _, entry := range entries {
+		uploadIDs, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
+		if err != nil {
+			continue
+		}
+		// Remove the trailing slash separator
+		for i := range uploadIDs {
+			uploadID := strings.TrimSuffix(uploadIDs[i], SlashSeparator)
+			result[uploadID] = pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID)
+		}
+	}
+	return
+}
+
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
 func (fs *FSObjects) cleanupStaleUploads(ctx context.Context) {
-	timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval())
-	defer timer.Stop()
+	expiryUploadsTimer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval())
+	defer expiryUploadsTimer.Stop()
+
+	bgAppendTmpCleaner := time.NewTimer(bgAppendsCleanupInterval)
+	defer bgAppendTmpCleaner.Stop()

 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-timer.C:
-			// Reset for the next interval
-			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
+		case <-bgAppendTmpCleaner.C:
+			bgAppendTmpCleaner.Reset(bgAppendsCleanupInterval)

-			expiry := globalAPIConfig.getStaleUploadsExpiry()
+			foundUploadIDs := fs.getAllUploadIDs(ctx)

-			now := time.Now()
-			entries, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket))
+			// Remove background append map from the memory
+			fs.appendFileMapMu.Lock()
+			for uploadID := range fs.appendFileMap {
+				_, ok := foundUploadIDs[uploadID]
+				if !ok {
+					delete(fs.appendFileMap, uploadID)
+				}
+			}
+			fs.appendFileMapMu.Unlock()
+
+			// Remove background appends file from the disk
+			bgAppendsDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, bgAppendsDirName)
+			entries, err := readDir(bgAppendsDir)
 			if err != nil {
-				continue
+				break
 			}
 			for _, entry := range entries {
-				uploadIDs, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
+				uploadID := strings.Split(entry, ".")[0]
+				_, ok := foundUploadIDs[uploadID]
+				if !ok {
+					fsRemoveFile(ctx, pathJoin(bgAppendsDir, entry))
+				}
+			}
+
+		case <-expiryUploadsTimer.C:
+			// Reset for the next interval
+			expiryUploadsTimer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
+
+			expiry := globalAPIConfig.getStaleUploadsExpiry()
+			now := time.Now()
+
+			uploadIDs := fs.getAllUploadIDs(ctx)
+
+			for uploadID, path := range uploadIDs {
+				fi, err := fsStatDir(ctx, path)
 				if err != nil {
 					continue
 				}
+				if now.Sub(fi.ModTime()) > expiry {
+					fsRemoveAll(ctx, path)
+					// Remove upload ID parent directory if empty
+					fsRemoveDir(ctx, filepath.Base(path))

-				// Remove the trailing slash separator
-				for i := range uploadIDs {
-					uploadIDs[i] = strings.TrimSuffix(uploadIDs[i], SlashSeparator)
-				}
-
-				for _, uploadID := range uploadIDs {
-					fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
-					if err != nil {
-						continue
-					}
-					if now.Sub(fi.ModTime()) > expiry {
-						fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
-						// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
-						fsRemoveDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
-
-						// Remove uploadID from the append file map and its corresponding temporary file
-						fs.appendFileMapMu.Lock()
-						bgAppend, ok := fs.appendFileMap[uploadID]
-						if ok {
-							_ = fsRemoveFile(ctx, bgAppend.filePath)
-							delete(fs.appendFileMap, uploadID)
-						}
-						fs.appendFileMapMu.Unlock()
+					// Remove uploadID from the append file map and its corresponding temporary file
+					fs.appendFileMapMu.Lock()
+					bgAppend, ok := fs.appendFileMap[uploadID]
+					if ok {
+						_ = fsRemoveFile(ctx, bgAppend.filePath)
+						delete(fs.appendFileMap, uploadID)
 					}
+					fs.appendFileMapMu.Unlock()
 				}
 			}
 		}
@@ -106,6 +106,10 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 		return err
 	}

+	if err := os.MkdirAll(pathJoin(metaTmpPath, bgAppendsDirName), 0o777); err != nil {
+		return err
+	}
+
 	if err := os.MkdirAll(pathJoin(fsPath, dataUsageBucket), 0o777); err != nil {
 		return err
 	}
@@ -329,7 +333,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 		logger.LogIf(ctx, totalCache.save(ctx, fs, dataUsageCacheName))
 		cloned := totalCache.clone()
 		updates <- cloned.dui(dataUsageRoot, buckets)
-		enforceFIFOQuotaBucket(ctx, fs, b.Name, cloned.bucketUsageInfo(b.Name))
+
 	}

 	return nil
@@ -1260,9 +1264,23 @@ func (fs *FSObjects) ListObjectVersions(ctx context.Context, bucket, prefix, mar

 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
-func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool,
-		fs.listDirFactory(), fs.isLeaf, fs.isLeafDir, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
+func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+	// listObjects may in rare cases not be able to find any valid results.
+	// Therefore, it cannot set a NextMarker.
+	// In that case we retry the operation, but we add a
+	// max limit, so we never end up in an infinite loop.
+	tries := 50
+	for {
+		loi, err = listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool,
+			fs.listDirFactory(), fs.isLeaf, fs.isLeafDir, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
+		if err != nil {
+			return loi, err
+		}
+		if !loi.IsTruncated || loi.NextMarker != "" || tries == 0 {
+			return loi, nil
+		}
+		tries--
+	}
 }

 // GetObjectTags - get object tags from an existing object
@@ -335,6 +335,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
 	switch minioErr.StatusCode {
 	case http.StatusMethodNotAllowed:
 		err = toObjectErr(errMethodNotAllowed, bucket, object)
+	case http.StatusBadGateway:
+		return BackendDown{Err: err.Error()}
 	}
 	return err
 }
@@ -31,6 +31,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/color"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -160,7 +161,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
-	logger.SetDeploymentID(globalDeploymentID)
+	xhttp.SetDeploymentID(globalDeploymentID)

 	if gw == nil {
 		logger.FatalIf(errUnexpected, "Gateway implementation not initialized")
@@ -175,7 +176,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {

 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(GlobalContext)
-	logger.AddTarget(globalConsoleSys)
+	logger.AddSystemTarget(globalConsoleSys)

 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
@@ -251,6 +252,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Add API router.
 	registerAPIRouter(router)

+	// Enable bucket forwarding handler only if bucket federation is enabled.
+	if globalDNSConfig != nil && globalBucketFederation {
+		globalHandlers = append(globalHandlers, setBucketForwardingHandler)
+	}
+
 	// Use all the middlewares
 	router.Use(globalHandlers...)

@@ -291,22 +297,24 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	newObject = NewGatewayLayerWithLocker(newObject)

 	// Calls all New() for all sub-systems.
-	newAllSubsystems()
+	initAllSubsystems()

 	// Once endpoints are finalized, initialize the new object api in safe mode.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()

+	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
+
 	if gatewayName == NASBackendGateway {
 		buckets, err := newObject.ListBuckets(GlobalContext)
 		if err != nil {
 			logger.Fatal(err, "Unable to list buckets")
 		}
-		logger.FatalIf(globalNotificationSys.Init(GlobalContext, buckets, newObject), "Unable to initialize notification system")
-	}
+		logger.FatalIf(globalBucketMetadataSys.Init(GlobalContext, buckets, newObject), "Unable to initialize bucket metadata")

-	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
+		logger.FatalIf(globalNotificationSys.InitBucketTargets(GlobalContext, newObject), "Unable to initialize bucket targets for notification system")
+	}

 	if globalCacheConfig.Enabled {
 		// initialize the new disk cache objects.
@@ -368,5 +376,15 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.Info("======")
 	}

+	// TODO: remove the following line by June 1st.
+	logger.Info(
+		color.RedBold(`
+===================================================================================
+**** WARNING: MinIO Gateway will be removed by June 1st from MinIO repository *****
+
+Please read https://github.com/minio/minio/issues/14331
+===================================================================================
+`))
+
 	<-globalOSSignalCh
 }
@@ -22,6 +22,7 @@ import (
 	"strings"

 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/logger"
 )

 // Prints the formatted startup message.
@@ -45,7 +46,7 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 	if globalMinioConsolePortAuto && globalBrowserEnabled {
 		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
 			globalMinioConsolePort)
-		logStartupMessage(color.RedBold(msg))
+		logger.Info(color.RedBold(msg))
 	}
 }

@@ -57,19 +58,19 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")

 	// Colorize the message and print.
-	logStartupMessage(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
+	logger.Info(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 	}
 	printEventNotifiers()

 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
-		logStartupMessage(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
+		logger.Info(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
 		if color.IsTerminal() && !globalCLIContext.Anonymous {
-			logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-			logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		}
 	}
 }
@@ -371,7 +371,8 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 // is obtained from centralized etcd configuration service.
 func setBucketForwardingHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
+		if globalDNSConfig == nil || !globalBucketFederation ||
+			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
 			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {
 			h.ServeHTTP(w, r)
 			return
@@ -330,8 +330,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		default:
 			tracker.bucketDone(bucket)
 			logger.LogIf(ctx, tracker.update(ctx))
-			logger.Info("Healing bucket %s content on %s erasure set complete",
-				bucket, humanize.Ordinal(tracker.SetIndex+1))
 		}
 	}
 	tracker.Object = ""
@@ -190,7 +190,7 @@ var (
 	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
 	// healthcheck readiness deadlines and cors settings.
-	globalAPIConfig = apiConfig{listQuorum: 3}
+	globalAPIConfig = apiConfig{listQuorum: -1}

 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -345,6 +345,11 @@ var (
 	globalIsCICD bool

 	globalRootDiskThreshold uint64
+
+	// Used for collecting stats for netperf
+	globalNetPerfMinDuration = time.Second * 10
+	globalNetPerfRX          netPerfRX
+
 	// Add new variable global values here.
 )

@@ -50,6 +50,7 @@ type apiConfig struct {
 	staleUploadsCleanupInterval time.Duration
 	deleteCleanupInterval       time.Duration
 	disableODirect              bool
+	gzipObjects                 bool
 }

 const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
@@ -152,6 +153,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
 	t.disableODirect = cfg.DisableODirect
+	t.gzipObjects = cfg.GzipObjects
 }

 func (t *apiConfig) isDisableODirect() bool {
@@ -161,6 +163,13 @@ func (t *apiConfig) isDisableODirect() bool {
 	return t.disableODirect
 }

+func (t *apiConfig) shouldGzipObjects() bool {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.gzipObjects
+}
+
 func (t *apiConfig) getListQuorum() int {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
@@ -235,6 +244,8 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 // maxClients throttles the S3 API calls
 func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		globalHTTPStats.incS3RequestsIncoming()
+
 		if val := globalServiceFreeze.Load(); val != nil {
 			if unlock, ok := val.(chan struct{}); ok && unlock != nil {
 				// Wait until unfrozen.
@@ -138,11 +138,13 @@ func (stats *HTTPAPIStats) Load() map[string]int {
 // HTTPStats holds statistics information about
 // HTTP requests made by all clients
 type HTTPStats struct {
+	s3RequestsInQueue       int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	_                       int32 // For 64 bits alignment
+	s3RequestsIncoming      uint64
 	rejectedRequestsAuth    uint64
 	rejectedRequestsTime    uint64
 	rejectedRequestsHeader  uint64
 	rejectedRequestsInvalid uint64
-	s3RequestsInQueue       int32
 	currentS3Requests       HTTPAPIStats
 	totalS3Requests         HTTPAPIStats
 	totalS3Errors           HTTPAPIStats
@@ -153,9 +155,15 @@ func (st *HTTPStats) addRequestsInQueue(i int32) {
 	atomic.AddInt32(&st.s3RequestsInQueue, i)
 }

+func (st *HTTPStats) incS3RequestsIncoming() {
+	// Golang automatically resets to zero if this overflows
+	atomic.AddUint64(&st.s3RequestsIncoming, 1)
+}
+
 // Converts http stats into struct to be sent back to the client.
 func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	serverStats := ServerHTTPStats{}
+	serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0)
 	serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue)
 	serverStats.TotalS3RejectedAuth = atomic.LoadUint64(&st.rejectedRequestsAuth)
 	serverStats.TotalS3RejectedTime = atomic.LoadUint64(&st.rejectedRequestsTime)
@@ -174,3 +174,26 @@ func (h *HTTPRangeSpec) String(resourceSize int64) string {
 	}
 	return fmt.Sprintf("%d-%d", off, off+length-1)
 }
+
+// ToHeader returns the Range header value.
+func (h *HTTPRangeSpec) ToHeader() (string, error) {
+	if h == nil {
+		return "", nil
+	}
+	start := strconv.Itoa(int(h.Start))
+	end := strconv.Itoa(int(h.End))
+	switch {
+	case h.Start >= 0 && h.End >= 0:
+		if h.Start > h.End {
+			return "", errInvalidRange
+		}
+	case h.IsSuffixLength:
+		end = strconv.Itoa(int(h.Start * -1))
+		start = ""
+	case h.Start > -1:
+		end = ""
+	default:
+		return "", fmt.Errorf("does not have valid range value")
+	}
+	return fmt.Sprintf("bytes=%s-%s", start, end), nil
+}
@@ -101,3 +101,46 @@ func TestHTTPRequestRangeSpec(t *testing.T) {
 		t.Errorf("Case %d: Expected errInvalidRange but: %v %v %d %d %v", i, rs, err1, o, l, err2)
 	}
 }
+
+func TestHTTPRequestRangeToHeader(t *testing.T) {
+	validRangeSpecs := []struct {
+		spec        string
+		errExpected bool
+	}{
+		{"bytes=0-", false},
+		{"bytes=1-", false},
+
+		{"bytes=0-9", false},
+		{"bytes=1-10", false},
+		{"bytes=1-1", false},
+		{"bytes=2-5", false},
+
+		{"bytes=-5", false},
+		{"bytes=-1", false},
+		{"bytes=-1000", false},
+		{"bytes=", true},
+		{"bytes= ", true},
+		{"byte=", true},
+		{"bytes=A-B", true},
+		{"bytes=1-B", true},
+		{"bytes=B-1", true},
+		{"bytes=-1-1", true},
+	}
+	for i, testCase := range validRangeSpecs {
+		rs, err := parseRequestRangeSpec(testCase.spec)
+		if err != nil {
+			if !testCase.errExpected || err == nil && testCase.errExpected {
+				t.Errorf("unexpected err: %v", err)
+			}
+			continue
+		}
+		h, err := rs.ToHeader()
+		if err != nil && !testCase.errExpected || err == nil && testCase.errExpected {
+			t.Errorf("expected error with invalid range: %v", err)
+		}
+		if h != testCase.spec {
+			t.Errorf("Case %d: translated to incorrect header: %s expected: %s",
+				i, h, testCase.spec)
+		}
+	}
+}
@@ -1450,19 +1450,34 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error

 // GetAllParentUsers - returns all distinct "parent-users" associated with STS or service
 // credentials.
-func (store *IAMStoreSys) GetAllParentUsers() []string {
+func (store *IAMStoreSys) GetAllParentUsers() map[string]string {
 	cache := store.rlock()
 	defer store.runlock()

-	res := set.NewStringSet()
+	res := map[string]string{}
 	for _, cred := range cache.iamUsersMap {
-		if cred.IsServiceAccount() || cred.IsTemp() {
+		if (cred.IsServiceAccount() || cred.IsTemp()) && cred.SessionToken != "" {
 			parentUser := cred.ParentUser
-			if cred.SessionToken != "" {
-				claims, err := getClaimsFromToken(cred.SessionToken)
+
+			var (
+				err    error
+				claims map[string]interface{}
+			)
+
+			if cred.IsServiceAccount() {
+				claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, cred.SecretKey)
 				if err != nil {
-					continue
+					claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
 				}
+			} else if cred.IsTemp() {
+				claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
+			}
+
+			if err != nil {
+				continue
+			}
+
+			if len(claims) > 0 {
 				if v, ok := claims[subClaim]; ok {
 					subFromToken, ok := v.(string)
 					if ok {
@@ -1470,11 +1485,18 @@ func (store *IAMStoreSys) GetAllParentUsers() []string {
 					}
 				}
 			}
-			res.Add(parentUser)
+
+			if parentUser == "" {
+				continue
+			}
+
+			if _, ok := res[parentUser]; !ok {
+				res[parentUser] = cred.ParentUser
+			}
 		}
 	}

-	return res.ToSlice()
+	return res
 }

 // SetUserStatus - sets current user status.
@@ -1559,6 +1581,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errNoSuchServiceAccount
 	}

+	currentSecretKey := cr.SecretKey
 	if opts.secretKey != "" {
 		if !auth.IsSecretKeyValid(opts.secretKey) {
 			return auth.ErrInvalidSecretKeyLength
@@ -1580,20 +1603,21 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errors.New("unknown account status value")
 	}

-	if opts.sessionPolicy != nil {
-		m, err := getClaimsFromToken(cr.SessionToken)
-		if err != nil {
-			return fmt.Errorf("unable to get svc acc claims: %v", err)
-		}
+	m, err := getClaimsFromTokenWithSecret(cr.SessionToken, currentSecretKey)
+	if err != nil {
+		return fmt.Errorf("unable to get svc acc claims: %v", err)
+	}

-		err = opts.sessionPolicy.Validate()
-		if err != nil {
+	if opts.sessionPolicy != nil {
+		if err := opts.sessionPolicy.Validate(); err != nil {
 			return err
 		}
+
 		policyBuf, err := json.Marshal(opts.sessionPolicy)
 		if err != nil {
 			return err
 		}
+
 		if len(policyBuf) > 16*humanize.KiByte {
 			return fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
@@ -1601,10 +1625,11 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		// Overwrite session policy claims.
 		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
 		m[iamPolicyClaimNameSA()] = "embedded-policy"
-		cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, globalActiveCred.SecretKey)
-		if err != nil {
-			return err
-		}
+	}
+
+	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
+	if err != nil {
+		return err
 	}

 	u := newUserIdentity(cr)
@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -66,7 +67,6 @@ type IAMSys struct {
 	sync.Mutex

 	iamRefreshInterval time.Duration
-	notificationSys    *NotificationSys

 	usersSysType UsersSysType

@@ -198,12 +198,11 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 }

 // Init - initializes config system by reading entries from config/iam
-func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, nSys *NotificationSys, iamRefreshInterval time.Duration) {
+func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
 	sys.Lock()
 	defer sys.Unlock()

 	sys.iamRefreshInterval = iamRefreshInterval
-	sys.notificationSys = nSys

 	// Initialize IAM store
 	sys.initStore(objAPI, etcdClient)
@@ -351,7 +350,7 @@ func (sys *IAMSys) printIAMRoles() {
 		msgs = append(msgs, color.Bold(arn))
 	}

-	logStartupMessage(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " ")))
+	logger.Info(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " ")))
 }

 // HasWatcher - returns if the IAM system has a watcher to be notified of
@@ -466,7 +465,7 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 	}

 	// Notify all other MinIO peers to delete policy
-	for _, nerr := range sys.notificationSys.DeletePolicy(policyName) {
+	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -524,7 +523,7 @@ func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy

 	if !sys.HasWatcher() {
 		// Notify all other MinIO peers to reload policy
-		for _, nerr := range sys.notificationSys.LoadPolicy(policyName) {
+		for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -546,7 +545,7 @@ func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers

 	// Notify all other MinIO peers to delete user.
 	if notifyPeers && !sys.HasWatcher() {
-		for _, nerr := range sys.notificationSys.DeleteUser(accessKey) {
+		for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -572,7 +571,7 @@ func (sys *IAMSys) CurrentPolicies(policyName string) string {
 func (sys *IAMSys) notifyForUser(ctx context.Context, accessKey string, isTemp bool) {
 	// Notify all other MinIO peers to reload user.
 	if !sys.HasWatcher() {
-		for _, nerr := range sys.notificationSys.LoadUser(accessKey, isTemp) {
+		for _, nerr := range globalNotificationSys.LoadUser(accessKey, isTemp) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -735,7 +734,7 @@ func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status m
 func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string) {
 	// Notify all other Minio peers to reload the service account
 	if !sys.HasWatcher() {
-		for _, nerr := range sys.notificationSys.LoadServiceAccount(accessKey) {
+		for _, nerr := range globalNotificationSys.LoadServiceAccount(accessKey) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -801,14 +800,17 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		}
 	}

-	var cred auth.Credentials
-
+	var accessKey, secretKey string
 	var err error
 	if len(opts.accessKey) > 0 {
-		cred, err = auth.CreateNewCredentialsWithMetadata(opts.accessKey, opts.secretKey, m, globalActiveCred.SecretKey)
+		accessKey, secretKey = opts.accessKey, opts.secretKey
 	} else {
-		cred, err = auth.GetNewCredentialsWithMetadata(m, globalActiveCred.SecretKey)
+		accessKey, secretKey, err = auth.GenerateCredentials()
+		if err != nil {
+			return auth.Credentials{}, err
+		}
 	}
+	cred, err := auth.CreateNewCredentialsWithMetadata(accessKey, secretKey, m, secretKey)
 	if err != nil {
 		return auth.Credentials{}, err
 	}
@@ -882,9 +884,12 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut

 	var embeddedPolicy *iampolicy.Policy

-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return auth.Credentials{}, nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return auth.Credentials{}, nil, err
+		}
 	}
 	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
 	sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
@@ -917,9 +922,12 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 		return nil, errNoSuchServiceAccount
 	}

-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return jwtClaims.Map(), nil
 }
@@ -940,7 +948,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 	}

 	if notifyPeers && !sys.HasWatcher() {
-		for _, nerr := range sys.notificationSys.DeleteServiceAccount(accessKey) {
+		for _, nerr := range globalNotificationSys.DeleteServiceAccount(accessKey) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -1005,7 +1013,7 @@ func (sys *IAMSys) SetUserSecretKey(ctx context.Context, accessKey string, secre
 func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var expiredUsers []string
-	for _, parentUser := range parentUsers {
+	for parentUser, expiredUser := range parentUsers {
 		u, err := globalOpenIDConfig.LookupUser(parentUser)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
@@ -1014,7 +1022,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 		// If user is set to "disabled", we will remove them
 		// subsequently.
 		if !u.Enabled {
-			expiredUsers = append(expiredUsers, parentUser)
+			expiredUsers = append(expiredUsers, expiredUser)
 		}
 	}

@@ -1027,12 +1035,12 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var allDistNames []string
-	for _, parentUser := range parentUsers {
+	for parentUser, expiredUser := range parentUsers {
 		if !globalLDAPConfig.IsLDAPUserDN(parentUser) {
 			continue
 		}

-		allDistNames = append(allDistNames, parentUser)
+		allDistNames = append(allDistNames, expiredUser)
 	}

 	expiredUsers, err := globalLDAPConfig.GetNonEligibleUserDistNames(allDistNames)
@@ -1065,12 +1073,28 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 		if _, ok := parentUserToCredsMap[cred.ParentUser]; !ok {
 			// Try to find the ldapUsername for this
 			// parentUser by extracting JWT claims
-			jwtClaims, err := auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
-			if err != nil {
-				// skip this cred - session token seems
-				// invalid
+			var (
+				jwtClaims *jwt.MapClaims
+				err       error
+			)
+
+			if cred.SessionToken == "" {
 				continue
 			}
+
+			if cred.IsServiceAccount() {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, cred.SecretKey)
+				if err != nil {
+					jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+				}
+			} else {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+			}
+			if err != nil {
+				// skip this cred - session token seems invalid
+				continue
+			}
+
 			ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
 			if !ok {
 				// skip this cred - we dont have the
@@ -1149,7 +1173,7 @@ func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Cre
 // Notify all other MinIO peers to load group.
 func (sys *IAMSys) notifyForGroup(ctx context.Context, group string) {
 	if !sys.HasWatcher() {
-		for _, nerr := range sys.notificationSys.LoadGroup(group) {
+		for _, nerr := range globalNotificationSys.LoadGroup(group) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -1256,7 +1280,7 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup

 	// Notify all other MinIO peers to reload policy
 	if !sys.HasWatcher() {
-		for _, nerr := range sys.notificationSys.LoadPolicyMapping(name, isGroup) {
+		for _, nerr := range globalNotificationSys.LoadPolicyMapping(name, isGroup) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -132,13 +132,13 @@ func authenticateURL(accessKey, secretKey string) (string, error) {
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
-func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, bool, error) {
+func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
 	token, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req)
 	if err != nil {
 		if err == jwtreq.ErrNoTokenInRequest {
-			return nil, false, errNoAuthToken
+			return nil, nil, false, errNoAuthToken
 		}
-		return nil, false, err
+		return nil, nil, false, err
 	}
 	claims := xjwt.NewMapClaims()
 	if err := xjwt.ParseWithClaims(token, claims, func(claims *xjwt.MapClaims) ([]byte, error) {
@@ -151,20 +151,21 @@ func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, bool, error) {
 		}
 		return []byte(cred.SecretKey), nil
 	}); err != nil {
-		return claims, false, errAuthentication
+		return claims, nil, false, errAuthentication
 	}
 	owner := true
+	var groups []string
 	if globalActiveCred.AccessKey != claims.AccessKey {
 		// Check if the access key is part of users credentials.
 		ucred, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey)
 		if !ok {
-			return nil, false, errInvalidAccessKeyID
+			return nil, nil, false, errInvalidAccessKeyID
 		}

 		// get embedded claims
 		eclaims, s3Err := checkClaimsFromToken(req, ucred)
 		if s3Err != ErrNone {
-			return nil, false, errAuthentication
+			return nil, nil, false, errAuthentication
 		}

 		for k, v := range eclaims {
@@ -177,9 +178,11 @@ func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, bool, error) {
 		} else {
 			owner = globalActiveCred.AccessKey == ucred.ParentUser
 		}
+
+		groups = ucred.Groups
 	}

-	return claims, owner, nil
+	return claims, groups, owner, nil
 }

 // newCachedAuthToken returns a token that is cached up to 15 seconds.
@@ -149,7 +149,7 @@ func TestWebRequestAuthenticate(t *testing.T) {
 	}

 	for i, testCase := range testCases {
-		_, _, gotErr := webRequestAuthenticate(testCase.req)
+		_, _, _, gotErr := metricsRequestAuthenticate(testCase.req)
 		if testCase.expectedErr != gotErr {
 			t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr)
 		}
@@ -349,25 +349,29 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()

-	for k := range l.lockMap {
-		found := false
-		// Since we mutate the value, remove one per loop.
-		for {
-			lris, ok := l.lockMap[k]
-			if !ok {
-				break
-			}
-			for _, lri := range lris {
-				if time.Since(lri.TimeLastRefresh) > interval {
-					l.removeEntry(lri.Name, dsync.LockArgs{Owner: lri.Owner, UID: lri.UID}, &lris)
-					found = true
+	for k, lris := range l.lockMap {
+		modified := false
+		for i := 0; i < len(lris); {
+			lri := &lris[i]
+			if time.Since(lri.TimeLastRefresh) > interval {
+				delete(l.lockUID, formatUUID(lri.UID, lri.idx))
+				if len(lris) == 1 {
+					// Remove the write lock.
+					delete(l.lockMap, lri.Name)
+					modified = false
 					break
 				}
+				modified = true
+				// Remove the appropriate lock.
+				lris = append(lris[:i], lris[i+1:]...)
+				// Check same i
+			} else {
+				// Move to next
+				i++
 			}
-			// We did not find any more to expire.
-			if !found {
-				break
-			}
+		}
+		if modified {
+			l.lockMap[k] = lris
 		}
 	}
 }
@@ -19,9 +19,13 @@ package cmd

 import (
 	"context"
+	"encoding/hex"
+	"fmt"
+	"math/rand"
 	"testing"
 	"time"

+	"github.com/google/uuid"
 	"github.com/minio/minio/internal/dsync"
 )

@@ -254,3 +258,89 @@ func TestLocalLockerUnlock(t *testing.T) {
 		t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), 0, 0)
 	}
 }
+
+func Test_localLocker_expireOldLocksExpire(t *testing.T) {
+	rng := rand.New(rand.NewSource(0))
+	// Numbers of unique locks
+	for _, locks := range []int{100, 1000, 1e6} {
+		if testing.Short() && locks > 100 {
+			continue
+		}
+		t.Run(fmt.Sprintf("%d-locks", locks), func(t *testing.T) {
+			// Number of readers per lock...
+			for _, readers := range []int{1, 10, 100} {
+				if locks > 1000 && readers > 1 {
+					continue
+				}
+				if testing.Short() && readers > 10 {
+					continue
+				}
+				t.Run(fmt.Sprintf("%d-read", readers), func(t *testing.T) {
+					l := newLocker()
+					for i := 0; i < locks; i++ {
+						var tmp [16]byte
+						rng.Read(tmp[:])
+						res := []string{hex.EncodeToString(tmp[:])}
+
+						for i := 0; i < readers; i++ {
+							rng.Read(tmp[:])
+							ok, err := l.RLock(context.Background(), dsync.LockArgs{
+								UID:       uuid.NewString(),
+								Resources: res,
+								Source:    hex.EncodeToString(tmp[:8]),
+								Owner:     hex.EncodeToString(tmp[8:]),
+								Quorum:    0,
+							})
+							if !ok || err != nil {
+								t.Fatal("failed:", err, ok)
+							}
+						}
+					}
+					start := time.Now()
+					l.expireOldLocks(time.Hour)
+					t.Logf("Scan Took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+					if len(l.lockMap) != locks {
+						t.Fatalf("objects deleted, want %d != got %d", locks, len(l.lockMap))
+					}
+					if len(l.lockUID) != locks*readers {
+						t.Fatalf("objects deleted, want %d != got %d", locks*readers, len(l.lockUID))
+					}
+
+					// Expire 50%
+					expired := time.Now().Add(-time.Hour * 2)
+					for _, v := range l.lockMap {
+						for i := range v {
+							if rng.Intn(2) == 0 {
+								v[i].TimeLastRefresh = expired
+							}
+						}
+					}
+					start = time.Now()
+					l.expireOldLocks(time.Hour)
+					t.Logf("Expire 50%% took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+
+					if len(l.lockUID) == locks*readers {
+						t.Fatalf("objects uids all remain, unlikely")
+					}
+					if len(l.lockMap) == 0 {
+						t.Fatalf("objects all deleted, 0 remains")
+					}
+					if len(l.lockUID) == 0 {
+						t.Fatalf("objects uids all deleted, 0 remains")
+					}
+
+					start = time.Now()
+					l.expireOldLocks(-time.Minute)
+					t.Logf("Expire rest took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap))
+
+					if len(l.lockMap) != 0 {
+						t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockMap))
+					}
+					if len(l.lockUID) != 0 {
+						t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockUID))
+					}
+				})
+			}
+		})
+	}
+}
@@ -44,7 +44,7 @@ var GlobalFlags = []cli.Flag{
 	},
 	cli.BoolFlag{
 		Name:  "quiet",
-		Usage: "disable startup information",
+		Usage: "disable startup and info messages",
 	},
 	cli.BoolFlag{
 		Name:  "anonymous",
@@ -52,7 +52,7 @@ var GlobalFlags = []cli.Flag{
 	},
 	cli.BoolFlag{
 		Name:  "json",
-		Usage: "output server logs and startup information in json format",
+		Usage: "output logs in JSON format",
 	},
 	// Deprecated flag, so its hidden now, existing deployments will keep working.
 	cli.BoolFlag{
@@ -191,6 +191,16 @@ func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 		}, nil
 	}
 	if e.cached != nil {
+		if len(e.cached.versions) == 0 {
+			// This special case is needed to handle xlMeta.versions == 0
+			return FileInfo{
+				Volume:   bucket,
+				Name:     e.name,
+				Deleted:  true,
+				IsLatest: true,
+				ModTime:  timeSentinel1970,
+			}, nil
+		}
 		return e.cached.ToFileInfo(bucket, e.name, "")
 	}
 	return getFileInfo(e.metadata, bucket, e.name, "", false)
@@ -379,11 +389,11 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa

 // firstFound returns the first found and the number of set entries.
 func (m metaCacheEntries) firstFound() (first *metaCacheEntry, n int) {
-	for _, entry := range m {
+	for i, entry := range m {
 		if entry.name != "" {
 			n++
 			if first == nil {
-				first = &entry
+				first = &m[i]
 			}
 		}
 	}
--- a/Show More
+++ b/Show More