fix: doc links and delete replication permissions enforcement (#11285 )

fix: use common logging implementation for DNSCache (#11284 )
fix broken URL to k8s operator
2026-11-04 23:02:04 +03:00 · 2021-01-15 15:22:55 -08:00 · 2021-01-15 14:04:56 -08:00 · 2021-01-15 11:55:36 -08:00 · 2021-01-15 11:54:05 -08:00 · 2021-01-14 22:58:53 -08:00
159 changed files with 2961 additions and 1443 deletions
@@ -10,9 +10,10 @@
 ## Types of changes
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
+- [ ] Optimization (provides speedup with no functional changes)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)

 ## Checklist:
 - [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
- [ ] Documentation needed
- [ ] Unit tests needed
+- [ ] Documentation updated
+- [ ] Unit tests added/updated
@@ -39,7 +39,7 @@ jobs:
          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
          sudo apt-get install devscripts shellcheck
          nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
-          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy
+          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
          go list -m all | ./nancy sleuth
          make
          diff -au <(gofmt -s -d cmd) <(printf "")
@@ -15,6 +15,8 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3

 ENV MINIO_ACCESS_KEY_FILE=access_key \
    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
@@ -17,6 +17,8 @@ ARG TARGETARCH

 ENV MINIO_ACCESS_KEY_FILE=access_key \
    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
@@ -12,6 +12,8 @@ COPY LICENSE /licenses/LICENSE
 ENV MINIO_UPDATE=off \
    MINIO_ACCESS_KEY_FILE=access_key \
    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
    MINIO_SSE_MASTER_KEY_FILE=sse_master_key

@@ -5,13 +5,15 @@ ARG TARGETARCH
 LABEL name="MinIO" \
      vendor="MinIO Inc <dev@min.io>" \
      maintainer="MinIO Inc <dev@min.io>" \
-      version="RELEASE.2020-12-29T23-29-29Z" \
-      release="RELEASE.2020-12-29T23-29-29Z" \
+      version="RELEASE.2021-01-08T21-18-21Z" \
+      release="RELEASE.2021-01-08T21-18-21Z" \
      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."

 ENV MINIO_ACCESS_KEY_FILE=access_key \
    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
@@ -17,9 +17,9 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
-	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...)
-	@which msgp 1>/dev/null || (echo "Installing msgp" && GO111MODULE=off go get github.com/tinylib/msgp)
-	@which stringer 1>/dev/null || (echo "Installing stringer" && GO111MODULE=off go get golang.org/x/tools/cmd/stringer)
+	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1)
+	@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
+	@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)

 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -5,80 +5,190 @@

 MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.

-## Docker Container
-### Stable
-```
-docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+This README provides quickstart instructions on running MinIO on baremetal hardware, including Docker-based installations. For Kubernetes environments, 
+use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md). 
+
+**Table of Contents**
+
+- [Docker Installation](#docker-installation)
+- [macOS Installation](#macos)
+- [GNU/Linux Installation](#gnulinux)
+- [Windows Installation](#microsoft-windows)
+- [FreeBSD Installation](#freebsd)
+- [Source Installation](#install-from-source)
+- [Deployment Recommendations](#deployment-recommendations)
+- [Test Connectivity to MinIO](#test-minio-connectivity)
+- [Upgrade MinIO](#upgrading-minio)
+- [Explore Further](#explore-further)
+- [Contribute to MinIO](#contribute-to-minio-project)
+- [License](#license)
+
+# Docker Installation
+
+Use the following commands to run a standalone MinIO server on a Docker container. 
+
+Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+## Stable
+
+Run the following command to run the latest stable image of MinIO on a Docker container using an ephemeral data volume: 
+
+```sh
+docker run -p 9000:9000 minio/minio server /data
 ```

-### Edge
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: To deploy MinIO on Docker with persistent storage, you must map local persistent directories from the host OS to the container using the
+  `docker -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the Docker container.
+
+## Edge
+
+Run the following command to run the bleeding-edge image of MinIO on a Docker container using an ephemeral data volume:
+
 ```
-docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio:edge server /data
+docker run -p 9000:9000 minio/minio:edge server /data
 ```

-> NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: To deploy MinIO on Docker with persistent storage, you must map local persistent directories from the host OS to the container using the
+  `docker -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the Docker container.
+
+# macOS
+
+Use the following commands to run a standalone MinIO server on macOS.
+
+Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+## Homebrew (recommended)
+
+Run the following command to install the latest stable MinIO package using [Homebrew](https://brew.sh/). Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.

-## macOS
-### Homebrew (recommended)
-Install minio packages using [Homebrew](https://brew.sh/)
 ```sh
 brew install minio/stable/minio
 minio server /data
 ```

 > NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
+
 ```sh
 brew uninstall minio
 brew install minio/stable/minio
 ```

-### Binary Download
-| Platform    | Architecture | URL                                                       |
-| ----------  | --------     | ------                                                    |
-| Apple macOS | 64-bit Intel | https://dl.min.io/server/minio/release/darwin-amd64/minio |
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+## Binary Download
+
+Use the following command to download and run a standalone MinIO server on macOS. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
+
 ```sh
-chmod 755 minio
+wget https://dl.min.io/server/minio/release/darwin-amd64/minio
+chmod +x minio
 ./minio server /data
 ```

-## GNU/Linux
-### Binary Download
-| Platform   | Architecture | URL                                                      |
-| ---------- | --------     | ------                                                   |
-| GNU/Linux  | 64-bit Intel | https://dl.min.io/server/minio/release/linux-amd64/minio |
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+# GNU/Linux
+
+Use the following command to run a standalone MinIO server on Linux hosts running 64-bit Intel/AMD architectures. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
+
 ```sh
 wget https://dl.min.io/server/minio/release/linux-amd64/minio
 chmod +x minio
 ./minio server /data
 ```

-| Platform   | Architecture | URL                                                        |
-| ---------- | --------     | ------                                                     |
-| GNU/Linux  | ppc64le      | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
+Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
+
+The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
+
+| Architecture                   | URL                                                        |
+| --------                       | ------                                                     |
+| 64-bit Intel/AMD               | https://dl.min.io/server/minio/release/linux-amd64/minio   |
+| 64-bit ARM                     | https://dl.min.io/server/minio/release/linux-arm64/minio   |
+| 64-bit PowerPC LE (ppc64le)    | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
+| IBM Z-Series (S390X)           | https://dl.min.io/server/minio/release/linux-s390x/minio   |
+
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+# Microsoft Windows
+
+To run MinIO on 64-bit Windows hosts, download the MinIO executable from the following URL:
+
 ```sh
-wget https://dl.min.io/server/minio/release/linux-ppc64le/minio
-chmod +x minio
-./minio server /data
+https://dl.min.io/server/minio/release/windows-amd64/minio.exe
 ```

-## Microsoft Windows
-### Binary Download
-| Platform          | Architecture | URL                                                            |
-| ----------        | --------     | ------                                                         |
-| Microsoft Windows | 64-bit       | https://dl.min.io/server/minio/release/windows-amd64/minio.exe |
+Use the following command to run a standalone MinIO server on the Windows host. Replace ``D:\`` with the path to the drive or directory in which you want MinIO to store data. You must change the terminal or powershell directory to the location of the ``minio.exe`` executable, *or* add the path to that directory to the system ``$PATH``:
+
 ```sh
-minio.exe server D:\Photos
+minio.exe server D:\
 ```

-## FreeBSD
-### Port
-Install minio packages using [pkg](https://github.com/freebsd/pkg), MinIO doesn't officially build FreeBSD binaries but is maintained by FreeBSD upstream [here](https://www.freshports.org/www/minio).
+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation.
+
+# FreeBSD
+
+MinIO does not provide an official FreeBSD binary. However, FreeBSD maintains an [upstream release](https://www.freshports.org/www/minio) using [pkg](https://github.com/freebsd/pkg):

 ```sh
 pkg install minio
@@ -87,13 +197,32 @@ sysrc minio_disks=/home/user/Photos
 service minio start
 ```

-## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
+# Install from Source
+
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)

 ```sh
 GO111MODULE=on go get github.com/minio/minio
 ```

+The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded 
+web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the 
+root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
+
+You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See 
+[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers, 
+see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
+
+
+> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication 
+require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically, 
+with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html) 
+for more complete documentation. 
+
+MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments. 
+
+# Deployment Recommendations
+
 ## Allow port access for Firewalls

 By default MinIO uses the port 9000 to listen for incoming connections. If your platform blocks the port by default, you may need to enable access to the port.
@@ -149,6 +278,13 @@ iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
 service iptables restart
 ```

+## Pre-existing data
+When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
+
+The above statement is also valid for all gateway backends.
+
+# Test MinIO Connectivity
+
 ## Test using MinIO Browser
 MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure your server has started successfully.

@@ -157,12 +293,7 @@ MinIO Server comes with an embedded web based object browser. Point your web bro
 ## Test using MinIO Client `mc`
 `mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.

-## Pre-existing data
-When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command  `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
-
-The above statement is also valid for all gateway backends.
-
-## Upgrading MinIO
+# Upgrading MinIO
 MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster simultaneously and restart them, as shown in the following command from the MinIO client (mc):

 ```
@@ -171,7 +302,7 @@ mc admin update <minio alias, e.g., myminio>

 > NOTE: some releases might not allow rolling upgrades, this is always called out in the release notes and it is generally advised to read release notes before upgrading. In such a situation `mc admin update` is the recommended upgrading mechanism to upgrade all servers at once.

-### Important things to remember during MinIO upgrades
+## Important things to remember during MinIO upgrades

 - `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
@@ -181,7 +312,7 @@ mc admin update <minio alias, e.g., myminio>
 - If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
 - If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md

-## Explore Further
+# Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
 - [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
@@ -189,8 +320,8 @@ mc admin update <minio alias, e.g., myminio>
 - [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
 - [The MinIO documentation website](https://docs.min.io)

-## Contribute to MinIO Project
+# Contribute to MinIO Project
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)

-## License
+# License
 Use of MinIO is governed by the Apache 2.0 License found at [LICENSE](https://github.com/minio/minio/blob/master/LICENSE).
@@ -8,16 +8,16 @@ MinIO是一个非常轻量的服务,可以很简单的和其他应用的结合
 ### 稳定版
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
  minio/minio server /data
 ```

 ### 尝鲜版
 ```
 docker run -p 9000:9000 \
-  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
-  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
  minio/minio:edge server /data
 ```

@@ -138,9 +138,10 @@ describe("Uploads actions", () => {
        objects: { currentPrefix: "pre1/" }
      })
      store.dispatch(uploadsActions.uploadFile(file))
+      const objectPath = encodeURIComponent("pre1/file1")
      expect(open).toHaveBeenCalledWith(
        "PUT",
-        "https://localhost:8080/upload/test1/pre1/file1",
+        "https://localhost:8080/upload/test1/" + objectPath,
        true
      )
      expect(send).toHaveBeenCalledWith(file)
@@ -94,7 +94,7 @@ export const uploadFile = file => {
      _filePath = _filePath.substring(1)
    }
    const filePath = _filePath
-    const objectName = `${currentPrefix}${filePath}`
+    const objectName = encodeURIComponent(`${currentPrefix}${filePath}`)
    const uploadUrl = `${
      window.location.origin
    }${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
@@ -1,69 +0,0 @@
-#!/bin/bash
-#
-# MinIO Cloud Storage, (C) 2019 MinIO, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-set -E
-set -o pipefail
-
-function start_minio_server()
-{
-    MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
-                    minio --quiet --json server /data --address 127.0.0.1:24242 > server.log 2>&1 &
-    server_pid=$!
-    sleep 10
-
-    echo "$server_pid"
-}
-
-function start_minio_gateway_s3()
-{
-    MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
-                    minio --quiet --json gateway s3 http://127.0.0.1:24242 \
-                    --address 127.0.0.1:24240 > gateway.log 2>&1 &
-    gw_pid=$!
-    sleep 10
-
-    echo "$gw_pid"
-}
-
-function main()
-{
-    sr_pid="$(start_minio_server)"
-    gw_pid="$(start_minio_gateway_s3)"
-
-    SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
-                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
-                   aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
-                   healthcheck mc minio-dotnet minio-js \
-                   minio-py s3cmd s3select security
-    rv=$?
-
-    kill "$sr_pid"
-    kill "$gw_pid"
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        echo "=========== Gateway ==========="
-        cat "gateway.log"
-        echo "=========== Server ==========="
-        cat "server.log"
-    fi
-
-    rm -f gateway.log server.log
-}
-
-main "$@"
@@ -3,5 +3,5 @@
 set -e

 for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=1 go test -v -race --timeout 50m "$d"
+    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done
@@ -63,8 +63,8 @@ function start_minio_erasure_sets()

 function start_minio_pool_erasure_sets()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
    "${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
    "${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
@@ -74,8 +74,8 @@ function start_minio_pool_erasure_sets()

 function start_minio_pool_erasure_sets_ipv6()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
    "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
    "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
@@ -85,8 +85,8 @@ function start_minio_pool_erasure_sets_ipv6()

 function start_minio_dist_erasure()
 {
-    export MINIO_ACCESS_KEY=$ACCESS_KEY
-    export MINIO_SECRET_KEY=$SECRET_KEY
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
    for i in $(seq 0 3); do
        "${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
@@ -29,8 +29,8 @@ MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )

 function start_minio_3_node() {
-    export MINIO_ACCESS_KEY=minio
-    export MINIO_SECRET_KEY=minio123
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
    export MINIO_ERASURE_SET_DRIVE_COUNT=6

    start_port=$(shuf -i 10000-65000 -n 1)
@@ -1412,7 +1412,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque

 	}()

-	ticker := time.NewTicker(30 * time.Second)
+	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()

 	for {
@@ -45,7 +45,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := IsServerResolvable(endpoint); err == nil {
+				if err := isServerResolvable(endpoint); err == nil {
 					network[nodeName] = "online"
 				} else {
 					network[nodeName] = "offline"
@@ -153,22 +153,22 @@ wait:
 					continue
 				}

-				zoneIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
-				if zoneIdx < 0 {
+				poolIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
+				if poolIdx < 0 {
 					continue
 				}

 				// Calculate the set index where the current endpoint belongs
-				z.serverPools[zoneIdx].erasureDisksMu.RLock()
+				z.serverPools[poolIdx].erasureDisksMu.RLock()
 				// Protect reading reference format.
-				setIndex, _, err := findDiskIndex(z.serverPools[zoneIdx].format, format)
-				z.serverPools[zoneIdx].erasureDisksMu.RUnlock()
+				setIndex, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
+				z.serverPools[poolIdx].erasureDisksMu.RUnlock()
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					continue
 				}

-				erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk)
+				erasureSetInZoneDisksToHeal[poolIdx][setIndex] = append(erasureSetInZoneDisksToHeal[poolIdx][setIndex], disk)
 			}

 			buckets, _ := z.ListBuckets(ctx)
@@ -181,11 +181,11 @@ wait:
 			for i, setMap := range erasureSetInZoneDisksToHeal {
 				for setIndex, disks := range setMap {
 					for _, disk := range disks {
-						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+						logger.Info("Healing disk '%s' on %s pool", disk, humanize.Ordinal(i+1))

 						// So someone changed the drives underneath, healing tracker missing.
 						if !disk.Healing() {
-							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s zone", disk, humanize.Ordinal(i+1))
+							logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
 							diskID, err := disk.GetDiskID()
 							if err != nil {
 								logger.LogIf(ctx, err)
@@ -209,7 +209,7 @@ wait:
 							continue
 						}

-						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
+						logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))

 						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
 							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
@@ -91,6 +91,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 // ReadAt() implementation which verifies the bitrot hash available as part of the stream.
 type streamingBitrotReader struct {
 	disk       StorageAPI
+	data       []byte
 	rc         io.Reader
 	volume     string
 	filePath   string
@@ -122,7 +123,11 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
-		b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		if len(b.data) == 0 {
+			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		} else {
+			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
+		}
 		if err != nil {
 			return 0, err
 		}
@@ -154,10 +159,11 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 }

 // Returns streaming bitrot reader implementation.
-func newStreamingBitrotReader(disk StorageAPI, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
+func newStreamingBitrotReader(disk StorageAPI, data []byte, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
 	h := algo.New()
 	return &streamingBitrotReader{
 		disk,
+		data,
 		nil,
 		volume,
 		filePath,
@@ -103,9 +103,9 @@ func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, alg
 	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }

-func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
+func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
 	if algo == HighwayHash256S {
-		return newStreamingBitrotReader(disk, bucket, filePath, tillOffset, algo, shardSize)
+		return newStreamingBitrotReader(disk, data, bucket, filePath, tillOffset, algo, shardSize)
 	}
 	return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum)
 }
@@ -62,7 +62,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	}
 	writer.(io.Closer).Close()

-	reader := newBitrotReader(disk, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
+	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
 	if _, err = reader.ReadAt(b, 0); err != nil {
 		log.Fatal(err)
@@ -405,9 +405,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		getObjectInfoFn = api.CacheAPI().GetObjectInfo
 	}
 	var (
-		hasLockEnabled, hasLifecycleConfig bool
-		goi                                ObjectInfo
-		gerr                               error
+		hasLockEnabled, hasLifecycleConfig, replicateSync bool
+		goi                                               ObjectInfo
+		gerr                                              error
 	)
 	replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjects.Objects)
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
@@ -455,11 +455,19 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			object.PurgeTransitioned = goi.TransitionStatus
 		}
 		if replicateDeletes {
-			delMarker, replicate := checkReplicateDelete(ctx, bucket, ObjectToDelete{
+			delMarker, replicate, repsync := checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectName: object.ObjectName,
 				VersionID:  object.VersionID,
 			}, goi, gerr)
+			replicateSync = repsync
 			if replicate {
+				if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object.ObjectName); apiErrCode != ErrNone {
+					if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
+						writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL, guessIsBrowserReq(r))
+						return
+					}
+					continue
+				}
 				if object.VersionID != "" {
 					object.VersionPurgeStatus = Pending
 					if delMarker {
@@ -549,10 +557,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	for _, dobj := range deletedObjects {
 		if replicateDeletes {
 			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
-				globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+				dv := DeletedObjectVersionInfo{
 					DeletedObject: dobj,
 					Bucket:        bucket,
-				})
+				}
+				scheduleReplicationDelete(ctx, dv, objectAPI, replicateSync)
 			}
 		}

@@ -340,13 +340,14 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 		return err
 	}
 	oi := gr.ObjInfo
-
 	if oi.TransitionStatus == lifecycle.TransitionComplete {
+		gr.Close()
 		return nil
 	}

 	putOpts := putTransitionOpts(oi)
-	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, "", "", putOpts); err != nil {
+	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, putOpts); err != nil {
+		gr.Close()
 		return err
 	}
 	gr.Close()
@@ -420,7 +421,7 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
 		}
 	}

-	reader, _, _, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
+	reader, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
 	if err != nil {
 		return nil, err
 	}
@@ -477,6 +477,15 @@ func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, ob
 	}
 }

+// Reset the state of the BucketMetadataSys.
+func (sys *BucketMetadataSys) Reset() {
+	sys.Lock()
+	for k := range sys.metadataMap {
+		delete(sys.metadataMap, k)
+	}
+	sys.Unlock()
+}
+
 // NewBucketMetadataSys - creates new policy system.
 func NewBucketMetadataSys() *BucketMetadataSys {
 	return &BucketMetadataSys{
@@ -88,35 +88,37 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return false, BucketRemoteTargetNotFound{Bucket: bucket}
 }

-func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) bool {
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) (replicate bool, sync bool) {
 	if permErr != ErrNone {
-		return false
+		return
 	}
 	return mustReplicater(ctx, bucket, object, meta, replStatus)
 }

-// mustReplicate returns true if object meets replication criteria.
-func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+// mustReplicate returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
+// a synchronous manner.
+func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) (replicate bool, sync bool) {
 	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
-		return false
+		return
 	}
 	return mustReplicater(ctx, bucket, object, meta, replStatus)
 }

-// mustReplicater returns true if object meets replication criteria.
-func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string) bool {
+// mustReplicater returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
+// a synchronous manner.
+func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string) (replicate bool, sync bool) {
 	if globalIsGateway {
-		return false
+		return replicate, sync
 	}
 	if rs, ok := meta[xhttp.AmzBucketReplicationStatus]; ok {
 		replStatus = rs
 	}
 	if replication.StatusType(replStatus) == replication.Replica {
-		return false
+		return replicate, sync
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		return false
+		return replicate, sync
 	}
 	opts := replication.ObjectOpts{
 		Name: object,
@@ -126,7 +128,20 @@ func mustReplicater(ctx context.Context, bucket, object string, meta map[string]
 	if ok {
 		opts.UserTags = tagStr
 	}
-	return cfg.Replicate(opts)
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
+	if tgt == nil || tgt.isOffline() {
+		return cfg.Replicate(opts), false
+	}
+	return cfg.Replicate(opts), tgt.replicateSync
+}
+
+// Standard headers that needs to be extracted from User metadata.
+var standardHeaders = []string{
+	"content-type",
+	"content-encoding",
+	xhttp.AmzStorageClass,
+	xhttp.AmzObjectTagging,
+	xhttp.AmzBucketReplicationStatus,
 }

 // returns true if any of the objects being deleted qualifies for replication.
@@ -143,33 +158,48 @@ func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToD
 	return false
 }

+// isStandardHeader returns true if header is a supported header and not a custom header
+func isStandardHeader(headerKey string) bool {
+	key := strings.ToLower(headerKey)
+	for _, header := range standardHeaders {
+		if strings.ToLower(header) == key {
+			return true
+		}
+	}
+	return false
+}
+
 // returns whether object version is a deletemarker and if object qualifies for replication
-func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, gerr error) (dm, replicate bool) {
+func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, gerr error) (dm, replicate, sync bool) {
 	rcfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || rcfg == nil {
-		return false, false
+		return false, false, sync
 	}
 	// when incoming delete is removal of a delete marker( a.k.a versioned delete),
 	// GetObjectInfo returns extra information even though it returns errFileNotFound
 	if gerr != nil {
 		validReplStatus := false
 		switch oi.ReplicationStatus {
-		case replication.Pending, replication.Complete, replication.Failed:
+		case replication.Pending, replication.Completed, replication.Failed:
 			validReplStatus = true
 		}
 		if oi.DeleteMarker && validReplStatus {
-			return oi.DeleteMarker, true
+			return oi.DeleteMarker, true, sync
 		}
-		return oi.DeleteMarker, false
+		return oi.DeleteMarker, false, sync
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
+	if tgt == nil || tgt.isOffline() {
+		return oi.DeleteMarker, false, false
 	}
 	opts := replication.ObjectOpts{
 		Name:         dobj.ObjectName,
 		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
 		UserTags:     oi.UserTags,
-		DeleteMarker: true,
+		DeleteMarker: oi.DeleteMarker,
 		VersionID:    dobj.VersionID,
 	}
-	return oi.DeleteMarker, rcfg.Replicate(opts)
+	return oi.DeleteMarker, rcfg.Replicate(opts), tgt.replicateSync
 }

 // replicate deletes to the designated replication target if replication configuration
@@ -216,7 +246,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
 		}
 	} else {
 		if dobj.VersionID == "" {
-			replicationStatus = string(replication.Complete)
+			replicationStatus = string(replication.Completed)
 		} else {
 			versionPurgeStatus = Complete
 		}
@@ -296,10 +326,10 @@ func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]
 func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
-		if k == xhttp.AmzBucketReplicationStatus {
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			continue
 		}
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if isStandardHeader(k) {
 			continue
 		}
 		meta[k] = v
@@ -413,6 +443,7 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		VersionID: objInfo.VersionID,
 	})
 	if err != nil {
+		logger.LogIf(ctx, err)
 		return
 	}
 	objInfo = gr.ObjInfo
@@ -440,38 +471,45 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 			return
 		}
 	}
+	replicationStatus := replication.Completed
+	if rtype != replicateAll {
+		gr.Close()

-	target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
-		return
-	}
-	putOpts := putReplicationOpts(ctx, dest, objInfo)
-	replicationStatus := replication.Complete
-
-	// Setup bandwidth throttling
-	peers, _ := globalEndpoints.peers()
-	totalNodesCount := len(peers)
-	if totalNodesCount == 0 {
-		totalNodesCount = 1 // For standalone erasure coding
-	}
-	b := target.BandwidthLimit / int64(totalNodesCount)
-	var headerSize int
-	for k, v := range putOpts.Header() {
-		headerSize += len(k) + len(v)
-	}
-	r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
-	if rtype == replicateAll {
-		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
-	} else {
 		// replicate metadata for object tagging/copy with metadata replacement
 		dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
-		_, err = tgt.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
-	}
+		c := &miniogo.Core{Client: tgt.Client}
+		_, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
+		if err != nil {
+			replicationStatus = replication.Failed
+		}
+	} else {
+		target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
+			gr.Close()
+			return
+		}

-	r.Close()
-	if err != nil {
-		replicationStatus = replication.Failed
+		putOpts := putReplicationOpts(ctx, dest, objInfo)
+		// Setup bandwidth throttling
+		peers, _ := globalEndpoints.peers()
+		totalNodesCount := len(peers)
+		if totalNodesCount == 0 {
+			totalNodesCount = 1 // For standalone erasure coding
+		}
+		b := target.BandwidthLimit / int64(totalNodesCount)
+		var headerSize int
+		for k, v := range putOpts.Header() {
+			headerSize += len(k) + len(v)
+		}
+
+		// r takes over closing gr.
+		r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, putOpts)
+		if err != nil {
+			replicationStatus = replication.Failed
+		}
+		r.Close()
 	}
 	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
 	if objInfo.UserTags != "" {
@@ -616,3 +654,140 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 		globalReplicationState.addWorker(ctx, objectAPI)
 	}
 }
+
+// get Reader from replication target if active-active replication is in place and
+// this node returns a 404
+func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, proxy bool) {
+	tgt, oi, proxy, err := proxyHeadToRepTarget(ctx, bucket, object, opts)
+	if !proxy || err != nil {
+		return nil, false
+	}
+	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	if err != nil {
+		return nil, false
+	}
+	gopts := miniogo.GetObjectOptions{
+		VersionID:            opts.VersionID,
+		ServerSideEncryption: opts.ServerSideEncryption,
+		Internal: miniogo.AdvancedGetOptions{
+			ReplicationProxyRequest: true,
+		},
+	}
+	// get correct offsets for encrypted object
+	if off >= 0 && length >= 0 {
+		if err := gopts.SetRange(off, off+length-1); err != nil {
+			return nil, false
+		}
+	}
+	c := miniogo.Core{Client: tgt.Client}
+
+	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
+	if err != nil {
+		return nil, false
+	}
+	closeReader := func() { obj.Close() }
+
+	reader, err := fn(obj, h, opts.CheckPrecondFn, closeReader)
+	if err != nil {
+		return nil, false
+	}
+	return reader, true
+}
+
+// isProxyable returns true if replication config found for this bucket
+func isProxyable(ctx context.Context, bucket string) bool {
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return false
+	}
+	dest := cfg.GetDestination()
+	return dest.Bucket == bucket
+}
+func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (tgt *TargetClient, oi ObjectInfo, proxy bool, err error) {
+	// this option is set when active-active replication is in place between site A -> B,
+	// and site B does not have the object yet.
+	if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
+		return nil, oi, false, nil
+	}
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return nil, oi, false, err
+	}
+	dest := cfg.GetDestination()
+	if dest.Bucket != bucket { // not active-active
+		return nil, oi, false, err
+	}
+	ssec := false
+	if opts.ServerSideEncryption != nil {
+		ssec = opts.ServerSideEncryption.Type() == encrypt.SSEC
+	}
+	ropts := replication.ObjectOpts{
+		Name: object,
+		SSEC: ssec,
+	}
+	if !cfg.Replicate(ropts) { // no matching rule for object prefix
+		return nil, oi, false, nil
+	}
+	tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
+	if tgt == nil || tgt.isOffline() {
+		return nil, oi, false, fmt.Errorf("missing target")
+	}
+
+	gopts := miniogo.GetObjectOptions{
+		VersionID:            opts.VersionID,
+		ServerSideEncryption: opts.ServerSideEncryption,
+		Internal: miniogo.AdvancedGetOptions{
+			ReplicationProxyRequest: true,
+		},
+	}
+
+	objInfo, err := tgt.StatObject(ctx, dest.Bucket, object, gopts)
+	if err != nil {
+		return nil, oi, false, err
+	}
+	tags, _ := tags.MapToObjectTags(objInfo.UserTags)
+	oi = ObjectInfo{
+		Bucket:            bucket,
+		Name:              object,
+		ModTime:           objInfo.LastModified,
+		Size:              objInfo.Size,
+		ETag:              objInfo.ETag,
+		VersionID:         objInfo.VersionID,
+		IsLatest:          objInfo.IsLatest,
+		DeleteMarker:      objInfo.IsDeleteMarker,
+		ContentType:       objInfo.ContentType,
+		Expires:           objInfo.Expires,
+		StorageClass:      objInfo.StorageClass,
+		ReplicationStatus: replication.StatusType(objInfo.ReplicationStatus),
+		UserDefined:       cloneMSS(objInfo.UserMetadata),
+		UserTags:          tags.String(),
+	}
+	if ce, ok := oi.UserDefined[xhttp.ContentEncoding]; ok {
+		oi.ContentEncoding = ce
+		delete(oi.UserDefined, xhttp.ContentEncoding)
+	}
+	return tgt, oi, true, nil
+}
+
+// get object info from replication target if active-active replication is in place and
+// this node returns a 404
+func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, proxy bool, err error) {
+	_, oi, proxy, err = proxyHeadToRepTarget(ctx, bucket, object, opts)
+	return oi, proxy, err
+}
+
+func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, sync bool) {
+	if sync {
+		replicateObject(ctx, objInfo, o)
+	} else {
+		globalReplicationState.queueReplicaTask(objInfo)
+	}
+}
+
+func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectVersionInfo, o ObjectLayer, sync bool) {
+	if sync {
+		replicateDelete(ctx, dv, o)
+	} else {
+		globalReplicationState.queueReplicaDeleteTask(dv)
+	}
+}
@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"

 	minio "github.com/minio/minio-go/v7"
@@ -34,10 +35,14 @@ import (
 	sha256 "github.com/minio/sha256-simd"
 )

+const (
+	defaultHealthCheckDuration = 60 * time.Second
+)
+
 // BucketTargetSys represents bucket targets subsystem
 type BucketTargetSys struct {
 	sync.RWMutex
-	arnRemotesMap map[string]*miniogo.Core
+	arnRemotesMap map[string]*TargetClient
 	targetsMap    map[string][]madmin.BucketTarget
 }

@@ -68,7 +73,6 @@ func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType str

 // ListBucketTargets - gets list of bucket targets for this bucket.
 func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
-
 	sys.RLock()
 	defer sys.RUnlock()

@@ -132,7 +136,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m

 	tgts := sys.targetsMap[bucket]
 	newtgts := make([]madmin.BucketTarget, len(tgts))
-	labels := make(map[string]struct{})
+	labels := make(map[string]struct{}, len(tgts))
 	found := false
 	for idx, t := range tgts {
 		labels[t.Label] = struct{}{}
@@ -198,9 +202,12 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	// delete ARN type from list of matching targets
 	sys.Lock()
 	defer sys.Unlock()
-	targets := make([]madmin.BucketTarget, 0)
 	found := false
-	tgts := sys.targetsMap[bucket]
+	tgts, ok := sys.targetsMap[bucket]
+	if !ok {
+		return BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	targets := make([]madmin.BucketTarget, 0, len(tgts))
 	for _, tgt := range tgts {
 		if tgt.Arn != arnStr {
 			targets = append(targets, tgt)
@@ -217,7 +224,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 }

 // GetRemoteTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
+func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *TargetClient {
 	sys.RLock()
 	defer sys.RUnlock()
 	return sys.arnRemotesMap[arn]
@@ -264,7 +271,7 @@ func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, a
 // NewBucketTargetSys - creates new replication system.
 func NewBucketTargetSys() *BucketTargetSys {
 	return &BucketTargetSys{
-		arnRemotesMap: make(map[string]*miniogo.Core),
+		arnRemotesMap: make(map[string]*TargetClient),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
 	}
 }
@@ -345,20 +352,33 @@ var getRemoteTargetInstanceTransport http.RoundTripper
 var getRemoteTargetInstanceTransportOnce sync.Once

 // Returns a minio-go Client configured to access remote host described in replication target config.
-func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
+func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {
 	config := tcfg.Credentials
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")

 	getRemoteTargetInstanceTransportOnce.Do(func() {
 		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
 	})
-
-	core, err := miniogo.NewCore(tcfg.URL().Host, &miniogo.Options{
+	api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,
 		Transport: getRemoteTargetInstanceTransport,
 	})
-	return core, err
+	if err != nil {
+		return nil, err
+	}
+	hcDuration := defaultHealthCheckDuration
+	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
+		hcDuration = tcfg.HealthCheckDuration
+	}
+	tc := &TargetClient{
+		Client:              api,
+		healthCheckDuration: hcDuration,
+		bucket:              tcfg.TargetBucket,
+		replicateSync:       tcfg.ReplicationSync,
+	}
+	go tc.healthCheck()
+	return tc, nil
 }

 // getRemoteARN gets existing ARN for an endpoint or generates a new one.
@@ -422,3 +442,29 @@ func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.Bu
 	}
 	return &t, nil
 }
+
+// TargetClient is the struct for remote target client.
+type TargetClient struct {
+	*miniogo.Client
+	up                  int32
+	healthCheckDuration time.Duration
+	bucket              string // remote bucket target
+	replicateSync       bool
+}
+
+func (tc *TargetClient) isOffline() bool {
+	return atomic.LoadInt32(&tc.up) == 0
+}
+
+func (tc *TargetClient) healthCheck() {
+	for {
+		_, err := tc.BucketExists(GlobalContext, tc.bucket)
+		if err != nil {
+			atomic.StoreInt32(&tc.up, 0)
+			time.Sleep(tc.healthCheckDuration * time.Second)
+			continue
+		}
+		atomic.StoreInt32(&tc.up, 1)
+		time.Sleep(tc.healthCheckDuration * time.Second)
+	}
+}
@@ -51,6 +51,11 @@ func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, erro
 	return globalBucketMetadataSys.GetVersioningConfig(bucket)
 }

+// Reset BucketVersioningSys to initial state.
+func (sys *BucketVersioningSys) Reset() {
+	// There is currently no internal state.
+}
+
 // NewBucketVersioningSys - creates new versioning system.
 func NewBucketVersioningSys() *BucketVersioningSys {
 	return &BucketVersioningSys{}
@@ -51,7 +51,7 @@ func init() {
 	logger.RegisterError(config.FmtError)

 	rand.Seed(time.Now().UTC().UnixNano())
-	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second)
+	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second, logger.LogOnceIf)

 	initGlobalContext()

@@ -286,6 +286,16 @@ func handleCommonEnvVars() {
 		globalConfigEncrypted = true
 	}

+	if env.IsSet(config.EnvRootUser) || env.IsSet(config.EnvRootPassword) {
+		cred, err := auth.CreateCredentials(env.Get(config.EnvRootUser, ""), env.Get(config.EnvRootPassword, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the shell environment")
+		}
+		globalActiveCred = cred
+		globalConfigEncrypted = true
+	}
+
 	if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
 		oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
 		if err != nil {
@@ -296,6 +306,17 @@ func handleCommonEnvVars() {
 		os.Unsetenv(config.EnvAccessKeyOld)
 		os.Unsetenv(config.EnvSecretKeyOld)
 	}
+
+	if env.IsSet(config.EnvRootUserOld) && env.IsSet(config.EnvRootPasswordOld) {
+		oldCred, err := auth.CreateCredentials(env.Get(config.EnvRootUserOld, ""), env.Get(config.EnvRootPasswordOld, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate the old credentials inherited from the shell environment")
+		}
+		globalOldCred = oldCred
+		os.Unsetenv(config.EnvRootUserOld)
+		os.Unsetenv(config.EnvRootPasswordOld)
+	}
 }

 func logStartupMessage(msg string) {
@@ -207,7 +207,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 	}

 	if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
-		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+		logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
 	}

 	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
@@ -294,7 +294,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 	}

 	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
-		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
+		logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
 	}

 	return saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
@@ -26,19 +26,22 @@ import (

 // Config represents the compression settings.
 type Config struct {
-	Enabled    bool     `json:"enabled"`
-	Extensions []string `json:"extensions"`
-	MimeTypes  []string `json:"mime-types"`
+	Enabled        bool     `json:"enabled"`
+	AllowEncrypted bool     `json:"allow_encryption"`
+	Extensions     []string `json:"extensions"`
+	MimeTypes      []string `json:"mime-types"`
 }

 // Compression environment variables
 const (
-	Extensions = "extensions"
-	MimeTypes  = "mime_types"
+	Extensions     = "extensions"
+	AllowEncrypted = "allow_encryption"
+	MimeTypes      = "mime_types"

-	EnvCompressState      = "MINIO_COMPRESS_ENABLE"
-	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
-	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIME_TYPES"
+	EnvCompressState           = "MINIO_COMPRESS_ENABLE"
+	EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
+	EnvCompressExtensions      = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes       = "MINIO_COMPRESS_MIME_TYPES"

 	// Include-list for compression.
 	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
@@ -52,6 +55,10 @@ var (
 			Key:   config.Enable,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   AllowEncrypted,
+			Value: config.EnableOff,
+		},
 		config.KV{
 			Key:   Extensions,
 			Value: DefaultExtensions,
@@ -101,6 +108,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		return cfg, nil
 	}

+	allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
+	cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
+	if err != nil {
+		return cfg, err
+	}
+
 	compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
 	compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
 	compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
@@ -23,17 +23,21 @@ const (

 // Top level common ENVs
 const (
-	EnvAccessKey    = "MINIO_ACCESS_KEY"
-	EnvSecretKey    = "MINIO_SECRET_KEY"
-	EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
-	EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
-	EnvBrowser      = "MINIO_BROWSER"
-	EnvDomain       = "MINIO_DOMAIN"
-	EnvRegionName   = "MINIO_REGION_NAME"
-	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
-	EnvFSOSync      = "MINIO_FS_OSYNC"
-	EnvArgs         = "MINIO_ARGS"
-	EnvDNSWebhook   = "MINIO_DNS_WEBHOOK_ENDPOINT"
+	EnvAccessKey       = "MINIO_ACCESS_KEY"
+	EnvSecretKey       = "MINIO_SECRET_KEY"
+	EnvRootUser        = "MINIO_ROOT_USER"
+	EnvRootPassword    = "MINIO_ROOT_PASSWORD"
+	EnvAccessKeyOld    = "MINIO_ACCESS_KEY_OLD"
+	EnvSecretKeyOld    = "MINIO_SECRET_KEY_OLD"
+	EnvRootUserOld     = "MINIO_ROOT_USER_OLD"
+	EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD"
+	EnvBrowser         = "MINIO_BROWSER"
+	EnvDomain          = "MINIO_DOMAIN"
+	EnvRegionName      = "MINIO_REGION_NAME"
+	EnvPublicIPs       = "MINIO_PUBLIC_IPS"
+	EnvFSOSync         = "MINIO_FS_OSYNC"
+	EnvArgs            = "MINIO_ARGS"
+	EnvDNSWebhook      = "MINIO_DNS_WEBHOOK_ENDPOINT"

 	EnvUpdate = "MINIO_UPDATE"

@@ -116,19 +116,19 @@ var (
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",
-		`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
+		`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
 	)

 	ErrInvalidCredentialsBackendEncrypted = newErrFn(
 		"Invalid credentials",
 		"Please set correct credentials in the environment for decryption",
-		`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+		`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
 	)

 	ErrMissingCredentialsBackendEncrypted = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
+		`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
 	)

 	ErrInvalidCredentials = newErrFn(
@@ -140,13 +140,13 @@ var (
 	ErrEnvCredentialsMissingGateway = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
 	)

 	ErrEnvCredentialsMissingDistributed = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
-		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
 	)

 	ErrInvalidErasureEndpoints = newErrFn(
@@ -993,7 +993,7 @@ func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, oi Objec
 	case replication.Failed:
 		sizeS.failedSize += oi.Size
 		globalReplicationState.queueReplicaTask(oi)
-	case replication.Complete:
+	case replication.Completed, "COMPLETE":
 		sizeS.replicatedSize += oi.Size
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
@@ -56,13 +56,29 @@ type dataUsageEntry struct {
 	Children               dataUsageHashMap
 }

-// dataUsageCache contains a cache of data usage entries.
+//msgp:tuple dataUsageEntryV2
+type dataUsageEntryV2 struct {
+	// These fields do no include any children.
+	Size     int64
+	Objects  uint64
+	ObjSizes sizeHistogram
+	Children dataUsageHashMap
+}
+
+// dataUsageCache contains a cache of data usage entries latest version 3.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
 	Disks []string
 	Cache map[string]dataUsageEntry
 }

+// dataUsageCache contains a cache of data usage entries version 2.
+type dataUsageCacheV2 struct {
+	Info  dataUsageCacheInfo
+	Disks []string
+	Cache map[string]dataUsageEntryV2
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -478,6 +494,7 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 		*d = dataUsageCache{}
 		return nil
 	}
+	defer r.Close()
 	if err := d.deserialize(r); err != nil {
 		*d = dataUsageCache{}
 		logger.LogOnceIf(ctx, err, err.Error())
@@ -512,12 +529,16 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // dataUsageCacheVer indicates the cache version.
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
-const dataUsageCacheVer = 3
+const (
+	dataUsageCacheVerV3 = 3
+	dataUsageCacheVerV2 = 2
+	dataUsageCacheVerV1 = 1
+)

 // serialize the contents of the cache.
 func (d *dataUsageCache) serializeTo(dst io.Writer) error {
 	// Add version and compress.
-	_, err := dst.Write([]byte{dataUsageCacheVer})
+	_, err := dst.Write([]byte{dataUsageCacheVerV3})
 	if err != nil {
 		return err
 	}
@@ -552,21 +573,43 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 		return io.ErrUnexpectedEOF
 	}
 	switch b[0] {
-	case 1, 2:
+	case dataUsageCacheVerV1:
 		return errors.New("cache version deprecated (will autoupdate)")
-	case dataUsageCacheVer:
-	default:
-		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
-	}
+	case dataUsageCacheVerV2:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()

-	// Zstd compressed.
-	dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
-	if err != nil {
-		return err
-	}
-	defer dec.Close()
+		dold := &dataUsageCacheV2{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Disks = dold.Disks
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		for k, v := range dold.Cache {
+			d.Cache[k] = dataUsageEntry{
+				Size:     v.Size,
+				Objects:  v.Objects,
+				ObjSizes: v.ObjSizes,
+				Children: v.Children,
+			}
+		}
+		return nil
+	case dataUsageCacheVerV3:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()

-	return d.DecodeMsg(msgp.NewReader(dec))
+		return d.DecodeMsg(msgp.NewReader(dec))
+	}
+	return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
 }

 // Trim this from start+end of hashes.
@@ -484,6 +484,277 @@ func (z *dataUsageCacheInfo) Msgsize() (s int) {
 	return
 }

+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheV2) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			err = z.Info.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Disks":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV2, zb0003)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0003 > 0 {
+				zb0003--
+				var za0002 string
+				var za0003 dataUsageEntryV2
+				za0002, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				err = za0003.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0002)
+					return
+				}
+				z.Cache[za0002] = za0003
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageCacheV2) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "Info"
+	err = en.Append(0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	if err != nil {
+		return
+	}
+	err = z.Info.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// write "Disks"
+	err = en.Append(0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Disks)))
+	if err != nil {
+		err = msgp.WrapError(err, "Disks")
+		return
+	}
+	for za0001 := range z.Disks {
+		err = en.WriteString(z.Disks[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "Disks", za0001)
+			return
+		}
+	}
+	// write "Cache"
+	err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Cache)))
+	if err != nil {
+		err = msgp.WrapError(err, "Cache")
+		return
+	}
+	for za0002, za0003 := range z.Cache {
+		err = en.WriteString(za0002)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache")
+			return
+		}
+		err = za0003.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0002)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageCacheV2) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "Info"
+	o = append(o, 0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	o, err = z.Info.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// string "Disks"
+	o = append(o, 0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Disks)))
+	for za0001 := range z.Disks {
+		o = msgp.AppendString(o, z.Disks[za0001])
+	}
+	// string "Cache"
+	o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Cache)))
+	for za0002, za0003 := range z.Cache {
+		o = msgp.AppendString(o, za0002)
+		o, err = za0003.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0002)
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			bts, err = z.Info.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Disks":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Disks")
+				return
+			}
+			if cap(z.Disks) >= int(zb0002) {
+				z.Disks = (z.Disks)[:zb0002]
+			} else {
+				z.Disks = make([]string, zb0002)
+			}
+			for za0001 := range z.Disks {
+				z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Disks", za0001)
+					return
+				}
+			}
+		case "Cache":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV2, zb0003)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0003 > 0 {
+				var za0002 string
+				var za0003 dataUsageEntryV2
+				zb0003--
+				za0002, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				bts, err = za0003.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0002)
+					return
+				}
+				z.Cache[za0002] = za0003
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageCacheV2) Msgsize() (s int) {
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
+	for za0001 := range z.Disks {
+		s += msgp.StringPrefixSize + len(z.Disks[za0001])
+	}
+	s += 6 + msgp.MapHeaderSize
+	if z.Cache != nil {
+		for za0002, za0003 := range z.Cache {
+			_ = za0003
+			s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
+		}
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 	var zb0001 uint32
@@ -705,6 +976,163 @@ func (z *dataUsageEntry) Msgsize() (s int) {
 	return
 }

+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntryV2) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, err = dc.ReadInt64()
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.DecodeMsg(dc)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageEntryV2) EncodeMsg(en *msgp.Writer) (err error) {
+	// array header, size 4
+	err = en.Append(0x94)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	err = en.WriteUint64(z.Objects)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		err = en.WriteUint64(z.ObjSizes[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageEntryV2) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// array header, size 4
+	o = append(o, 0x94)
+	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.Objects)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z.ObjSizes {
+		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
+	}
+	o, err = z.Children.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntryV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	bts, err = z.Children.UnmarshalMsg(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntryV2) Msgsize() (s int) {
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{
@@ -235,6 +235,119 @@ func BenchmarkDecodedataUsageCacheInfo(b *testing.B) {
 	}
 }

+func TestMarshalUnmarshaldataUsageCacheV2(t *testing.T) {
+	v := dataUsageCacheV2{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageCacheV2(b *testing.B) {
+	v := dataUsageCacheV2{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageCacheV2(b *testing.B) {
+	v := dataUsageCacheV2{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageCacheV2(b *testing.B) {
+	v := dataUsageCacheV2{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageCacheV2(t *testing.T) {
+	v := dataUsageCacheV2{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageCacheV2 Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageCacheV2{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageCacheV2(b *testing.B) {
+	v := dataUsageCacheV2{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageCacheV2(b *testing.B) {
+	v := dataUsageCacheV2{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaldataUsageEntry(t *testing.T) {
 	v := dataUsageEntry{}
 	bts, err := v.MarshalMsg(nil)
@@ -348,6 +461,119 @@ func BenchmarkDecodedataUsageEntry(b *testing.B) {
 	}
 }

+func TestMarshalUnmarshaldataUsageEntryV2(t *testing.T) {
+	v := dataUsageEntryV2{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageEntryV2(b *testing.B) {
+	v := dataUsageEntryV2{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageEntryV2(b *testing.B) {
+	v := dataUsageEntryV2{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageEntryV2(b *testing.B) {
+	v := dataUsageEntryV2{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageEntryV2(t *testing.T) {
+	v := dataUsageEntryV2{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageEntryV2 Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageEntryV2{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageEntryV2(b *testing.B) {
+	v := dataUsageEntryV2{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageEntryV2(b *testing.B) {
+	v := dataUsageEntryV2{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
 	v := sizeHistogram{}
 	bts, err := v.MarshalMsg(nil)
@@ -353,9 +353,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,

 // DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
 // reader
-func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
-	length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
-	io.Reader, error) {
+func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) {

 	bucket, object := oi.Bucket, oi.Name
 	// Single part case
@@ -196,7 +196,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}, nil
 }

-// ZoneEndpoints represent endpoints in a given zone
+// ZoneEndpoints represent endpoints in a given pool
 // along with its setCount and setDriveCount.
 type ZoneEndpoints struct {
 	SetCount     int
@@ -207,8 +207,8 @@ type ZoneEndpoints struct {
 // EndpointServerPools - list of list of endpoints
 type EndpointServerPools []ZoneEndpoints

-// GetLocalZoneIdx returns the zone which endpoint belongs to locally.
-// if ep is remote this code will return -1 zoneIndex
+// GetLocalZoneIdx returns the pool which endpoint belongs to locally.
+// if ep is remote this code will return -1 poolIndex
 func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
 	for i, zep := range l {
 		for _, cep := range zep.Endpoints {
@@ -222,7 +222,7 @@ func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
 	return -1
 }

-// Add add zone endpoints
+// Add add pool endpoints
 func (l *EndpointServerPools) Add(zeps ZoneEndpoints) error {
 	existSet := set.NewStringSet()
 	for _, zep := range *l {
@@ -478,8 +478,8 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 						// participate atleast one disk and be local.
 						//
 						// In special cases for replica set with expanded
-						// zone setups we need to make sure to provide
-						// value of foundPrevLocal from zone1 if we already
+						// pool setups we need to make sure to provide
+						// value of foundPrevLocal from pool1 if we already
 						// found a local setup. Only if we haven't found
 						// previous local we continue to wait to look for
 						// atleast one local.
@@ -134,7 +134,7 @@ func TestErasureDecode(t *testing.T) {
 			}
 			tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)

-			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
+			bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 		}

 		writer := bytes.NewBuffer(nil)
@@ -164,7 +164,7 @@ func TestErasureDecode(t *testing.T) {
 					continue
 				}
 				tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
-				bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
+				bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
 				if bitrotReaders[j] == nil {
@@ -270,7 +270,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 				continue
 			}
 			tillOffset := erasure.ShardFileOffset(offset, readLen, length)
-			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
+			bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length, nil)
 		closeBitrotReaders(bitrotReaders)
@@ -332,7 +332,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 				continue
 			}
 			tillOffset := erasure.ShardFileOffset(0, size, size)
-			bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
+			bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil {
 			panic(err)
@@ -99,7 +99,7 @@ func TestErasureHeal(t *testing.T) {
 		readers := make([]io.ReaderAt, len(disks))
 		for i, disk := range disks {
 			shardFilesize := erasure.ShardFileSize(test.size)
-			readers[i] = newBitrotReader(disk, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize())
+			readers[i] = newBitrotReader(disk, nil, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize())
 		}

 		// setup stale disks for the test case
@@ -186,7 +186,7 @@ func TestListOnlineDisks(t *testing.T) {
 			t.Fatalf("Failed to putObject %v", err)
 		}

-		partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+		partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 		fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
 		if err != nil {
 			t.Fatalf("Failed to getLatestFileInfo %v", err)
@@ -287,7 +287,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	readQuorum := len(erasureDisks) / 2
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 		t.Fatalf("Failed to read xl meta data %v", reducedErr)
@@ -295,7 +295,7 @@ func TestDisksWithAllParts(t *testing.T) {

 	// Test that all disks are returned without any failures with
 	// unmodified meta data
-	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	if err != nil {
 		t.Fatalf("Failed to read xl meta data %v", err)
 	}
@@ -45,12 +45,12 @@ func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts mad
 	writeQuorum := getWriteQuorum(len(storageDisks))

 	// Heal bucket.
-	return healBucket(ctx, storageDisks, storageEndpoints, bucket, writeQuorum, opts.DryRun)
+	return healBucket(ctx, storageDisks, storageEndpoints, bucket, writeQuorum, opts)
 }

 // Heal bucket - create buckets on disks where it does not exist.
 func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []string, bucket string, writeQuorum int,
-	dryRun bool) (res madmin.HealResultItem, err error) {
+	opts madmin.HealOpts) (res madmin.HealResultItem, err error) {

 	// Initialize sync waitgroup.
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -84,7 +84,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 				afterState[index] = madmin.DriveStateMissing

 				// mutate only if not a dry-run
-				if dryRun {
+				if opts.DryRun {
 					return nil
 				}

@@ -99,7 +99,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 	errs := g.Wait()

 	reducedErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum-1)
-	if reducedErr == errVolumeNotFound {
+	if errors.Is(reducedErr, errVolumeNotFound) && !opts.Recreate {
 		return res, nil
 	}

@@ -395,7 +395,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				if latestMeta.XLV1 {
 					partPath = pathJoin(object, fmt.Sprintf("part.%d", partNumber))
 				}
-				readers[i] = newBitrotReader(disk, bucket, partPath, tillOffset, checksumAlgo, checksumInfo.Hash, erasure.ShardSize())
+				readers[i] = newBitrotReader(disk, nil, bucket, partPath, tillOffset, checksumAlgo, checksumInfo.Hash, erasure.ShardSize())
 			}
 			writers := make([]io.Writer, len(outDatedDisks))
 			for i, disk := range outDatedDisks {
@@ -811,7 +811,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	storageEndpoints := er.getEndpoints()

 	// Read metadata files from all the disks
-	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID)
+	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)

 	if isAllNotFound(errs) {
 		err = toObjectErr(errFileNotFound, bucket, object)
@@ -67,7 +67,7 @@ func TestHealing(t *testing.T) {
 	}

 	disk := er.getDisks()[0]
-	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
+	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -84,7 +84,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}

-	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
+	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -113,7 +113,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}

-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "")
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -214,7 +214,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}

-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -239,7 +239,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}

-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -265,7 +265,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}

-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "")
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -115,7 +115,7 @@ func hashOrder(key string, cardinality int) []int {

 // Reads all `xl.meta` metadata as a FileInfo slice.
 // Returns error slice indicating the failed metadata reads.
-func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string) ([]FileInfo, []error) {
+func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, readData bool) ([]FileInfo, []error) {
 	metadataArray := make([]FileInfo, len(disks))

 	g := errgroup.WithNErrs(len(disks))
@@ -126,7 +126,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID)
+			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, readData)
 			if err != nil {
 				if !IsErr(err, errFileNotFound, errVolumeNotFound, errFileVersionNotFound, errDiskNotFound) {
 					logger.LogOnceIf(ctx, err, disks[index].String())
@@ -46,7 +46,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	disks := er.getDisks()

 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "")
+	metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "", false)

 	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -113,7 +113,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		}
 		for _, uploadIDDir := range uploadIDDirs {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
-			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "")
+			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "", false)
 			if err != nil {
 				continue
 			}
@@ -127,7 +127,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		return
 	}
 	for _, tmpDir := range tmpDirs {
-		fi, err := disk.ReadVersion(ctx, minioMetaTmpBucket, tmpDir, "")
+		fi, err := disk.ReadVersion(ctx, minioMetaTmpBucket, tmpDir, "", false)
 		if err != nil {
 			continue
 		}
@@ -181,7 +181,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		if populatedUploadIds.Contains(uploadID) {
 			continue
 		}
-		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
+		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "", false)
 		if err != nil {
 			return result, toObjectErr(err, bucket, object)
 		}
@@ -371,7 +371,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo

 	// Read metadata associated with the object from all disks.
 	partsMetadata, errs = readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket,
-		uploadIDPath, "")
+		uploadIDPath, "", false)

 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -474,7 +474,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	}

 	// Read metadata again because it might be updated with parallel upload of another part.
-	partsMetadata, errs = readAllFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs = readAllFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
 	reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if reducedErr == errErasureWriteQuorum {
 		return pi, toObjectErr(reducedErr, bucket, object)
@@ -552,7 +552,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 	storageDisks := er.getDisks()

 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID)
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID, false)

 	// get Quorum for this object
 	readQuorum, _, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -600,7 +600,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	storageDisks := er.getDisks()

 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)

 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -704,7 +704,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	storageDisks := er.getDisks()

 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)

 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -889,7 +889,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)

 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "")
+	partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "", false)

 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
@@ -18,6 +18,7 @@ package cmd

 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -58,7 +59,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d

 	// Read metadata associated with the object from all disks.
 	storageDisks := er.getDisks()
-	metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, false)

 	// get Quorum for this object
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
@@ -157,8 +158,16 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		unlockOnDefer = true
 	}

-	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
+	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
+		if isProxyable(ctx, bucket) && (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) {
+			// proxy to replication target if active-active replication is in place.
+			reader, proxy := proxyGetToReplicationTarget(ctx, bucket, object, rs, h, opts)
+			if reader == nil || !proxy {
+				return nil, toObjectErr(err, bucket, object)
+			}
+			return reader, nil
+		}
 		return nil, toObjectErr(err, bucket, object)
 	}

@@ -298,7 +307,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 			}
 			checksumInfo := metaArr[index].Erasure.GetChecksumInfo(partNumber)
 			partPath := pathJoin(object, metaArr[index].DataDir, fmt.Sprintf("part.%d", partNumber))
-			readers[index] = newBitrotReader(disk, bucket, partPath, tillOffset,
+			data := metaArr[index].Data
+			readers[index] = newBitrotReader(disk, data, bucket, partPath, tillOffset,
 				checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())

 			// Prefer local disks
@@ -337,7 +347,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje

 // getObject wrapper for erasure GetObject
 func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset, length int64, writer io.Writer, opts ObjectOptions) error {
-	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
+	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
 	}
@@ -364,11 +374,11 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }

-func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
+func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
 	disks := er.getDisks()

 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)

 	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -410,9 +420,17 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s

 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts)
+	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
+		// proxy HEAD to replication target if active-active replication configured on bucket
+		if isProxyable(ctx, bucket) && (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) {
+			oi, proxy, err := proxyHeadToReplicationTarget(ctx, bucket, object, opts)
+			if proxy {
+				return oi, err
+			}
+		}
 		return objInfo, toObjectErr(err, bucket, object)
+
 	}
 	objInfo = fi.ToObjectInfo(bucket, object)
 	if objInfo.TransitionStatus == lifecycle.TransitionComplete {
@@ -1073,7 +1091,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	disks := er.getDisks()

 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)

 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -1134,7 +1152,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	disks := er.getDisks()

 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)

 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -19,7 +19,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	crand "crypto/rand"
 	"errors"
+	"io"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -288,9 +290,13 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	bucket := "bucket"
 	object := "object"
 	opts := ObjectOptions{}
+	buf := make([]byte, 129<<10)
+	if _, err = io.ReadFull(crand.Reader, buf); err != nil {
+		t.Fatal(err)
+	}

 	// Test use case 1: All disks are online, xl.meta are present, but data are missing
-	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
+	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -304,7 +310,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		}
 	}

-	err = xl.GetObject(ctx, bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts)
+	err = xl.GetObject(ctx, bucket, object, 0, int64(len(buf)), ioutil.Discard, "", opts)
 	if err != toObjectErr(errFileNotFound, bucket, object) {
 		t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
 	}
@@ -315,7 +321,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	// invocations, where f - [0,2)

 	// Create "object" under "bucket".
-	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
+	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -521,7 +527,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "")
+	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "", false)

 	parts1SC := globalStorageClass

@@ -534,7 +540,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "")
+	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "", false)
 	parts2SC := globalStorageClass

 	// Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class
@@ -546,7 +552,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "")
+	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "", false)
 	parts3SC := globalStorageClass

 	// Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class
@@ -564,7 +570,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "")
+	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "", false)
 	parts4SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 6,
@@ -587,7 +593,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "")
+	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "", false)
 	parts5SC := storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
@@ -609,7 +615,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "")
+	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "", false)
 	parts6SC := storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
@@ -632,7 +638,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}

-	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "")
+	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "", false)
 	parts7SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 5,
@@ -33,6 +33,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -51,7 +52,7 @@ func (z *erasureServerPools) SingleZone() bool {
 	return len(z.serverPools) == 1
 }

-// Initialize new zone of erasure sets.
+// Initialize new pool of erasure sets.
 func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) {
 	var (
 		deploymentID string
@@ -127,9 +128,9 @@ func (z *erasureServerPools) SetDriveCount() int {
 	return z.serverPools[0].SetDriveCount()
 }

-type serverPoolsAvailableSpace []zoneAvailableSpace
+type serverPoolsAvailableSpace []poolAvailableSpace

-type zoneAvailableSpace struct {
+type poolAvailableSpace struct {
 	Index     int
 	Available uint64
 }
@@ -154,10 +155,10 @@ func (z *erasureServerPools) getAvailableZoneIdx(ctx context.Context, size int64
 	// choose when we reach this many
 	choose := rand.Uint64() % total
 	atTotal := uint64(0)
-	for _, zone := range serverPools {
-		atTotal += zone.Available
-		if atTotal > choose && zone.Available > 0 {
-			return zone.Index
+	for _, pool := range serverPools {
+		atTotal += pool.Available
+		if atTotal > choose && pool.Available > 0 {
+			return pool.Index
 		}
 	}
 	// Should not happen, but print values just in case.
@@ -165,8 +166,8 @@ func (z *erasureServerPools) getAvailableZoneIdx(ctx context.Context, size int64
 	return -1
 }

-// getServerPoolsAvailableSpace will return the available space of each zone after storing the content.
-// If there is not enough space the zone will return 0 bytes available.
+// getServerPoolsAvailableSpace will return the available space of each pool after storing the content.
+// If there is not enough space the pool will return 0 bytes available.
 // Negative sizes are seen as 0 bytes.
 func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, size int64) serverPoolsAvailableSpace {
 	if size < 0 {
@@ -208,7 +209,7 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
 				available = 0
 			}
 		}
-		serverPools[i] = zoneAvailableSpace{
+		serverPools[i] = poolAvailableSpace{
 			Index:     i,
 			Available: available,
 		}
@@ -216,14 +217,14 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
 	return serverPools
 }

-// getZoneIdx returns the found previous object and its corresponding zone idx,
-// if none are found falls back to most available space zone.
+// getZoneIdx returns the found previous object and its corresponding pool idx,
+// if none are found falls back to most available space pool.
 func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object string, opts ObjectOptions, size int64) (idx int, err error) {
 	if z.SingleZone() {
 		return 0, nil
 	}
-	for i, zone := range z.serverPools {
-		objInfo, err := zone.GetObjectInfo(ctx, bucket, object, opts)
+	for i, pool := range z.serverPools {
+		objInfo, err := pool.GetObjectInfo(ctx, bucket, object, opts)
 		switch err.(type) {
 		case ObjectNotFound:
 			// VersionId was not specified but found delete marker or no versions exist.
@@ -236,7 +237,7 @@ func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object stri
 			}
 		}
 		// delete marker not specified means no versions
-		// exist continue to next zone.
+		// exist continue to next pool.
 		if !objInfo.DeleteMarker && err != nil {
 			continue
 		}
@@ -332,6 +333,12 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
 		return err
 	}

+	if len(allBuckets) == 0 {
+		logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl")
+		updates <- DataUsageInfo{} // no buckets found update data usage to reflect latest state
+		return nil
+	}
+
 	// Crawl latest allBuckets first.
 	sort.Slice(allBuckets, func(i, j int) bool {
 		return allBuckets[i].Created.After(allBuckets[j].Created)
@@ -376,7 +383,7 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
 		defer updateTicker.Stop()
 		var lastUpdate time.Time

-		// We need to merge since we will get the same buckets from each zone.
+		// We need to merge since we will get the same buckets from each pool.
 		// Therefore to get the exact bucket sizes we must merge before we can convert.
 		var allMerged dataUsageCache

@@ -477,8 +484,8 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object

 	object = encodeDirObject(object)

-	for _, zone := range z.serverPools {
-		gr, err = zone.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+	for _, pool := range z.serverPools {
+		gr, err = pool.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -500,8 +507,8 @@ func (z *erasureServerPools) GetObject(ctx context.Context, bucket, object strin

 	object = encodeDirObject(object)

-	for _, zone := range z.serverPools {
-		if err := zone.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
+	for _, pool := range z.serverPools {
+		if err := pool.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
 			}
@@ -521,8 +528,8 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 	}

 	object = encodeDirObject(object)
-	for _, zone := range z.serverPools {
-		objInfo, err = zone.GetObjectInfo(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		objInfo, err = pool.GetObjectInfo(ctx, bucket, object, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -538,7 +545,7 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 	return objInfo, ObjectNotFound{Bucket: bucket, Object: object}
 }

-// PutObject - writes an object to least used erasure zone.
+// PutObject - writes an object to least used erasure pool.
 func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
 	// Validate put object input args.
 	if err := checkPutObjectArgs(ctx, bucket, object, z); err != nil {
@@ -556,7 +563,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		return ObjectInfo{}, err
 	}

-	// Overwrite the object at the right zone
+	// Overwrite the object at the right pool
 	return z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
 }

@@ -570,8 +577,8 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	if z.SingleZone() {
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
-	for _, zone := range z.serverPools {
-		objInfo, err = zone.DeleteObject(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		objInfo, err = pool.DeleteObject(ctx, bucket, object, opts)
 		if err == nil {
 			return objInfo, nil
 		}
@@ -608,8 +615,8 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		return z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
 	}

-	for _, zone := range z.serverPools {
-		deletedObjects, errs := zone.DeleteObjects(ctx, bucket, objects, opts)
+	for _, pool := range z.serverPools {
+		deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objects, opts)
 		for i, derr := range errs {
 			if derr != nil {
 				derrs[i] = derr
@@ -626,7 +633,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec

 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))

-	zoneIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
+	poolIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
@@ -634,12 +641,12 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 	if cpSrcDstSame && srcInfo.metadataOnly {
 		// Version ID is set for the destination and source == destination version ID.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
-			return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+			return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		// Destination is not versioned and source version ID is empty
 		// perform an in-place update.
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
-			return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+			return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		// Destination is versioned, source is not destination version,
 		// as a special case look for if the source object is not legacy
@@ -649,7 +656,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 			// CopyObject optimization where we don't create an entire copy
 			// of the content, instead we add a reference.
 			srcInfo.versionOnly = true
-			return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+			return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 	}

@@ -661,7 +668,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		MTime:                dstOpts.MTime,
 	}

-	return z.serverPools[zoneIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
+	return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
 }

 func (z *erasureServerPools) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
@@ -783,20 +790,20 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
 		return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	}

-	var zoneResult = ListMultipartsInfo{}
-	zoneResult.MaxUploads = maxUploads
-	zoneResult.KeyMarker = keyMarker
-	zoneResult.Prefix = prefix
-	zoneResult.Delimiter = delimiter
-	for _, zone := range z.serverPools {
-		result, err := zone.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
+	var poolResult = ListMultipartsInfo{}
+	poolResult.MaxUploads = maxUploads
+	poolResult.KeyMarker = keyMarker
+	poolResult.Prefix = prefix
+	poolResult.Delimiter = delimiter
+	for _, pool := range z.serverPools {
+		result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
 			delimiter, maxUploads)
 		if err != nil {
 			return result, err
 		}
-		zoneResult.Uploads = append(zoneResult.Uploads, result.Uploads...)
+		poolResult.Uploads = append(poolResult.Uploads, result.Uploads...)
 	}
-	return zoneResult, nil
+	return poolResult, nil
 }

 // Initiate a new multipart upload on a hashedSet based on object name.
@@ -838,14 +845,14 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}

-	for _, zone := range z.serverPools {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
+			return pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
-			// Look for information on the next zone
+			// Look for information on the next pool
 			continue
 		}
 		// Any other unhandled errors such as quorum return.
@@ -867,14 +874,14 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 	if z.SingleZone() {
 		return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
-	for _, zone := range z.serverPools {
-		mi, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		mi, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
 			return mi, nil
 		}
 		switch err.(type) {
 		case InvalidUploadID:
-			// upload id not found, continue to the next zone.
+			// upload id not found, continue to the next pool.
 			continue
 		}
 		// any other unhandled error return right here.
@@ -897,10 +904,10 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 	if z.SingleZone() {
 		return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
-	for _, zone := range z.serverPools {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
+			return pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
@@ -925,14 +932,14 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}

-	for _, zone := range z.serverPools {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	for _, pool := range z.serverPools {
+		_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
+			return pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
-			// upload id not found move to next zone
+			// upload id not found move to next pool
 			continue
 		}
 		return err
@@ -955,17 +962,17 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 	}

 	// Purge any existing object.
-	for _, zone := range z.serverPools {
-		zone.DeleteObject(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		pool.DeleteObject(ctx, bucket, object, opts)
 	}

-	for _, zone := range z.serverPools {
-		result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
+	for _, pool := range z.serverPools {
+		result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
 		if err != nil {
 			return objInfo, err
 		}
 		if result.Lookup(uploadID) {
-			return zone.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
+			return pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 		}
 	}
 	return objInfo, InvalidUploadID{
@@ -988,8 +995,8 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (
 		}
 		return bucketInfo, nil
 	}
-	for _, zone := range z.serverPools {
-		bucketInfo, err = zone.GetBucketInfo(ctx, bucket)
+	for _, pool := range z.serverPools {
+		bucketInfo, err = pool.GetBucketInfo(ctx, bucket)
 		if err != nil {
 			if isErrBucketNotFound(err) {
 				continue
@@ -1114,8 +1121,8 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketI
 	if z.SingleZone() {
 		buckets, err = z.serverPools[0].ListBuckets(ctx)
 	} else {
-		for _, zone := range z.serverPools {
-			buckets, err = zone.ListBuckets(ctx)
+		for _, pool := range z.serverPools {
+			buckets, err = pool.ListBuckets(ctx)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				continue
@@ -1149,8 +1156,8 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 	}

 	var countNoHeal int
-	for _, zone := range z.serverPools {
-		result, err := zone.HealFormat(ctx, dryRun)
+	for _, pool := range z.serverPools {
+		result, err := pool.HealFormat(ctx, dryRun)
 		if err != nil && !errors.Is(err, errNoHealRequired) {
 			logger.LogIf(ctx, err)
 			continue
@@ -1183,8 +1190,8 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 	// Attempt heal on the bucket metadata, ignore any failures
 	_, _ = z.HealObject(ctx, minioMetaBucket, pathJoin(bucketConfigPrefix, bucket, bucketMetadataFile), "", opts)

-	for _, zone := range z.serverPools {
-		result, err := zone.HealBucket(ctx, bucket, opts)
+	for _, pool := range z.serverPools {
+		result, err := pool.HealBucket(ctx, bucket, opts)
 		if err != nil {
 			switch err.(type) {
 			case BucketNotFound:
@@ -1344,8 +1351,8 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 		defer lk.RUnlock()
 	}

-	for _, zone := range z.serverPools {
-		result, err := zone.HealObject(ctx, bucket, object, versionID, opts)
+	for _, pool := range z.serverPools {
+		result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -1374,12 +1381,12 @@ func (z *erasureServerPools) GetMetrics(ctx context.Context) (*Metrics, error) {
 }

 func (z *erasureServerPools) getZoneAndSet(id string) (int, int, error) {
-	for zoneIdx := range z.serverPools {
-		format := z.serverPools[zoneIdx].format
+	for poolIdx := range z.serverPools {
+		format := z.serverPools[poolIdx].format
 		for setIdx, set := range format.Erasure.Sets {
 			for _, diskID := range set {
 				if diskID == id {
-					return zoneIdx, setIdx, nil
+					return poolIdx, setIdx, nil
 				}
 			}
 		}
@@ -1419,12 +1426,12 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea

 	for _, localDiskIDs := range diskIDs {
 		for _, id := range localDiskIDs {
-			zoneIdx, setIdx, err := z.getZoneAndSet(id)
+			poolIdx, setIdx, err := z.getZoneAndSet(id)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				continue
 			}
-			erasureSetUpCount[zoneIdx][setIdx]++
+			erasureSetUpCount[poolIdx][setIdx]++
 		}
 	}

@@ -1461,16 +1468,16 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		}
 	}

-	for zoneIdx := range erasureSetUpCount {
-		for setIdx := range erasureSetUpCount[zoneIdx] {
-			if erasureSetUpCount[zoneIdx][setIdx] < writeQuorum {
+	for poolIdx := range erasureSetUpCount {
+		for setIdx := range erasureSetUpCount[poolIdx] {
+			if erasureSetUpCount[poolIdx][setIdx] < writeQuorum {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
-					fmt.Errorf("Write quorum may be lost on zone: %d, set: %d, expected write quorum: %d",
-						zoneIdx, setIdx, writeQuorum))
+					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
+						poolIdx, setIdx, writeQuorum))
 				return HealthResult{
 					Healthy:       false,
 					HealingDrives: len(aggHealStateResult.HealDisks),
-					ZoneID:        zoneIdx,
+					ZoneID:        poolIdx,
 					SetID:         setIdx,
 					WriteQuorum:   writeQuorum,
 				}
@@ -1501,8 +1508,8 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}

-	for _, zone := range z.serverPools {
-		err := zone.PutObjectTags(ctx, bucket, object, tags, opts)
+	for _, pool := range z.serverPools {
+		err := pool.PutObjectTags(ctx, bucket, object, tags, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -1530,8 +1537,8 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 	if z.SingleZone() {
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
-	for _, zone := range z.serverPools {
-		err := zone.DeleteObjectTags(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		err := pool.DeleteObjectTags(ctx, bucket, object, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -1559,8 +1566,8 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 	if z.SingleZone() {
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
-	for _, zone := range z.serverPools {
-		tags, err := zone.GetObjectTags(ctx, bucket, object, opts)
+	for _, pool := range z.serverPools {
+		tags, err := pool.GetObjectTags(ctx, bucket, object, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -271,8 +271,8 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 	return fr, st.Size(), nil
 }

-// Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer.
-func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) {
+// Creates a file and copies data from incoming reader.
+func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, fallocSize int64) (int64, error) {
 	if filePath == "" || reader == nil {
 		logger.LogIf(ctx, errInvalidArgument)
 		return 0, errInvalidArgument
@@ -317,21 +317,10 @@ func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []
 		}
 	}

-	var bytesWritten int64
-	if buf != nil {
-		bytesWritten, err = io.CopyBuffer(writer, reader, buf)
-		if err != nil {
-			if err != io.ErrUnexpectedEOF {
-				logger.LogIf(ctx, err)
-			}
-			return 0, err
-		}
-	} else {
-		bytesWritten, err = io.Copy(writer, reader)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return 0, err
-		}
+	bytesWritten, err := io.Copy(writer, reader)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return 0, err
 	}

 	return bytesWritten, nil
@@ -75,7 +75,7 @@ func TestFSStats(t *testing.T) {
 	}

 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -85,7 +85,7 @@ func TestFSStats(t *testing.T) {
 		t.Fatal("Unexpected error", err)
 	}

-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "path/to/success-file"), reader, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -192,7 +192,7 @@ func TestFSCreateAndOpen(t *testing.T) {
 		t.Fatalf("Unable to create directory, %s", err)
 	}

-	if _, err = fsCreateFile(GlobalContext, "", nil, nil, 0); err != errInvalidArgument {
+	if _, err = fsCreateFile(GlobalContext, "", nil, 0); err != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}

@@ -201,7 +201,7 @@ func TestFSCreateAndOpen(t *testing.T) {
 	}

 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -229,7 +229,7 @@ func TestFSCreateAndOpen(t *testing.T) {
 	}

 	for i, testCase := range testCases {
-		_, err = fsCreateFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
+		_, err = fsCreateFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), reader, 0)
 		if err != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
@@ -258,9 +258,8 @@ func TestFSDeletes(t *testing.T) {
 		t.Fatalf("Unable to create directory, %s", err)
 	}

-	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, reader.Size()); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -396,13 +395,13 @@ func TestFSRemoves(t *testing.T) {
 	}

 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
 	reader.Seek(0, 0)

-	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file-new"), reader, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -515,7 +514,7 @@ func TestFSRemoveMeta(t *testing.T) {
 	filePath := pathJoin(fsPath, "success-vol", "success-file")

 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(GlobalContext, filePath, reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, filePath, reader, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}

@@ -313,14 +313,8 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		return pi, toObjectErr(err, bucket, object)
 	}

-	bufSize := int64(readSizeV1)
-	if size := data.Size(); size > 0 && bufSize > size {
-		bufSize = size
-	}
-	buf := make([]byte, bufSize)
-
 	tmpPartPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID+"."+mustGetUUID()+"."+strconv.Itoa(partID))
-	bytesWritten, err := fsCreateFile(ctx, tmpPartPath, data, buf, data.Size())
+	bytesWritten, err := fsCreateFile(ctx, tmpPartPath, data, data.Size())

 	// Delete temporary part in case of failure. If
 	// PutObjectPart succeeds then there would be nothing to
@@ -857,11 +857,6 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
 	}
 	defer reader.Close()

-	bufSize := int64(readSizeV1)
-	if length > 0 && bufSize > length {
-		bufSize = length
-	}
-
 	// For negative length we read everything.
 	if length < 0 {
 		length = size - offset
@@ -874,10 +869,7 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
 		return err
 	}

-	// Allocate a staging buffer.
-	buf := make([]byte, int(bufSize))
-
-	_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
+	_, err = io.Copy(writer, io.LimitReader(reader, length))
 	// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
 	if err == io.ErrClosedPipe {
 		err = nil
@@ -1199,15 +1191,8 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	// so that cleaning it up will be easy if the server goes down.
 	tempObj := mustGetUUID()

-	// Allocate a buffer to Read() from request body
-	bufSize := int64(readSizeV1)
-	if size := data.Size(); size > 0 && bufSize > size {
-		bufSize = size
-	}
-
-	buf := make([]byte, int(bufSize))
 	fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
-	bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size())
+	bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, data.Size())

 	// Delete the temporary object in the case of a
 	// failure. If PutObject succeeds, then there would be
@@ -58,10 +58,10 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")

 	// Colorize the message and print.
-	logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
+	logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(color.Blue("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 	}
 	printEventNotifiers()

@@ -51,30 +51,19 @@ import (
 	minio "github.com/minio/minio/cmd"
 )

-var (
-	azureUploadChunkSize   = getUploadChunkSizeFromEnv(azureChunkSizeEnvVar, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
-	azureSdkTimeout        = time.Duration(azureUploadChunkSize/humanize.MiByte) * azureSdkTimeoutPerMb
-	azureUploadConcurrency = azureUploadMaxMemoryUsage / azureUploadChunkSize
+const (
+	azureDefaultUploadChunkSizeMB = 25
+	azureDownloadRetryAttempts    = 5
+	azureS3MinPartSize            = 5 * humanize.MiByte
+	metadataObjectNameTemplate    = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
+	azureMarkerPrefix             = "{minio}"
+	metadataPartNamePrefix        = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x"
+	maxPartsCount                 = 10000
 )

-const (
-	// The defaultDialTimeout for communicating with the cloud backends is set
-	// to 30 seconds in utils.go; the Azure SDK recommends to set a timeout of 60
-	// seconds per MB of data a client expects to upload so we must transfer less
-	// than 0.5 MB per chunk to stay within the defaultDialTimeout tolerance.
-	// See https://github.com/Azure/azure-storage-blob-go/blob/fc70003/azblob/zc_policy_retry.go#L39-L44 for more details.
-	// To change the upload chunk size, set the environmental variable MINIO_AZURE_CHUNK_SIZE_MB with a (float) value between 0 and 100
-	azureDefaultUploadChunkSize = 25 * humanize.MiByte
-	azureSdkTimeoutPerMb        = 60 * time.Second
-	azureUploadMaxMemoryUsage   = 100 * humanize.MiByte
-	azureChunkSizeEnvVar        = "MINIO_AZURE_CHUNK_SIZE_MB"
-
-	azureDownloadRetryAttempts = 5
-	azureS3MinPartSize         = 5 * humanize.MiByte
-	metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
-	azureMarkerPrefix          = "{minio}"
-	metadataPartNamePrefix     = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x"
-	maxPartsCount              = 10000
+var (
+	azureUploadChunkSize   int
+	azureUploadConcurrency int
 )

 func init() {
@@ -92,22 +81,21 @@ ENDPOINT:

 EXAMPLES:
  1. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}0.25
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}azureaccountname
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}azureaccountkey
     {{.Prompt}} {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint

  2. Start minio gateway server for Azure Blob Storage backend with edge caching enabled.
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}azureaccountname
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}azureaccountkey
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}25
     {{.Prompt}} {{.HelpName}}
+
 `

 	minio.RegisterGatewayCommand(cli.Command{
@@ -140,27 +128,6 @@ func azureGatewayMain(ctx *cli.Context) {
 	minio.StartGateway(ctx, &Azure{host})
 }

-// getUploadChunkSizeFromEnv returns the parsed chunk size from the environmental variable 'MINIO_AZURE_CHUNK_SIZE_MB'
-// The environmental variable should be a floating point number between 0 and 100 representing the MegaBytes
-// The returned value is an int representing the size in bytes
-func getUploadChunkSizeFromEnv(envvar string, defaultValue string) int {
-	envChunkSize := env.Get(envvar, defaultValue)
-
-	i, err := strconv.ParseFloat(envChunkSize, 64)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return azureDefaultUploadChunkSize
-	}
-
-	if i <= 0 || i > 100 {
-		logger.LogIf(context.Background(), fmt.Errorf("ENV '%v' should be a floating point value between 0 and 100.\n"+
-			"The upload chunk size is set to its default: %s\n", azureChunkSizeEnvVar, defaultValue))
-		return azureDefaultUploadChunkSize
-	}
-
-	return int(i * humanize.MiByte)
-}
-
 // Azure implements Gateway.
 type Azure struct {
 	host string
@@ -188,6 +155,20 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 		return nil, err
 	}

+	azureUploadChunkSize, err = env.GetInt("MINIO_AZURE_CHUNK_SIZE_MB", azureDefaultUploadChunkSizeMB)
+	if err != nil {
+		return nil, err
+	}
+	azureUploadChunkSize *= humanize.MiByte
+	if azureUploadChunkSize <= 0 || azureUploadChunkSize > 100*humanize.MiByte {
+		return nil, fmt.Errorf("MINIO_AZURE_CHUNK_SIZE_MB should be an integer value between 0 and 100")
+	}
+
+	azureUploadConcurrency, err = env.GetInt("MINIO_AZURE_UPLOAD_CONCURRENCY", 4)
+	if err != nil {
+		return nil, err
+	}
+
 	credential, err := azblob.NewSharedKeyCredential(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		if _, ok := err.(base64.CorruptInputError); ok {
@@ -208,7 +189,9 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro

 	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
 		Retry: azblob.RetryOptions{
-			TryTimeout: azureSdkTimeout,
+			// Azure SDK recommends to set a timeout of 60 seconds per MB of data so we
+			// calculate here the timeout for the configured upload chunck size.
+			TryTimeout: time.Duration(azureUploadChunkSize/humanize.MiByte) * 60 * time.Second,
 		},
 		HTTPSender: pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
 			return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
@@ -914,7 +897,6 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	blobURL := a.client.NewContainerURL(bucket).NewBlockBlobURL(object)

 	_, err = azblob.UploadStreamToBlockBlob(ctx, data, blobURL, azblob.UploadStreamToBlockBlobOptions{
-		BufferSize:      azureUploadChunkSize,
 		MaxBuffers:      azureUploadConcurrency,
 		BlobHTTPHeaders: properties,
 		Metadata:        metadata,
@@ -20,13 +20,9 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net/http"
-	"os"
 	"reflect"
-	"strconv"
 	"testing"

-	"github.com/dustin/go-humanize"
-
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	minio "github.com/minio/minio/cmd"
 )
@@ -268,36 +264,3 @@ func TestCheckAzureUploadID(t *testing.T) {
 		}
 	}
 }
-
-func TestParsingUploadChunkSize(t *testing.T) {
-	key := "MINIO_AZURE_CHUNK_SIZE_MB"
-	invalidValues := []string{
-		"",
-		"0,3",
-		"100.1",
-		"-1",
-	}
-
-	for i, chunkValue := range invalidValues {
-		os.Setenv(key, chunkValue)
-		result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
-		if result != azureDefaultUploadChunkSize {
-			t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result)
-		}
-	}
-
-	validValues := []string{
-		"1",
-		"1.25",
-		"50",
-		"99",
-	}
-	for i, chunkValue := range validValues {
-		os.Setenv(key, chunkValue)
-		result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
-		if result == azureDefaultUploadChunkSize {
-			t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result)
-		}
-	}
-
-}
@@ -112,14 +112,14 @@ GOOGLE_APPLICATION_CREDENTIALS:
 EXAMPLES:
  1. Start minio gateway server for GCS backend
     {{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.HelpName}} mygcsprojectid

  2. Start minio gateway server for GCS backend with edge caching enabled
     {{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
@@ -65,13 +65,13 @@ HDFS-NAMENODE:

 EXAMPLES:
  1. Start minio gateway server for HDFS backend
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.HelpName}} hdfs://namenode:8200

  2. Start minio gateway server for HDFS with edge caching enabled
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
@@ -39,13 +39,13 @@ PATH:

 EXAMPLES:
  1. Start minio gateway server for NAS backend
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.HelpName}} /shared/nasvol

  2. Start minio gateway server for NAS with edge caching enabled
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
@@ -55,13 +55,13 @@ ENDPOINT:

 EXAMPLES:
  1. Start minio gateway server for AWS S3 backend
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.HelpName}}

  2. Start minio gateway server for AWS S3 backend with edge caching enabled
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
@@ -136,7 +136,7 @@ func randString(n int, src rand.Source, prefix string) string {
 // Chains all credential types, in the following order:
 //  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
 //  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
-//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
+//  - Static credentials provided by user (i.e. MINIO_ROOT_USER)
 var defaultProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
@@ -192,6 +192,8 @@ var (

 	globalEndpoints EndpointServerPools

+	globalRemoteEndpoints map[string]Endpoint
+
 	// Global server's network statistics
 	globalConnStats = newConnStats()

@@ -57,7 +57,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
 		// max requests per node is calculated as
 		// total_ram / ram_per_request
 		// ram_per_request is 4MiB * setDriveCount + 2 * 10MiB (default erasure block size)
-		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*readBlockSize+blockSizeV1*2))
+		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*(blockSizeLarge+blockSizeSmall)+blockSizeV1*2))
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
 		if len(globalEndpoints.Hostnames()) > 0 {
@@ -18,7 +18,6 @@ package http

 import (
 	"context"
-	"log"
 	"math/rand"
 	"net"
 	"sync"
@@ -96,6 +95,7 @@ type DNSCache struct {

 	lookupHostFn  func(ctx context.Context, host string) ([]string, error)
 	lookupTimeout time.Duration
+	loggerOnce    func(ctx context.Context, err error, id interface{}, errKind ...interface{})

 	cache    map[string][]string
 	doneOnce sync.Once
@@ -105,7 +105,7 @@ type DNSCache struct {
 // NewDNSCache initializes DNS cache resolver and starts auto refreshing
 // in a new goroutine. To stop auto refreshing, call `Stop()` function.
 // Once `Stop()` is called auto refreshing cannot be resumed.
-func NewDNSCache(freq time.Duration, lookupTimeout time.Duration) *DNSCache {
+func NewDNSCache(freq time.Duration, lookupTimeout time.Duration, loggerOnce func(ctx context.Context, err error, id interface{}, errKind ...interface{})) *DNSCache {
 	if freq <= 0 {
 		freq = defaultFreq
 	}
@@ -117,6 +117,7 @@ func NewDNSCache(freq time.Duration, lookupTimeout time.Duration) *DNSCache {
 	r := &DNSCache{
 		lookupHostFn:  net.DefaultResolver.LookupHost,
 		lookupTimeout: lookupTimeout,
+		loggerOnce:    loggerOnce,
 		cache:         make(map[string][]string, cacheSize),
 		doneCh:        make(chan struct{}),
 	}
@@ -179,7 +180,7 @@ func (r *DNSCache) Refresh() {
 	for _, host := range hosts {
 		ctx, cancelF := context.WithTimeout(context.Background(), r.lookupTimeout)
 		if _, err := r.LookupHost(ctx, host); err != nil {
-			log.Println("failed to refresh DNS cache, resolver is unavailable", err)
+			r.loggerOnce(ctx, err, host)
 		}
 		cancelF()
 	}
@@ -31,9 +31,13 @@ var (
 	testDefaultLookupTimeout = 1 * time.Second
 )

+func logOnce(ctx context.Context, err error, id interface{}, errKind ...interface{}) {
+	// no-op
+}
+
 func testDNSCache(t *testing.T) *DNSCache {
 	t.Helper() // skip printing file and line information from this function
-	return NewDNSCache(testFreq, testDefaultLookupTimeout)
+	return NewDNSCache(testFreq, testDefaultLookupTimeout, logOnce)
 }

 func TestDialContextWithDNSCache(t *testing.T) {
@@ -163,6 +163,8 @@ const (
 	MinIODeleteReplicationStatus = "X-Minio-Replication-Delete-Status"
 	// Header indicates delete-marker replication status.
 	MinIODeleteMarkerReplicationStatus = "X-Minio-Replication-DeleteMarker-Status"
+	// Header indicates if its a GET/HEAD proxy request for active-active replication
+	MinIOSourceProxyRequest = "x-minio-source-proxy-request"
 )

 // Common http query params S3 API
@@ -444,6 +444,9 @@ func (sys *IAMSys) Initialized() bool {

 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
+	// Initialize IAM store
+	sys.InitStore(objAPI)
+
 	retryCtx, cancel := context.WithCancel(ctx)

 	// Indicate to our routine to exit cleanly upon return.
@@ -228,6 +228,11 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 				// Check whether uid is still active
 				for _, entry := range lri {
 					if entry.UID == args.UID && entry.Owner == args.Owner {
+						if ep, ok := globalRemoteEndpoints[args.Owner]; ok {
+							if err = isServerResolvable(ep); err != nil {
+								return true, nil
+							}
+						}
 						return false, nil
 					}
 				}
@@ -21,6 +21,7 @@ import (
 	"errors"
 	"io"
 	"path"
+	"strings"
 	"sync"
 	"time"

@@ -53,11 +54,10 @@ func (z *erasureServerPools) listPath(ctx context.Context, o listPathOptions) (e
 	}

 	// For delimiter and prefix as '/' we do not list anything at all
-	// since according to s3 spec we stop at the 'delimiter'
 	// along // with the prefix. On a flat namespace with 'prefix'
 	// as '/' we don't have any entries, since all the keys are
 	// of form 'keyName/...'
-	if o.Separator == SlashSeparator && o.Prefix == SlashSeparator {
+	if strings.HasPrefix(o.Prefix, SlashSeparator) {
 		return entries, io.EOF
 	}

@@ -149,8 +149,8 @@ func (z *erasureServerPools) listPath(ctx context.Context, o listPathOptions) (e
 	allAtEOF := true
 	mu.Lock()
 	// Ask all sets and merge entries.
-	for _, zone := range z.serverPools {
-		for _, set := range zone.sets {
+	for _, pool := range z.serverPools {
+		for _, set := range pool.sets {
 			wg.Add(1)
 			go func(i int, set *erasureObjects) {
 				defer wg.Done()
@@ -388,7 +388,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				continue
 			}

-			_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(0), "")
+			_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(0), "", false)
 			if err != nil {
 				time.Sleep(retryDelay)
 				retries++
@@ -397,7 +397,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 		}

 		// Read metadata associated with the object from all disks.
-		fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{})
+		fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{}, true)
 		if err != nil {
 			switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) {
 			case ObjectNotFound:
@@ -463,7 +463,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 						continue
 					}

-					_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(partN), "")
+					_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(partN), "", false)
 					if err != nil {
 						time.Sleep(retryDelay)
 						retries++
@@ -471,7 +471,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 					}
 				}
 				// Load first part metadata...
-				fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{})
+				fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true)
 				if err != nil {
 					time.Sleep(retryDelay)
 					retries++
@@ -654,11 +654,11 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
 				meta.endedCycle = intDataUpdateTracker.current()
 				meta, err = o.updateMetacacheListing(meta, rpc)
 				if meta.status == scanStateError {
+					logger.LogIf(ctx, err)
 					cancel()
 					exit = true
 				}
 				metaMu.Unlock()
-				logger.LogIf(ctx, err)
 			}
 		}()

@@ -772,7 +772,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
 				metaMu.Lock()
 				meta.error = err.Error()
 				meta.status = scanStateError
-				meta, err = o.updateMetacacheListing(meta, rpc)
+				meta, _ = o.updateMetacacheListing(meta, rpc)
 				metaMu.Unlock()
 			}
 		}
@@ -834,15 +834,15 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 		}
 		// Send request to each disk.
 		go func() {
-			err := d.WalkDir(ctx, WalkDirOptions{
+			werr := d.WalkDir(ctx, WalkDirOptions{
 				Bucket:         opts.bucket,
 				BaseDir:        opts.path,
 				Recursive:      opts.recursive,
 				ReportNotFound: opts.reportNotFound,
 				FilterPrefix:   opts.filterPrefix}, w)
-			w.CloseWithError(err)
-			if err != io.EOF && err != nil && err.Error() != errFileNotFound.Error() {
-				logger.LogIf(ctx, err)
+			w.CloseWithError(werr)
+			if werr != io.EOF && werr != nil && werr.Error() != errFileNotFound.Error() && werr.Error() != errVolumeNotFound.Error() {
+				logger.LogIf(ctx, werr)
 			}
 		}()
 	}
@@ -878,7 +878,11 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 					fnf++
 					continue
 				}
-
+				if err.Error() == errVolumeNotFound.Error() {
+					atEOF++
+					fnf++
+					continue
+				}
 				hasErr++
 				errs[i] = err
 				continue
@@ -252,11 +252,11 @@ func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi
 	return d.disk.DeleteVersion(ctx, volume, path, fi)
 }

-func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
+func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
 	if err := d.calcError(); err != nil {
 		return FileInfo{}, err
 	}
-	return d.disk.ReadVersion(ctx, volume, path, versionID)
+	return d.disk.ReadVersion(ctx, volume, path, versionID, readData)
 }

 func (d *naughtyDisk) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) {
@@ -31,9 +31,6 @@ const (
 	// Block size used for all internal operations version 1.
 	blockSizeV1 = 10 * humanize.MiByte

-	// Staging buffer read size for all internal operations version 1.
-	readSizeV1 = 1 * humanize.MiByte
-
 	// Buckets meta prefix.
 	bucketMetaPrefix = "buckets"

@@ -31,7 +31,7 @@ import (

 // Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup.
 func TestGetObject(t *testing.T) {
-	ExecObjectLayerTest(t, testGetObject)
+	ExecExtendedObjectLayerTest(t, testGetObject)
 }

 // ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.
@@ -52,6 +52,8 @@ type ObjectOptions struct {
 	VersionPurgeStatus            VersionPurgeStatusType // Is only set in DELETE operations for delete marker version to be permanently deleted.
 	TransitionStatus              string                 // status of the transition
 	NoLock                        bool                   // indicates to lower layers if the caller is expecting to hold locks.
+	ProxyRequest                  bool                   // only set for GET/HEAD in active-active replication scenario
+
 }

 // BucketOptions represents bucket options for ObjectLayer bucket operations
@@ -531,6 +531,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		{"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true},
 		{"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true},
 		{"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true},
+		{"test-bucket-list-object", "/obj", "", "", 0, ListObjectsInfo{}, nil, true},
 		// Testing with prefix and truncation (29-30).
 		{"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true},
 		{"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true},
@@ -157,7 +157,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE

 // Wrapper for calling PutObjectPart tests for both Erasure multiple disks and single node setup.
 func TestObjectAPIPutObjectPart(t *testing.T) {
-	ExecObjectLayerTest(t, testObjectAPIPutObjectPart)
+	ExecExtendedObjectLayerTest(t, testObjectAPIPutObjectPart)
 }

 // Tests validate correctness of PutObjectPart.
@@ -289,7 +289,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH

 // Wrapper for calling TestListMultipartUploads tests for both Erasure multiple disks and single node setup.
 func TestListMultipartUploads(t *testing.T) {
-	ExecObjectLayerTest(t, testListMultipartUploads)
+	ExecExtendedObjectLayerTest(t, testListMultipartUploads)
 }

 // testListMultipartUploads - Tests validate listing of multipart uploads.
@@ -1643,7 +1643,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)

 // Test for validating complete Multipart upload.
 func TestObjectCompleteMultipartUpload(t *testing.T) {
-	ExecObjectLayerTest(t, testObjectCompleteMultipartUpload)
+	ExecExtendedObjectLayerTest(t, testObjectCompleteMultipartUpload)
 }

 // Tests validate CompleteMultipart functionality.
@@ -66,6 +66,7 @@ func getDefaultOpts(header http.Header, copySource bool, metadata map[string]str
 	if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
 		opts.ServerSideEncryption = encrypt.NewSSE()
 	}
+	opts.ProxyRequest = header.Get(xhttp.MinIOSourceProxyRequest) == "true"
 	return
 }

@@ -37,7 +37,7 @@ func md5Header(data []byte) map[string]string {

 // Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup.
 func TestObjectAPIPutObjectSingle(t *testing.T) {
-	ExecObjectLayerTest(t, testObjectAPIPutObject)
+	ExecExtendedObjectLayerTest(t, testObjectAPIPutObject)
 }

 // Tests validate correctness of PutObject.
@@ -394,9 +394,6 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
 	if !ok {
 		return false, nil
 	}
-	if crypto.IsEncrypted(o.UserDefined) {
-		return true, fmt.Errorf("compression %q and encryption enabled on same object", scheme)
-	}
 	switch scheme {
 	case compressionAlgorithmV1, compressionAlgorithmV2:
 		return true, nil
@@ -415,9 +412,6 @@ func (o ObjectInfo) GetActualETag(h http.Header) string {

 // GetActualSize - returns the actual size of the stored object
 func (o ObjectInfo) GetActualSize() (int64, error) {
-	if crypto.IsEncrypted(o.UserDefined) {
-		return o.DecryptedSize()
-	}
 	if o.IsCompressed() {
 		sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
 		if !ok {
@@ -429,6 +423,10 @@ func (o ObjectInfo) GetActualSize() (int64, error) {
 		}
 		return size, nil
 	}
+	if crypto.IsEncrypted(o.UserDefined) {
+		return o.DecryptedSize()
+	}
+
 	return o.Size, nil
 }

@@ -441,7 +439,7 @@ func isCompressible(header http.Header, object string) bool {
 	globalCompressConfigMu.Unlock()

 	_, ok := crypto.IsRequested(header)
-	if !cfg.Enabled || ok || excludeForCompression(header, object, cfg) {
+	if !cfg.Enabled || (ok && !cfg.AllowEncrypted) || excludeForCompression(header, object, cfg) {
 		return false
 	}
 	return true
@@ -461,16 +459,15 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi
 	}

 	// Filter compression includes.
-	if len(cfg.Extensions) == 0 || len(cfg.MimeTypes) == 0 {
-		return false
+	exclude := len(cfg.Extensions) > 0 || len(cfg.MimeTypes) > 0
+	if len(cfg.Extensions) > 0 && hasStringSuffixInSlice(objStr, cfg.Extensions) {
+		exclude = false
 	}

-	extensions := cfg.Extensions
-	mimeTypes := cfg.MimeTypes
-	if hasStringSuffixInSlice(objStr, extensions) || hasPattern(mimeTypes, contentType) {
-		return false
+	if len(cfg.MimeTypes) > 0 && hasPattern(cfg.MimeTypes, contentType) {
+		exclude = false
 	}
-	return true
+	return exclude
 }

 // Utility which returns if a string is present in the list.
@@ -520,21 +517,29 @@ func partNumberToRangeSpec(oi ObjectInfo, partNumber int) *HTTPRangeSpec {
 }

 // Returns the compressed offset which should be skipped.
-func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (int64, int64) {
-	var compressedOffset int64
+// If encrypted offsets are adjusted for encrypted block headers/trailers.
+// Since de-compression is after decryption encryption overhead is only added to compressedOffset.
+func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (compressedOffset int64, partSkip int64) {
 	var skipLength int64
 	var cumulativeActualSize int64
+	var firstPartIdx int
 	if len(objectInfo.Parts) > 0 {
-		for _, part := range objectInfo.Parts {
+		for i, part := range objectInfo.Parts {
 			cumulativeActualSize += part.ActualSize
 			if cumulativeActualSize <= offset {
 				compressedOffset += part.Size
 			} else {
+				firstPartIdx = i
 				skipLength = cumulativeActualSize - part.ActualSize
 				break
 			}
 		}
 	}
+	if isEncryptedMultipart(objectInfo) && firstPartIdx > 0 {
+		off, _, _, _, _, err := objectInfo.GetDecryptedRange(partNumberToRangeSpec(objectInfo, firstPartIdx))
+		logger.LogIf(context.Background(), err)
+		compressedOffset += off
+	}
 	return compressedOffset, offset - skipLength
 }

@@ -604,9 +609,92 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 		isEncrypted = false
 	}
 	var skipLen int64
-	// Calculate range to read (different for
-	// e.g. encrypted/compressed objects)
+	// Calculate range to read (different for encrypted/compressed objects)
 	switch {
+	case isCompressed:
+		// If compressed, we start from the beginning of the part.
+		// Read the decompressed size from the meta.json.
+		actualSize, err := oi.GetActualSize()
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		off, length = int64(0), oi.Size
+		decOff, decLength := int64(0), actualSize
+		if rs != nil {
+			off, length, err = rs.GetOffsetLength(actualSize)
+			if err != nil {
+				return nil, 0, 0, err
+			}
+			// In case of range based queries on multiparts, the offset and length are reduced.
+			off, decOff = getCompressedOffsets(oi, off)
+			decLength = length
+			length = oi.Size - off
+
+			// For negative length we read everything.
+			if decLength < 0 {
+				decLength = actualSize - decOff
+			}
+
+			// Reply back invalid range if the input offset and length fall out of range.
+			if decOff > actualSize || decOff+decLength > actualSize {
+				return nil, 0, 0, errInvalidRange
+			}
+		}
+		fn = func(inputReader io.Reader, h http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
+			cFns = append(cleanUpFns, cFns...)
+			if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
+				// Call the cleanup funcs
+				for i := len(cFns) - 1; i >= 0; i-- {
+					cFns[i]()
+				}
+				return nil, PreConditionFailed{}
+			}
+			if isEncrypted {
+				copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != ""
+				// Attach decrypter on inputReader
+				inputReader, err = DecryptBlocksRequestR(inputReader, h, 0, opts.PartNumber, oi, copySource)
+				if err != nil {
+					// Call the cleanup funcs
+					for i := len(cFns) - 1; i >= 0; i-- {
+						cFns[i]()
+					}
+					return nil, err
+				}
+			}
+			// Decompression reader.
+			s2Reader := s2.NewReader(inputReader)
+			// Apply the skipLen and limit on the decompressed stream.
+			err = s2Reader.Skip(decOff)
+			if err != nil {
+				// Call the cleanup funcs
+				for i := len(cFns) - 1; i >= 0; i-- {
+					cFns[i]()
+				}
+				return nil, err
+			}
+
+			decReader := io.LimitReader(s2Reader, decLength)
+			if decLength > compReadAheadSize {
+				rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize)
+				if err == nil {
+					decReader = rah
+					cFns = append(cFns, func() {
+						rah.Close()
+					})
+				}
+			}
+			oi.Size = decLength
+
+			// Assemble the GetObjectReader
+			r = &GetObjectReader{
+				ObjInfo:    oi,
+				pReader:    decReader,
+				cleanUpFns: cFns,
+				opts:       opts,
+			}
+			return r, nil
+		}
+
 	case isEncrypted:
 		var seqNumber uint32
 		var partStart int
@@ -635,8 +723,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 			cFns = append(cleanUpFns, cFns...)
 			// Attach decrypter on inputReader
 			var decReader io.Reader
-			decReader, err = DecryptBlocksRequestR(inputReader, h,
-				off, length, seqNumber, partStart, oi, copySource)
+			decReader, err = DecryptBlocksRequestR(inputReader, h, seqNumber, partStart, oi, copySource)
 			if err != nil {
 				// Call the cleanup funcs
 				for i := len(cFns) - 1; i >= 0; i-- {
@@ -659,76 +746,6 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 			// decrypted stream
 			decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength)

-			// Assemble the GetObjectReader
-			r = &GetObjectReader{
-				ObjInfo:    oi,
-				pReader:    decReader,
-				cleanUpFns: cFns,
-				opts:       opts,
-			}
-			return r, nil
-		}
-	case isCompressed:
-		// Read the decompressed size from the meta.json.
-		actualSize, err := oi.GetActualSize()
-		if err != nil {
-			return nil, 0, 0, err
-		}
-		off, length = int64(0), oi.Size
-		decOff, decLength := int64(0), actualSize
-		if rs != nil {
-			off, length, err = rs.GetOffsetLength(actualSize)
-			if err != nil {
-				return nil, 0, 0, err
-			}
-			// In case of range based queries on multiparts, the offset and length are reduced.
-			off, decOff = getCompressedOffsets(oi, off)
-			decLength = length
-			length = oi.Size - off
-
-			// For negative length we read everything.
-			if decLength < 0 {
-				decLength = actualSize - decOff
-			}
-
-			// Reply back invalid range if the input offset and length fall out of range.
-			if decOff > actualSize || decOff+decLength > actualSize {
-				return nil, 0, 0, errInvalidRange
-			}
-		}
-		fn = func(inputReader io.Reader, _ http.Header, pcfn CheckPreconditionFn, cFns ...func()) (r *GetObjectReader, err error) {
-			cFns = append(cleanUpFns, cFns...)
-			if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) {
-				// Call the cleanup funcs
-				for i := len(cFns) - 1; i >= 0; i-- {
-					cFns[i]()
-				}
-				return nil, PreConditionFailed{}
-			}
-			// Decompression reader.
-			s2Reader := s2.NewReader(inputReader)
-			// Apply the skipLen and limit on the decompressed stream.
-			err = s2Reader.Skip(decOff)
-			if err != nil {
-				// Call the cleanup funcs
-				for i := len(cFns) - 1; i >= 0; i-- {
-					cFns[i]()
-				}
-				return nil, err
-			}
-
-			decReader := io.LimitReader(s2Reader, decLength)
-			if decLength > compReadAheadSize {
-				rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize)
-				if err == nil {
-					decReader = rah
-					cFns = append(cFns, func() {
-						rah.Close()
-					})
-				}
-			}
-			oi.Size = decLength
-
 			// Assemble the GetObjectReader
 			r = &GetObjectReader{
 				ObjInfo:    oi,
@@ -317,7 +317,7 @@ func TestIsCompressed(t *testing.T) {
 		result  bool
 		err     bool
 	}{
-		{
+		0: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1,
 					"content-type": "application/octet-stream",
@@ -325,7 +325,7 @@ func TestIsCompressed(t *testing.T) {
 			},
 			result: true,
 		},
-		{
+		1: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
 					"content-type": "application/octet-stream",
@@ -333,7 +333,7 @@ func TestIsCompressed(t *testing.T) {
 			},
 			result: true,
 		},
-		{
+		2: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type",
 					"content-type": "application/octet-stream",
@@ -342,7 +342,7 @@ func TestIsCompressed(t *testing.T) {
 			result: true,
 			err:    true,
 		},
-		{
+		3: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
 					"content-type": "application/octet-stream",
@@ -351,9 +351,9 @@ func TestIsCompressed(t *testing.T) {
 				},
 			},
 			result: true,
-			err:    true,
+			err:    false,
 		},
-		{
+		4: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2",
 					"content-type": "application/octet-stream",
@@ -361,7 +361,7 @@ func TestIsCompressed(t *testing.T) {
 			},
 			result: false,
 		},
-		{
+		5: {
 			objInfo: ObjectInfo{
 				UserDefined: map[string]string{"content-type": "application/octet-stream",
 					"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
@@ -977,18 +977,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}

 	var reader io.Reader
-	var length = srcInfo.Size

-	// Set the actual size to the decrypted size if encrypted.
-	actualSize := srcInfo.Size
-	if crypto.IsEncrypted(srcInfo.UserDefined) {
-		actualSize, err = srcInfo.DecryptedSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		length = actualSize
+	// Set the actual size to the compressed/decrypted size if encrypted.
+	actualSize, err := srcInfo.GetActualSize()
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
 	}
+	length := actualSize

 	if !cpSrcDstSame {
 		if err := enforceBucketQuota(ctx, dstBucket, actualSize); err != nil {
@@ -1000,8 +996,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	var compressMetadata map[string]string
 	// No need to compress for remote etcd calls
 	// Pass the decompressed stream to such calls.
-	isCompressed := objectAPI.IsCompressionSupported() && isCompressible(r.Header, srcObject) && !isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
-	if isCompressed {
+	isDstCompressed := objectAPI.IsCompressionSupported() &&
+		isCompressible(r.Header, srcObject) &&
+		!isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI)
+	if isDstCompressed {
 		compressMetadata = make(map[string]string, 2)
 		// Preserving the compression metadata.
 		compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
@@ -1034,7 +1032,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	_, objectEncryption := crypto.IsRequested(r.Header)
 	objectEncryption = objectEncryption || crypto.IsSourceEncrypted(srcInfo.UserDefined)
 	var encMetadata = make(map[string]string)
-	if objectAPI.IsEncryptionSupported() && !isCompressed {
+	if objectAPI.IsEncryptionSupported() {
 		// Encryption parameters not applicable for this object.
 		if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
@@ -1105,8 +1103,10 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			var targetSize int64

 			switch {
+			case isDstCompressed:
+				targetSize = -1
 			case !isSourceEncrypted && !isTargetEncrypted:
-				targetSize = srcInfo.Size
+				targetSize, _ = srcInfo.GetActualSize()
 			case isSourceEncrypted && isTargetEncrypted:
 				objInfo := ObjectInfo{Size: actualSize}
 				targetSize = objInfo.EncryptedSize()
@@ -1131,7 +1131,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			}

 			// do not try to verify encrypted content
-			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", targetSize, globalCLIContext.StrictS3Compat)
+			srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1193,7 +1193,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if rs := r.Header.Get(xhttp.AmzBucketReplicationStatus); rs != "" {
 		srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = rs
 	}
-	if mustReplicate(ctx, r, dstBucket, dstObject, srcInfo.UserDefined, srcInfo.ReplicationStatus.String()) {
+	if ok, _ := mustReplicate(ctx, r, dstBucket, dstObject, srcInfo.UserDefined, srcInfo.ReplicationStatus.String()); ok {
 		srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 	// Store the preserved compression metadata.
@@ -1273,8 +1273,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
-	if mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		globalReplicationState.queueReplicaTask(objInfo)
+	if replicate, sync := mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String()); replicate {
+		scheduleReplication(ctx, objInfo, objectAPI, sync)
 	}

 	setPutObjHeaders(w, objInfo, false)
@@ -1519,7 +1519,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if mustReplicate(ctx, r, bucket, object, metadata, "") {
+	if ok, _ := mustReplicate(ctx, r, bucket, object, metadata, ""); ok {
 		metadata[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 	if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
@@ -1541,10 +1541,15 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
-			info := ObjectInfo{Size: size}
+
+			wantSize := int64(-1)
+			if size >= 0 {
+				info := ObjectInfo{Size: size}
+				wantSize = info.EncryptedSize()
+			}

 			// do not try to verify encrypted content
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
+			hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1564,10 +1569,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}

 	switch {
-	case objInfo.IsCompressed():
-		if !strings.HasSuffix(objInfo.ETag, "-1") {
-			objInfo.ETag = objInfo.ETag + "-1"
-		}
 	case crypto.IsEncrypted(objInfo.UserDefined):
 		switch {
 		case crypto.S3.IsEncrypted(objInfo.UserDefined):
@@ -1581,9 +1582,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				objInfo.ETag = objInfo.ETag[len(objInfo.ETag)-32:]
 			}
 		}
+	case objInfo.IsCompressed():
+		if !strings.HasSuffix(objInfo.ETag, "-1") {
+			objInfo.ETag = objInfo.ETag + "-1"
+		}
 	}
-	if mustReplicate(ctx, r, bucket, object, metadata, "") {
-		globalReplicationState.queueReplicaTask(objInfo)
+	if replicate, sync := mustReplicate(ctx, r, bucket, object, metadata, ""); replicate {
+		scheduleReplication(ctx, objInfo, objectAPI, sync)
 	}
 	setPutObjHeaders(w, objInfo, false)

@@ -1699,7 +1704,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if mustReplicate(ctx, r, bucket, object, metadata, "") {
+	if ok, _ := mustReplicate(ctx, r, bucket, object, metadata, ""); ok {
 		metadata[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 	// We need to preserve the encryption headers set in EncryptRequest,
@@ -1892,7 +1897,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt

 	actualPartSize := srcInfo.Size
 	if crypto.IsEncrypted(srcInfo.UserDefined) {
-		actualPartSize, err = srcInfo.DecryptedSize()
+		actualPartSize, err = srcInfo.GetActualSize()
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -1991,7 +1996,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt

 	isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	var objectEncryptionKey crypto.ObjectKey
-	if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted {
+	if objectAPI.IsEncryptionSupported() && isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r))
 			return
@@ -2022,8 +2027,13 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			return
 		}

-		info := ObjectInfo{Size: length}
-		srcInfo.Reader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", length, globalCLIContext.StrictS3Compat)
+		wantSize := int64(-1)
+		if length >= 0 {
+			info := ObjectInfo{Size: length}
+			wantSize = info.EncryptedSize()
+		}
+
+		srcInfo.Reader, err = hash.NewReader(reader, wantSize, "", "", actualPartSize, globalCLIContext.StrictS3Compat)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -2226,7 +2236,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http

 	isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	var objectEncryptionKey crypto.ObjectKey
-	if objectAPI.IsEncryptionSupported() && !isCompressed && isEncrypted {
+	if objectAPI.IsEncryptionSupported() && isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL, guessIsBrowserReq(r))
 			return
@@ -2267,9 +2277,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		info := ObjectInfo{Size: size}
+		wantSize := int64(-1)
+		if size >= 0 {
+			info := ObjectInfo{Size: size}
+			wantSize = info.EncryptedSize()
+		}
 		// do not try to verify encrypted content
-		hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size, globalCLIContext.StrictS3Compat)
+		hashReader, err = hash.NewReader(reader, wantSize, "", "", actualSize, globalCLIContext.StrictS3Compat)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -2652,10 +2666,9 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}

 	setPutObjHeaders(w, objInfo, false)
-	if mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		globalReplicationState.queueReplicaTask(objInfo)
+	if replicate, sync := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String()); replicate {
+		scheduleReplication(ctx, objInfo, objectAPI, sync)
 	}
-
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)

@@ -2733,7 +2746,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			VersionID: opts.VersionID,
 		})
 	}
-	_, replicateDel := checkReplicateDelete(ctx, bucket, ObjectToDelete{ObjectName: object, VersionID: opts.VersionID}, goi, gerr)
+	_, replicateDel, replicateSync := checkReplicateDelete(ctx, bucket, ObjectToDelete{ObjectName: object, VersionID: opts.VersionID}, goi, gerr)
 	if replicateDel {
 		if opts.VersionID != "" {
 			opts.VersionPurgeStatus = Pending
@@ -2745,7 +2758,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
 		// check if replica has permission to be deleted.
 		if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object); apiErrCode != ErrNone {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL, guessIsBrowserReq(r))
 			return
 		}
 		opts.DeleteMarkerReplicationStatus = replication.Replica.String()
@@ -2794,7 +2807,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		} else {
 			versionID = objInfo.VersionID
 		}
-		globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+		dobj := DeletedObjectVersionInfo{
 			DeletedObject: DeletedObject{
 				ObjectName:                    object,
 				VersionID:                     versionID,
@@ -2805,7 +2818,8 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 				VersionPurgeStatus:            objInfo.VersionPurgeStatus,
 			},
 			Bucket: bucket,
-		})
+		}
+		scheduleReplicationDelete(ctx, dobj, objectAPI, replicateSync)
 	}

 	if goi.TransitionStatus == lifecycle.TransitionComplete { // clean up transitioned tier
@@ -2893,7 +2907,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 	if objInfo.UserTags != "" {
 		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-	replicate := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, "")
+	replicate, sync := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, "")
 	if replicate {
 		objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
@@ -2909,7 +2923,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 	if replicate {
-		globalReplicationState.queueReplicaTask(objInfo)
+		scheduleReplication(ctx, objInfo, objectAPI, sync)
 	}
 	writeSuccessResponseHeadersOnly(w)
 	// Notify object  event.
@@ -3066,7 +3080,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 	if objInfo.UserTags != "" {
 		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-	replicate := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, "")
+	replicate, sync := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, "")
 	if replicate {
 		objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
@@ -3081,7 +3095,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 	if replicate {
-		globalReplicationState.queueReplicaTask(objInfo)
+		scheduleReplication(ctx, objInfo, objectAPI, sync)
 	}

 	writeSuccessNoContent(w)
@@ -3248,7 +3262,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}

-	replicate := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: tags.String()}, "")
+	replicate, sync := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: tags.String()}, "")
 	if replicate {
 		opts.UserDefined = make(map[string]string)
 		opts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
@@ -3263,7 +3277,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h

 	if replicate {
 		if objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
-			globalReplicationState.queueReplicaTask(objInfo)
+			scheduleReplication(ctx, objInfo, objAPI, sync)
 		}
 	}

@@ -3313,7 +3327,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	replicate := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: oi.UserTags}, "")
+	replicate, sync := mustReplicate(ctx, r, bucket, object, map[string]string{xhttp.AmzObjectTagging: oi.UserTags}, "")
 	if replicate {
 		opts.UserDefined = make(map[string]string)
 		opts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
@@ -3329,7 +3343,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 	}

 	if replicate {
-		globalReplicationState.queueReplicaTask(oi)
+		scheduleReplication(ctx, oi, objAPI, sync)
 	}

 	writeSuccessNoContent(w)
@@ -327,7 +327,7 @@ func TestAPIGetObjectHandler(t *testing.T) {
 	defer func() { globalPolicySys = nil }()

 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"})
 }

 func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -651,7 +651,7 @@ func TestAPIGetObjectWithMPHandler(t *testing.T) {
 	defer func() { globalPolicySys = nil }()

 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
 }

 func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -849,7 +849,7 @@ func TestAPIGetObjectWithPartNumberHandler(t *testing.T) {
 	defer func() { globalPolicySys = nil }()

 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
 }

 func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -971,7 +971,7 @@ func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucket
 // Wrapper for calling PutObject API handler tests using streaming signature v4 for both Erasure multiple disks and FS single drive setup.
 func TestAPIPutObjectStreamSigV4Handler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"})
 }

 func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1289,7 +1289,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 // Wrapper for calling PutObject API handler tests for both Erasure multiple disks and FS single drive setup.
 func TestAPIPutObjectHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"})
 }

 func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1538,7 +1538,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 // expected.
 func TestAPICopyObjectPartHandlerSanity(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"CopyObjectPart"})
 }

 func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1649,7 +1649,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 // Wrapper for calling Copy Object Part API handler tests for both Erasure multiple disks and single node setup.
 func TestAPICopyObjectPartHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"})
 }

 func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -1965,7 +1965,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 // Wrapper for calling Copy Object API handler tests for both Erasure multiple disks and single node setup.
 func TestAPICopyObjectHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"})
+	ExecExtendedObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject"})
 }

 func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -2044,8 +2044,11 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// expected output.
 		expectedRespStatus int
 	}{
+		0: {
+			expectedRespStatus: http.StatusMethodNotAllowed,
+		},
 		// Test case - 1, copy metadata from newObject1, ignore request headers.
-		{
+		1: {
 			bucketName:       bucketName,
 			newObjectName:    "newObject1",
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2059,7 +2062,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,

 		// Test case - 2.
 		// Test case with invalid source object.
-		{
+		2: {
 			bucketName:       bucketName,
 			newObjectName:    "newObject1",
 			copySourceHeader: url.QueryEscape(SlashSeparator),
@@ -2071,7 +2074,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,

 		// Test case - 3.
 		// Test case with new object name is same as object to be copied.
-		{
+		3: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2084,7 +2087,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case - 4.
 		// Test case with new object name is same as object to be copied.
 		// But source copy is without leading slash
-		{
+		4: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(bucketName + SlashSeparator + objectName),
@@ -2097,7 +2100,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case - 5.
 		// Test case with new object name is same as object to be copied
 		// but metadata is updated.
-		{
+		5: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2113,7 +2116,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,

 		// Test case - 6.
 		// Test case with invalid metadata-directive.
-		{
+		6: {
 			bucketName:       bucketName,
 			newObjectName:    "newObject1",
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2130,7 +2133,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case - 7.
 		// Test case with new object name is same as object to be copied
 		// fail with BadRequest.
-		{
+		7: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2148,7 +2151,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case with non-existent source file.
 		// Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`.
 		// Expecting the response status code to http.StatusNotFound (404).
-		{
+		8: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + "non-existent-object"),
@@ -2162,7 +2165,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Test case with non-existent source file.
 		// Case for the purpose of failing `api.ObjectAPI.PutObject`.
 		// Expecting the response status code to http.StatusNotFound (404).
-		{
+		9: {
 			bucketName:       "non-existent-destination-bucket",
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2174,7 +2177,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,

 		// Test case - 10.
 		// Case with invalid AccessKey.
-		{
+		10: {
 			bucketName:       bucketName,
 			newObjectName:    objectName,
 			copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2184,7 +2187,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 11, copy metadata from newObject1 with satisfying modified header.
-		{
+		11: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2194,7 +2197,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusOK,
 		},
 		// Test case - 12, copy metadata from newObject1 with unsatisfying modified header.
-		{
+		12: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2204,7 +2207,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusPreconditionFailed,
 		},
 		// Test case - 13, copy metadata from newObject1 with wrong modified header format
-		{
+		13: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2214,7 +2217,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusOK,
 		},
 		// Test case - 14, copy metadata from newObject1 with satisfying unmodified header.
-		{
+		14: {
 			bucketName:           bucketName,
 			newObjectName:        "newObject1",
 			copySourceHeader:     url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2224,7 +2227,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus:   http.StatusOK,
 		},
 		// Test case - 15, copy metadata from newObject1 with unsatisfying unmodified header.
-		{
+		15: {
 			bucketName:           bucketName,
 			newObjectName:        "newObject1",
 			copySourceHeader:     url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2234,7 +2237,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus:   http.StatusPreconditionFailed,
 		},
 		// Test case - 16, copy metadata from newObject1 with incorrect unmodified header format.
-		{
+		16: {
 			bucketName:           bucketName,
 			newObjectName:        "newObject1",
 			copySourceHeader:     url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName),
@@ -2244,7 +2247,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus:   http.StatusOK,
 		},
 		// Test case - 17, copy metadata from newObject1 with null versionId
-		{
+		17: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=null",
@@ -2253,7 +2256,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			expectedRespStatus: http.StatusOK,
 		},
 		// Test case - 18, copy metadata from newObject1 with non null versionId
-		{
+		18: {
 			bucketName:         bucketName,
 			newObjectName:      "newObject1",
 			copySourceHeader:   url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=17",
@@ -2273,7 +2276,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			0, nil, testCase.accessKey, testCase.secretKey, nil)

 		if err != nil {
-			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
+			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i, err)
 		}
 		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
 		if testCase.copySourceHeader != "" {
@@ -2303,25 +2306,35 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		apiRouter.ServeHTTP(rec, req)
 		// Assert the response code with the expected status.
 		if rec.Code != testCase.expectedRespStatus {
-			t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+			t.Errorf("Test %d: %s:  Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code)
 			continue
 		}
 		if rec.Code == http.StatusOK {
 			var cpObjResp CopyObjectResponse
 			if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil {
-				t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i+1, instanceType, err)
+				t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i, instanceType, err)
 			}

 			// See if the new object is formed.
 			// testing whether the copy was successful.
-			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
-			if err != nil {
-				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
+			// Note that this goes directly to the file system,
+			// so encryption/compression may interfere at some point.
+
+			globalCompressConfigMu.Lock()
+			cfg := globalCompressConfig
+			globalCompressConfigMu.Unlock()
+			if !cfg.Enabled {
+				err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
+				if err != nil {
+					t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i, instanceType, err)
+				}
+				if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) {
+					t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i, instanceType)
+				}
+				buffers[0].Reset()
+			} else {
+				t.Log("object not validated due to compression")
 			}
-			if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) {
-				t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i+1, instanceType)
-			}
-			buffers[0].Reset()
 		}

 		// Verify response of the V2 signed HTTP request.
@@ -2330,7 +2343,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,

 		reqV2, err = newTestRequest(http.MethodPut, getCopyObjectURL("", testCase.bucketName, testCase.newObjectName), 0, nil)
 		if err != nil {
-			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
+			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i, err)
 		}
 		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
 		if testCase.copySourceHeader != "" {
@@ -2366,7 +2379,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		// Call the ServeHTTP to execute the handler.
 		apiRouter.ServeHTTP(recV2, reqV2)
 		if recV2.Code != testCase.expectedRespStatus {
-			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
+			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, recV2.Code)
 		}
 	}

@@ -3304,7 +3317,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 // when the request signature type is `streaming signature`.
 func TestAPIPutObjectPartHandlerStreaming(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"})
 }

 func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -3392,7 +3405,7 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN
 //  for variety of inputs.
 func TestAPIPutObjectPartHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"})
+	ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"})
 }

 func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -3797,7 +3810,7 @@ func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketN
 //  for variety of success/failure cases.
 func TestAPIListObjectPartsHandler(t *testing.T) {
 	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"})
+	ExecExtendedObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"})
 }

 func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
@@ -21,10 +21,12 @@ import (
 	"context"
 	"io"
 	"math/rand"
+	"os"
 	"strconv"
 	"testing"

-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/crypto"
 )

 // Return pointer to testOneByteReadEOF{}
@@ -68,10 +70,8 @@ func (r *testOneByteReadNoEOF) Read(p []byte) (n int, err error) {
 	return n, nil
 }

-type ObjectLayerAPISuite struct{}
-
 // Wrapper for calling testMakeBucket for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMakeBucket(t *testing.T) {
+func TestMakeBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testMakeBucket)
 }

@@ -84,8 +84,8 @@ func testMakeBucket(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }

 // Wrapper for calling testMultipartObjectCreation for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) {
-	ExecObjectLayerTest(t, testMultipartObjectCreation)
+func TestMultipartObjectCreation(t *testing.T) {
+	ExecExtendedObjectLayerTest(t, testMultipartObjectCreation)
 }

 // Tests validate creation of part files during Multipart operation.
@@ -128,7 +128,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
 }

 // Wrapper for calling testMultipartObjectAbort for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) {
+func TestMultipartObjectAbort(t *testing.T) {
 	ExecObjectLayerTest(t, testMultipartObjectAbort)
 }

@@ -173,8 +173,8 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
 }

 // Wrapper for calling testMultipleObjectCreation for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) {
-	ExecObjectLayerTest(t, testMultipleObjectCreation)
+func TestMultipleObjectCreation(t *testing.T) {
+	ExecExtendedObjectLayerTest(t, testMultipleObjectCreation)
 }

 // Tests validate object creation.
@@ -230,7 +230,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 }

 // Wrapper for calling TestPaging for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestPaging(t *testing.T) {
+func TestPaging(t *testing.T) {
 	ExecObjectLayerTest(t, testPaging)
 }

@@ -434,7 +434,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }

 // Wrapper for calling testObjectOverwriteWorks for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestObjectOverwriteWorks(t *testing.T) {
+func TestObjectOverwriteWorks(t *testing.T) {
 	ExecObjectLayerTest(t, testObjectOverwriteWorks)
 }

@@ -471,7 +471,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 }

 // Wrapper for calling testNonExistantBucketOperations for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) {
+func TestNonExistantBucketOperations(t *testing.T) {
 	ExecObjectLayerTest(t, testNonExistantBucketOperations)
 }

@@ -488,7 +488,7 @@ func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t Tes
 }

 // Wrapper for calling testBucketRecreateFails for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestBucketRecreateFails(t *testing.T) {
+func TestBucketRecreateFails(t *testing.T) {
 	ExecObjectLayerTest(t, testBucketRecreateFails)
 }

@@ -508,9 +508,68 @@ func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHand
 	}
 }

+func execExtended(t *testing.T, fn func(t *testing.T)) {
+	// Exec with default settings...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.Enabled = false
+	globalCompressConfigMu.Unlock()
+	t.Run("default", func(t *testing.T) {
+		fn(t)
+	})
+	if testing.Short() {
+		return
+	}
+
+	// Enable compression and exec...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.Enabled = true
+	globalCompressConfig.MimeTypes = nil
+	globalCompressConfig.Extensions = nil
+	globalCompressConfigMu.Unlock()
+	t.Run("compressed", func(t *testing.T) {
+		fn(t)
+	})
+
+	globalAutoEncryption = true
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	var err error
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("encrypted", func(t *testing.T) {
+		fn(t)
+	})
+
+	// Enable compression of encrypted and exec...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.AllowEncrypted = true
+	globalCompressConfigMu.Unlock()
+	t.Run("compressed+encrypted", func(t *testing.T) {
+		fn(t)
+	})
+
+	// Reset...
+	globalCompressConfigMu.Lock()
+	globalCompressConfig.Enabled = false
+	globalCompressConfig.AllowEncrypted = false
+	globalCompressConfigMu.Unlock()
+	globalAutoEncryption = false
+}
+
+// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
+// This can be used to test functionality when reading and writing data.
+func ExecExtendedObjectLayerTest(t *testing.T, objTest objTestType) {
+	execExtended(t, func(t *testing.T) {
+		ExecObjectLayerTest(t, objTest)
+	})
+}
+
 // Wrapper for calling testPutObject for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestPutObject(t *testing.T) {
-	ExecObjectLayerTest(t, testPutObject)
+func TestPutObject(t *testing.T) {
+	ExecExtendedObjectLayerTest(t, testPutObject)
 }

 // Tests validate PutObject without prefix.
@@ -553,8 +612,8 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }

 // Wrapper for calling testPutObjectInSubdir for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestPutObjectInSubdir(t *testing.T) {
-	ExecObjectLayerTest(t, testPutObjectInSubdir)
+func TestPutObjectInSubdir(t *testing.T) {
+	ExecExtendedObjectLayerTest(t, testPutObjectInSubdir)
 }

 // Tests validate PutObject with subdirectory prefix.
@@ -585,7 +644,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
 }

 // Wrapper for calling testListBuckets for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestListBuckets(t *testing.T) {
+func TestListBuckets(t *testing.T) {
 	ExecObjectLayerTest(t, testListBuckets)
 }

@@ -644,7 +703,7 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 }

 // Wrapper for calling testListBucketsOrder for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestListBucketsOrder(t *testing.T) {
+func TestListBucketsOrder(t *testing.T) {
 	ExecObjectLayerTest(t, testListBucketsOrder)
 }

@@ -678,7 +737,7 @@ func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler
 }

 // Wrapper for calling testListObjectsTestsForNonExistantBucket for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestListObjectsTestsForNonExistantBucket(t *testing.T) {
+func TestListObjectsTestsForNonExistantBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testListObjectsTestsForNonExistantBucket)
 }

@@ -700,7 +759,7 @@ func testListObjectsTestsForNonExistantBucket(obj ObjectLayer, instanceType stri
 }

 // Wrapper for calling testNonExistantObjectInBucket for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestNonExistantObjectInBucket(t *testing.T) {
+func TestNonExistantObjectInBucket(t *testing.T) {
 	ExecObjectLayerTest(t, testNonExistantObjectInBucket)
 }

@@ -716,8 +775,8 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
 		t.Fatalf("%s: Expected error but found nil", instanceType)
 	}
 	if isErrObjectNotFound(err) {
-		if err.Error() != "Object not found: bucket#dir1" {
-			t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket#dir1", err.Error())
+		if err.Error() != "Object not found: bucket/dir1" {
+			t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket/dir1", err.Error())
 		}
 	} else {
 		if err.Error() != "fails" {
@@ -727,7 +786,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
 }

 // Wrapper for calling testGetDirectoryReturnsObjectNotFound for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
+func TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
 	ExecObjectLayerTest(t, testGetDirectoryReturnsObjectNotFound)
 }

@@ -770,7 +829,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
 }

 // Wrapper for calling testContentType for both Erasure and FS.
-func (s *ObjectLayerAPISuite) TestContentType(t *testing.T) {
+func TestContentType(t *testing.T) {
 	ExecObjectLayerTest(t, testContentType)
 }

@@ -672,8 +672,8 @@ func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *
 func getLocalDiskIDs(z *erasureServerPools) []string {
 	var ids []string

-	for zoneIdx := range z.serverPools {
-		for _, set := range z.serverPools[zoneIdx].sets {
+	for poolIdx := range z.serverPools {
+		for _, set := range z.serverPools[poolIdx].sets {
 			disks := set.getDisks()
 			for _, disk := range disks {
 				if disk == nil {
@@ -163,9 +163,9 @@ func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error {
 // https://github.com/minio/minio/issues/5667
 var errErasureV3ThisEmpty = fmt.Errorf("Erasure format version 3 has This field empty")

-// IsServerResolvable - checks if the endpoint is resolvable
+// isServerResolvable - checks if the endpoint is resolvable
 // by sending a naked HTTP request with liveness checks.
-func IsServerResolvable(endpoint Endpoint) error {
+func isServerResolvable(endpoint Endpoint) error {
 	serverURL := &url.URL{
 		Scheme: endpoint.Scheme,
 		Host:   endpoint.Host,
@@ -228,7 +228,7 @@ func IsServerResolvable(endpoint Endpoint) error {
 // connect to list of endpoints and load all Erasure disk formats, validate the formats are correct
 // and are in quorum, if no formats are found attempt to initialize all of them for the first
 // time. additionally make sure to close all the disks used in this attempt.
-func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, zoneCount, setCount, setDriveCount int, deploymentID string) (storageDisks []StorageAPI, format *formatErasureV3, err error) {
+func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID string) (storageDisks []StorageAPI, format *formatErasureV3, err error) {
 	// Initialize all storage disks
 	storageDisks, errs := initStorageDisksWithErrors(endpoints)

@@ -244,7 +244,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 				return nil, nil, fmt.Errorf("Disk %s: %w", endpoints[i], err)
 			}
 			if retryCount >= 5 {
-				logger.Info("Unable to connect to %s: %v\n", endpoints[i], IsServerResolvable(endpoints[i]))
+				logger.Info("Unable to connect to %s: %v\n", endpoints[i], isServerResolvable(endpoints[i]))
 			}
 		}
 	}
@@ -272,8 +272,8 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,

 	// All disks report unformatted we should initialized everyone.
 	if shouldInitErasureDisks(sErrs) && firstDisk {
-		logger.Info("Formatting %s zone, %v set(s), %v drives per set.",
-			humanize.Ordinal(zoneCount), setCount, setDriveCount)
+		logger.Info("Formatting %s pool, %v set(s), %v drives per set.",
+			humanize.Ordinal(poolCount), setCount, setDriveCount)

 		// Initialize erasure code format on disks
 		format, err = initFormatErasure(GlobalContext, storageDisks, setCount, setDriveCount, "", deploymentID, sErrs)
@@ -345,7 +345,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 }

 // Format disks before initialization of object layer.
-func waitForFormatErasure(firstDisk bool, endpoints Endpoints, zoneCount, setCount, setDriveCount int, deploymentID string) ([]StorageAPI, *formatErasureV3, error) {
+func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID string) ([]StorageAPI, *formatErasureV3, error) {
 	if len(endpoints) == 0 || setCount == 0 || setDriveCount == 0 {
 		return nil, nil, errInvalidArgument
 	}
@@ -372,7 +372,7 @@ func waitForFormatErasure(firstDisk bool, endpoints Endpoints, zoneCount, setCou
 	for {
 		select {
 		case <-ticker.C:
-			storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, zoneCount, setCount, setDriveCount, deploymentID)
+			storageDisks, format, err := connectLoadInitFormats(tries, firstDisk, endpoints, poolCount, setCount, setDriveCount, deploymentID)
 			if err != nil {
 				tries++
 				switch err {
@@ -83,13 +83,13 @@ EXAMPLES:
     {{.Prompt}} {{.HelpName}} /mnt/data{1...64}

  3. Start distributed minio server on an 32 node setup with 32 drives each, run following command on all the nodes
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}minio
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}minio
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}miniostorage
     {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export{1...32}

  4. Start distributed minio server in an expanded setup, run the following command on all the nodes
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}minio
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}minio
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}miniostorage
     {{.Prompt}} {{.HelpName}} http://node{1...16}.example.com/mnt/export{1...32} \
            http://node{17...64}.example.com/mnt/export{1...64}
 `,
@@ -139,6 +139,16 @@ func serverHandleCmdArgs(ctx *cli.Context) {

 	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
 	globalEndpoints, setupType, err = createServerEndpoints(globalCLIContext.Addr, serverCmdArgs(ctx)...)
+
+	globalRemoteEndpoints = make(map[string]Endpoint)
+	for _, z := range globalEndpoints {
+		for _, ep := range z.Endpoints {
+			if ep.IsLocal {
+				continue
+			}
+			globalRemoteEndpoints[ep.Host] = ep
+		}
+	}
 	logger.FatalIf(err, "Invalid command line arguments")

 	// allow transport to be HTTP/1.1 for proxying.
@@ -183,7 +193,12 @@ func newAllSubsystems() {
 	globalNotificationSys = NewNotificationSys(globalEndpoints)

 	// Create new bucket metadata system.
-	globalBucketMetadataSys = NewBucketMetadataSys()
+	if globalBucketMetadataSys == nil {
+		globalBucketMetadataSys = NewBucketMetadataSys()
+	} else {
+		// Reinitialize safely when testing.
+		globalBucketMetadataSys.Reset()
+	}

 	// Create the bucket bandwidth monitor
 	globalBucketMonitor = bandwidth.NewMonitor(GlobalServiceDoneCh)
@@ -210,7 +225,11 @@ func newAllSubsystems() {
 	globalBucketQuotaSys = NewBucketQuotaSys()

 	// Create new bucket versioning subsystem
-	globalBucketVersioningSys = NewBucketVersioningSys()
+	if globalBucketVersioningSys == nil {
+		globalBucketVersioningSys = NewBucketVersioningSys()
+	} else {
+		globalBucketVersioningSys.Reset()
+	}

 	// Create new bucket replication subsytem
 	globalBucketTargetSys = NewBucketTargetSys()
@@ -327,7 +346,7 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 			}
 		}
 		for _, bucket := range buckets {
-			if _, err = newObject.HealBucket(ctx, bucket.Name, madmin.HealOpts{}); err != nil {
+			if _, err = newObject.HealBucket(ctx, bucket.Name, madmin.HealOpts{Recreate: true}); err != nil {
 				return fmt.Errorf("Unable to list buckets to heal: %w", err)
 			}
 		}
@@ -347,9 +366,6 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize config, some features may be missing %w", err))
 	}

-	// Initialize IAM store
-	globalIAMSys.InitStore(newObject)
-
 	// Populate existing buckets to the etcd backend
 	if globalDNSConfig != nil {
 		// Background this operation.
@@ -511,7 +527,7 @@ func serverMain(ctx *cli.Context) {
 	printStartupMessage(getAPIEndpoints(), err)

 	if globalActiveCred.Equal(auth.DefaultCredentials) {
-		msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ACCESS_KEY' and 'MINIO_SECRET_KEY'", globalActiveCred)
+		msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD'", globalActiveCred)
 		logger.StartupMessage(color.RedBold(msg))
 	}

@@ -135,12 +135,12 @@ func printServerCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")

 	// Colorize the message and print.
-	logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
+	logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(color.Blue("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		if region != "" {
-			logStartupMessage(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+			logStartupMessage(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 2), region)))
 		}
 	}
 	printEventNotifiers()
@@ -164,7 +164,7 @@ func printEventNotifiers() {

 	arnMsg := color.Blue("SQS ARNs: ")
 	for _, arn := range arns {
-		arnMsg += color.Bold(fmt.Sprintf(getFormatStr(len(arn), 1), arn))
+		arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
 	}

 	logStartupMessage(arnMsg)
@@ -151,6 +151,8 @@ type FileInfo struct {
 	MarkDeleted                   bool // mark this version as deleted
 	DeleteMarkerReplicationStatus string
 	VersionPurgeStatus            VersionPurgeStatusType
+
+	Data []byte // optionally carries object data
 }

 // VersionPurgeStatusKey denotes purge status in metadata
@@ -245,8 +245,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 17 {
-		err = msgp.ArrayError{Wanted: 17, Got: zb0001}
+	if zb0001 != 18 {
+		err = msgp.ArrayError{Wanted: 18, Got: zb0001}
 		return
 	}
 	z.Volume, err = dc.ReadString()
@@ -375,13 +375,18 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		z.VersionPurgeStatus = VersionPurgeStatusType(zb0004)
 	}
+	z.Data, err = dc.ReadBytes(z.Data)
+	if err != nil {
+		err = msgp.WrapError(err, "Data")
+		return
+	}
 	return
 }

 // EncodeMsg implements msgp.Encodable
 func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 17
-	err = en.Append(0xdc, 0x0, 0x11)
+	// array header, size 18
+	err = en.Append(0xdc, 0x0, 0x12)
 	if err != nil {
 		return
 	}
@@ -489,14 +494,19 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "VersionPurgeStatus")
 		return
 	}
+	err = en.WriteBytes(z.Data)
+	if err != nil {
+		err = msgp.WrapError(err, "Data")
+		return
+	}
 	return
 }

 // MarshalMsg implements msgp.Marshaler
 func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 17
-	o = append(o, 0xdc, 0x0, 0x11)
+	// array header, size 18
+	o = append(o, 0xdc, 0x0, 0x12)
 	o = msgp.AppendString(o, z.Volume)
 	o = msgp.AppendString(o, z.Name)
 	o = msgp.AppendString(o, z.VersionID)
@@ -529,6 +539,7 @@ func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendBool(o, z.MarkDeleted)
 	o = msgp.AppendString(o, z.DeleteMarkerReplicationStatus)
 	o = msgp.AppendString(o, string(z.VersionPurgeStatus))
+	o = msgp.AppendBytes(o, z.Data)
 	return
 }

@@ -540,8 +551,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 17 {
-		err = msgp.ArrayError{Wanted: 17, Got: zb0001}
+	if zb0001 != 18 {
+		err = msgp.ArrayError{Wanted: 18, Got: zb0001}
 		return
 	}
 	z.Volume, bts, err = msgp.ReadStringBytes(bts)
@@ -670,6 +681,11 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		z.VersionPurgeStatus = VersionPurgeStatusType(zb0004)
 	}
+	z.Data, bts, err = msgp.ReadBytesBytes(bts, z.Data)
+	if err != nil {
+		err = msgp.WrapError(err, "Data")
+		return
+	}
 	o = bts
 	return
 }
@@ -687,7 +703,7 @@ func (z *FileInfo) Msgsize() (s int) {
 	for za0003 := range z.Parts {
 		s += z.Parts[za0003].Msgsize()
 	}
-	s += z.Erasure.Msgsize() + msgp.BoolSize + msgp.StringPrefixSize + len(z.DeleteMarkerReplicationStatus) + msgp.StringPrefixSize + len(string(z.VersionPurgeStatus))
+	s += z.Erasure.Msgsize() + msgp.BoolSize + msgp.StringPrefixSize + len(z.DeleteMarkerReplicationStatus) + msgp.StringPrefixSize + len(string(z.VersionPurgeStatus)) + msgp.BytesPrefixSize + len(z.Data)
 	return
 }

@@ -58,7 +58,7 @@ type StorageAPI interface {
 	DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error
 	DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error
 	WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error
-	ReadVersion(ctx context.Context, volume, path, versionID string) (FileInfo, error)
+	ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (FileInfo, error)
 	RenameData(ctx context.Context, srcVolume, srcPath, dataDir, dstVolume, dstPath string) error

 	// File operations.
@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	xnet "github.com/minio/minio/pkg/net"
+	xbufio "github.com/philhofer/fwd"
 	"github.com/tinylib/msgp/msgp"
 )

@@ -226,8 +227,7 @@ func (client *storageRESTClient) DiskInfo(ctx context.Context) (info DiskInfo, e
 				return info, err
 			}
 			defer http.DrainBody(respBody)
-			err = msgp.Decode(respBody, &info)
-			if err != nil {
+			if err = msgp.Decode(respBody, &info); err != nil {
 				return info, err
 			}
 			if info.Error != "" {
@@ -398,18 +398,38 @@ func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcP
 	return err
 }

-func (client *storageRESTClient) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
+// where we keep old *Readers
+var readMsgpReaderPool = sync.Pool{New: func() interface{} { return &msgp.Reader{} }}
+
+// mspNewReader returns a *Reader that reads from the provided reader.
+// The reader will be buffered.
+func msgpNewReader(r io.Reader) *msgp.Reader {
+	p := readMsgpReaderPool.Get().(*msgp.Reader)
+	if p.R == nil {
+		p.R = xbufio.NewReaderSize(r, 8<<10)
+	} else {
+		p.R.Reset(r)
+	}
+	return p
+}
+
+func (client *storageRESTClient) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
 	values.Set(storageRESTVersionID, versionID)
+	values.Set(storageRESTReadData, strconv.FormatBool(readData))

 	respBody, err := client.call(ctx, storageRESTMethodReadVersion, values, nil, -1)
 	if err != nil {
 		return fi, err
 	}
 	defer http.DrainBody(respBody)
-	err = msgp.Decode(respBody, &fi)
+
+	dec := msgpNewReader(respBody)
+	defer readMsgpReaderPool.Put(dec)
+
+	err = fi.DecodeMsg(dec)
 	return fi, err
 }

@@ -479,10 +499,12 @@ func (client *storageRESTClient) WalkVersions(ctx context.Context, volume, dirPa
 		defer close(ch)
 		defer http.DrainBody(respBody)

-		decoder := msgp.NewReader(respBody)
+		dec := msgpNewReader(respBody)
+		defer readMsgpReaderPool.Put(dec)
+
 		for {
 			var fi FileInfoVersions
-			if gerr := fi.DecodeMsg(decoder); gerr != nil {
+			if gerr := fi.DecodeMsg(dec); gerr != nil {
 				// Upon error return
 				if msgp.Cause(gerr) != io.EOF {
 					logger.LogIf(GlobalContext, gerr)
@@ -17,7 +17,7 @@
 package cmd

 const (
-	storageRESTVersion       = "v23" // Add small file optimization
+	storageRESTVersion       = "v24" // Add more small file optimization
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )
@@ -59,6 +59,7 @@ const (
 	storageRESTDirPath        = "dir-path"
 	storageRESTFilePath       = "file-path"
 	storageRESTVersionID      = "version-id"
+	storageRESTReadData       = "read-data"
 	storageRESTTotalVersions  = "total-versions"
 	storageRESTSrcVolume      = "source-volume"
 	storageRESTSrcPath        = "source-path"
@@ -327,11 +327,18 @@ func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Re
 	volume := vars[storageRESTVolume]
 	filePath := vars[storageRESTFilePath]
 	versionID := vars[storageRESTVersionID]
-	fi, err := s.storage.ReadVersion(r.Context(), volume, filePath, versionID)
+	readData, err := strconv.ParseBool(vars[storageRESTReadData])
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
+
+	fi, err := s.storage.ReadVersion(r.Context(), volume, filePath, versionID, readData)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
 	logger.LogIf(r.Context(), msgp.Encode(w, &fi))
 }

@@ -515,15 +522,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
 	defer rc.Close()

 	w.Header().Set(xhttp.ContentLength, strconv.Itoa(length))
-
-	if length >= readBlockSize {
-		bufp := s.storage.pool.Get().(*[]byte)
-		defer s.storage.pool.Put(bufp)
-
-		io.CopyBuffer(w, rc, *bufp)
-	} else {
-		io.Copy(w, rc)
-	}
+	io.Copy(w, rc)
 	w.(http.Flusher).Flush()
 }

@@ -1041,7 +1040,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersion).HandlerFunc(httpTraceHdrs(server.DeleteVersionHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadVersion).HandlerFunc(httpTraceHdrs(server.ReadVersionHandler)).
-				Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTVersionID)...)
+				Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTVersionID, storageRESTReadData)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameData).HandlerFunc(httpTraceHdrs(server.RenameDataHandler)).
 				Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDataDir,
 					storageRESTDstVolume, storageRESTDstPath)...)
@@ -107,7 +107,7 @@ func TestMain(m *testing.M) {
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background())

-	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
+	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second, logger.LogOnceIf)

 	globalInternodeTransport = newInternodeHTTPTransport(nil, rest.DefaultTimeout)()

@@ -1866,6 +1866,14 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 	removeRoots(append(erasureDisks, fsDir))
 }

+// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
+// This can be used to test functionality when reading and writing data.
+func ExecExtendedObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) {
+	execExtended(t, func(t *testing.T) {
+		ExecObjectLayerAPITest(t, objAPITest, endpoints)
+	})
+}
+
 // function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests.
 type objAPITestType func(obj ObjectLayer, instanceType string, bucketName string,
 	apiRouter http.Handler, credentials auth.Credentials, t *testing.T)
@@ -709,7 +709,10 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
 		Versioned:        globalBucketVersioningSys.Enabled(args.BucketName),
 		VersionSuspended: globalBucketVersioningSys.Suspended(args.BucketName),
 	}
-	var err error
+	var (
+		err           error
+		replicateSync bool
+	)
 next:
 	for _, objectName := range args.Objects {
 		// If not a directory, remove the object.
@@ -751,7 +754,7 @@ next:
 			}
 			if hasReplicationRules(ctx, args.BucketName, []ObjectToDelete{{ObjectName: objectName}}) || hasLifecycleConfig {
 				goi, gerr = getObjectInfoFn(ctx, args.BucketName, objectName, opts)
-				if _, replicateDel = checkReplicateDelete(ctx, args.BucketName, ObjectToDelete{ObjectName: objectName}, goi, gerr); replicateDel {
+				if _, replicateDel, replicateSync = checkReplicateDelete(ctx, args.BucketName, ObjectToDelete{ObjectName: objectName, VersionID: goi.VersionID}, goi, gerr); replicateDel {
 					opts.DeleteMarkerReplicationStatus = string(replication.Pending)
 					opts.DeleteMarker = true
 				}
@@ -759,7 +762,7 @@ next:

 			oi, err := deleteObject(ctx, objectAPI, web.CacheAPI(), args.BucketName, objectName, nil, r, opts)
 			if replicateDel && err == nil {
-				globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+				dobj := DeletedObjectVersionInfo{
 					DeletedObject: DeletedObject{
 						ObjectName:                    objectName,
 						DeleteMarkerVersionID:         oi.VersionID,
@@ -769,7 +772,8 @@ next:
 						VersionPurgeStatus:            oi.VersionPurgeStatus,
 					},
 					Bucket: args.BucketName,
-				})
+				}
+				scheduleReplicationDelete(ctx, dobj, objectAPI, replicateSync)
 			}
 			if goi.TransitionStatus == lifecycle.TransitionComplete && err == nil && goi.VersionID == "" {
 				action := lifecycle.DeleteAction
@@ -855,8 +859,8 @@ next:
 						}
 					}
 				}
+				_, replicateDel, _ := checkReplicateDelete(ctx, args.BucketName, ObjectToDelete{ObjectName: obj.Name, VersionID: obj.VersionID}, obj, nil)
 				// since versioned delete is not available on web browser, yet - this is a simple DeleteMarker replication
-				_, replicateDel := checkReplicateDelete(ctx, args.BucketName, ObjectToDelete{ObjectName: obj.Name}, obj, nil)
 				objToDel := ObjectToDelete{ObjectName: obj.Name}
 				if replicateDel {
 					objToDel.DeleteMarkerReplicationStatus = string(replication.Pending)
@@ -904,10 +908,11 @@ next:
 					Host:       handlers.GetSourceIP(r),
 				})
 				if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
-					globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+					dv := DeletedObjectVersionInfo{
 						DeletedObject: dobj,
 						Bucket:        args.BucketName,
-					})
+					}
+					scheduleReplicationDelete(ctx, dv, objectAPI, replicateSync)
 				}
 			}
 		}
@@ -1228,7 +1233,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}
 	}

-	mustReplicate := mustReplicateWeb(ctx, r, bucket, object, metadata, "", replPerms)
+	mustReplicate, sync := mustReplicateWeb(ctx, r, bucket, object, metadata, "", replPerms)
 	if mustReplicate {
 		metadata[xhttp.AmzBucketReplicationStatus] = string(replication.Pending)
 	}
@@ -1298,7 +1303,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	if mustReplicate {
-		globalReplicationState.queueReplicaTask(objInfo)
+		scheduleReplication(ctx, objInfo, objectAPI, sync)
 	}

 	// Notify object created event.
@@ -272,12 +272,12 @@ func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path s
 	return p.storage.WriteMetadata(ctx, volume, path, fi)
 }

-func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
+func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
 	if err = p.checkDiskStale(); err != nil {
 		return fi, err
 	}

-	return p.storage.ReadVersion(ctx, volume, path, versionID)
+	return p.storage.ReadVersion(ctx, volume, path, versionID, readData)
 }

 func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
@@ -55,7 +55,8 @@ import (
 const (
 	nullVersionID     = "null"
 	diskMinTotalSpace = 900 * humanize.MiByte // Min 900MiB total space.
-	readBlockSize     = 4 * humanize.MiByte   // Default read block size 4MiB.
+	blockSizeLarge    = 1 * humanize.MiByte   // Default r/w block size for larger objects.
+	blockSizeSmall    = 128 * humanize.KiByte // Default r/w block size for smaller objects.

 	// On regular files bigger than this;
 	readAheadSize = 16 << 20
@@ -64,6 +65,11 @@ const (
 	// Size of each buffer.
 	readAheadBufSize = 1 << 20

+	// Small file threshold below which data accompanies metadata from storage layer.
+	smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs
+	// For hardrives it is possible to set this to a lower value to avoid any
+	// spike in latency. But currently we are simply keeping it optimal for SSDs.
+
 	// XL metadata file carries per object metadata.
 	xlStorageFormatFile = "xl.meta"
 )
@@ -90,15 +96,17 @@ type xlStorage struct {
 	diskPath string
 	endpoint Endpoint

-	pool sync.Pool
-
 	globalSync bool

+	poolLarge sync.Pool
+	poolSmall sync.Pool
+
 	rootDisk bool

 	diskID string

 	formatFileInfo  os.FileInfo
+	formatLegacy    bool
 	formatLastCheck time.Time

 	diskInfoCache timedValue
@@ -254,9 +262,15 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) {
 	p := &xlStorage{
 		diskPath: path,
 		endpoint: ep,
-		pool: sync.Pool{
+		poolLarge: sync.Pool{
 			New: func() interface{} {
-				b := disk.AlignedBlock(readBlockSize)
+				b := disk.AlignedBlock(blockSizeLarge)
+				return &b
+			},
+		},
+		poolSmall: sync.Pool{
+			New: func() interface{} {
+				b := disk.AlignedBlock(blockSizeSmall)
 				return &b
 			},
 		},
@@ -484,13 +498,13 @@ func (s *xlStorage) GetDiskID() (string, error) {
 	}

 	s.Lock()
-	defer s.Unlock()
-
 	// If somebody else updated the disk ID and changed the time, return what they got.
 	if !lastCheck.IsZero() && !s.formatLastCheck.Equal(lastCheck) && diskID != "" {
+		s.Unlock()
 		// Somebody else got the lock first.
 		return diskID, nil
 	}
+	s.Unlock()

 	formatFile := pathJoin(s.diskPath, minioMetaBucket, formatConfigFile)
 	fi, err := os.Stat(formatFile)
@@ -517,8 +531,10 @@ func (s *xlStorage) GetDiskID() (string, error) {
 	}

 	if xioutil.SameFile(fi, fileInfo) && diskID != "" {
+		s.Lock()
 		// If the file has not changed, just return the cached diskID information.
 		s.formatLastCheck = time.Now()
+		s.Unlock()
 		return diskID, nil
 	}

@@ -552,7 +568,10 @@ func (s *xlStorage) GetDiskID() (string, error) {
 		return "", errCorruptedFormat
 	}

+	s.Lock()
+	defer s.Unlock()
 	s.diskID = format.Erasure.This
+	s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoLegacy
 	s.formatFileInfo = fi
 	s.formatLastCheck = time.Now()
 	return s.diskID, nil
@@ -671,12 +690,16 @@ func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, er
 	var st os.FileInfo
 	st, err = os.Stat(volumeDir)
 	if err != nil {
-		if osIsNotExist(err) {
+		switch {
+		case osIsNotExist(err):
 			return VolInfo{}, errVolumeNotFound
-		} else if isSysErrIO(err) {
+		case osIsPermission(err):
+			return VolInfo{}, errDiskAccessDenied
+		case isSysErrIO(err):
 			return VolInfo{}, errFaultyDisk
+		default:
+			return VolInfo{}, err
 		}
-		return VolInfo{}, err
 	}
 	// As os.Stat() doesn't carry other than ModTime(), use ModTime()
 	// as CreatedTime.
@@ -723,58 +746,6 @@ func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bo
 	return nil
 }

-const guidSplunk = "guidSplunk"
-
-// ListDirSplunk - return all the entries at the given directory path.
-// If an entry is a directory it will be returned with a trailing SlashSeparator.
-func (s *xlStorage) ListDirSplunk(volume, dirPath string, count int) (entries []string, err error) {
-	guidIndex := strings.Index(dirPath, guidSplunk)
-	if guidIndex != -1 {
-		return nil, nil
-	}
-
-	atomic.AddInt32(&s.activeIOCount, 1)
-	defer func() {
-		atomic.AddInt32(&s.activeIOCount, -1)
-	}()
-
-	// Verify if volume is valid and it exists.
-	volumeDir, err := s.getVolDir(volume)
-	if err != nil {
-		return nil, err
-	}
-
-	if _, err = os.Stat(volumeDir); err != nil {
-		if osIsNotExist(err) {
-			return nil, errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return nil, errFaultyDisk
-		}
-		return nil, err
-	}
-
-	dirPathAbs := pathJoin(volumeDir, dirPath)
-	if count > 0 {
-		entries, err = readDirN(dirPathAbs, count)
-	} else {
-		entries, err = readDir(dirPathAbs)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	return entries, nil
-}
-
-func (s *xlStorage) isLeafSplunk(volume string, leafPath string) bool {
-	const receiptJSON = "receipt.json"
-
-	if path.Base(leafPath) != receiptJSON {
-		return false
-	}
-	return s.isLeaf(volume, leafPath)
-}
-
 func (s *xlStorage) isLeaf(volume string, leafPath string) bool {
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
@@ -790,7 +761,7 @@ func (s *xlStorage) isLeaf(volume string, leafPath string) bool {
 		// legacy `xl.json`, in such situation we just rename
 		// and proceed if rename is successful we know that it
 		// is the leaf since `xl.json` was present.
-		return s.renameLegacyMetadata(volume, leafPath) == nil
+		return s.renameLegacyMetadata(volumeDir, leafPath) == nil
 	}
 	return false
 }
@@ -903,15 +874,6 @@ func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count i
 		return nil, err
 	}

-	if _, err = os.Stat(volumeDir); err != nil {
-		if osIsNotExist(err) {
-			return nil, errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return nil, errFaultyDisk
-		}
-		return nil, err
-	}
-
 	dirPathAbs := pathJoin(volumeDir, dirPath)
 	if count > 0 {
 		entries, err = readDirN(dirPathAbs, count)
@@ -919,6 +881,15 @@ func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count i
 		entries, err = readDir(dirPathAbs)
 	}
 	if err != nil {
+		if err == errFileNotFound {
+			if _, verr := os.Stat(volumeDir); verr != nil {
+				if osIsNotExist(verr) {
+					return nil, errVolumeNotFound
+				} else if isSysErrIO(verr) {
+					return nil, errFaultyDisk
+				}
+			}
+		}
 		return nil, err
 	}

@@ -1057,30 +1028,21 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 	return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
 }

-func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
+func (s *xlStorage) renameLegacyMetadata(volumeDir, path string) (err error) {
+	s.RLock()
+	legacy := s.formatLegacy
+	s.RUnlock()
+	if !legacy {
+		// if its not a legacy backend then this function is
+		// a no-op always returns errFileNotFound
+		return errFileNotFound
+	}
+
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
 	}()

-	volumeDir, err := s.getVolDir(volume)
-	if err != nil {
-		return err
-	}
-
-	//gi Stat a volume entry.
-	_, err = os.Stat(volumeDir)
-	if err != nil {
-		if osIsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return errFaultyDisk
-		} else if isSysErrTooManyFiles(err) {
-			return errTooManyOpenFiles
-		}
-		return err
-	}
-
 	// Validate file path length, before reading.
 	filePath := pathJoin(volumeDir, path)
 	if err = checkPathLength(filePath); err != nil {
@@ -1122,13 +1084,18 @@ func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
 }

 // ReadVersion - reads metadata and returns FileInfo at path `xl.meta`
-// for all objects less than `128KiB` this call returns data as well
+// for all objects less than `32KiB` this call returns data as well
 // along with metadata.
-func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID string) (fi FileInfo, err error) {
+func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return fi, err
+	}
+
 	buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
 	if err != nil {
 		if err == errFileNotFound {
-			if err = s.renameLegacyMetadata(volume, path); err != nil {
+			if err = s.renameLegacyMetadata(volumeDir, path); err != nil {
 				if err == errFileNotFound {
 					if versionID != "" {
 						return fi, errFileVersionNotFound
@@ -1159,7 +1126,65 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 		return fi, errFileNotFound
 	}

-	return getFileInfo(buf, volume, path, versionID)
+	fi, err = getFileInfo(buf, volume, path, versionID)
+	if err != nil {
+		return fi, err
+	}
+
+	if readData {
+		// Reading data for small objects when
+		// - object has not yet transitioned
+		// - object size lesser than 32KiB
+		// - object has maximum of 1 parts
+		if fi.TransitionStatus == "" && fi.DataDir != "" && fi.Size <= smallFileThreshold && len(fi.Parts) == 1 {
+			fi.Data, err = s.readAllData(volumeDir, pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", fi.Parts[0].Number)))
+			if err != nil {
+				return FileInfo{}, err
+			}
+		}
+	}
+
+	return fi, nil
+}
+
+func (s *xlStorage) readAllData(volumeDir, filePath string) (buf []byte, err error) {
+	var f *os.File
+	if globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
+		f, err = disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666)
+	} else {
+		f, err = os.Open(filePath)
+	}
+	if err != nil {
+		if osIsNotExist(err) {
+			// Check if the object doesn't exist because its bucket
+			// is missing in order to return the correct error.
+			_, err = os.Stat(volumeDir)
+			if err != nil && osIsNotExist(err) {
+				return nil, errVolumeNotFound
+			}
+			return nil, errFileNotFound
+		} else if osIsPermission(err) {
+			return nil, errFileAccessDenied
+		} else if isSysErrNotDir(err) || isSysErrIsDir(err) {
+			return nil, errFileNotFound
+		} else if isSysErrHandleInvalid(err) {
+			// This case is special and needs to be handled for windows.
+			return nil, errFileNotFound
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		} else if isSysErrTooManyFiles(err) {
+			return nil, errTooManyOpenFiles
+		} else if isSysErrInvalidArg(err) {
+			return nil, errFileNotFound
+		}
+		return nil, err
+	}
+
+	atomic.AddInt32(&s.activeIOCount, 1)
+	rd := &odirectReader{f, nil, nil, true, true, s, nil}
+	defer rd.Close()
+
+	return ioutil.ReadAll(rd)
 }

 // ReadAll reads from r until an error or EOF and returns the data it read.
@@ -1300,11 +1325,8 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of
 		return int64(n), err
 	}

-	bufp := s.pool.Get().(*[]byte)
-	defer s.pool.Put(bufp)
-
 	h := verifier.algorithm.New()
-	if _, err = io.CopyBuffer(h, io.LimitReader(file, offset), *bufp); err != nil {
+	if _, err = io.Copy(h, io.LimitReader(file, offset)); err != nil {
 		return 0, err
 	}

@@ -1316,7 +1338,7 @@ func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, of
 		return 0, err
 	}

-	if _, err = io.CopyBuffer(h, file, *bufp); err != nil {
+	if _, err = io.Copy(h, file); err != nil {
 		return 0, err
 	}

@@ -1384,22 +1406,27 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err
 }

 // To support O_DIRECT reads for erasure backends.
-type odirectreader struct {
+type odirectReader struct {
 	f         *os.File
 	buf       []byte
 	bufp      *[]byte
 	freshRead bool
+	smallFile bool
 	s         *xlStorage
 	err       error
 }

 // Read - Implements Reader interface.
-func (o *odirectreader) Read(buf []byte) (n int, err error) {
+func (o *odirectReader) Read(buf []byte) (n int, err error) {
 	if o.err != nil {
 		return 0, o.err
 	}
 	if o.buf == nil {
-		o.bufp = o.s.pool.Get().(*[]byte)
+		if o.smallFile {
+			o.bufp = o.s.poolSmall.Get().(*[]byte)
+		} else {
+			o.bufp = o.s.poolLarge.Get().(*[]byte)
+		}
 	}
 	if o.freshRead {
 		o.buf = *o.bufp
@@ -1427,9 +1454,15 @@ func (o *odirectreader) Read(buf []byte) (n int, err error) {
 }

 // Close - Release the buffer and close the file.
-func (o *odirectreader) Close() error {
-	o.s.pool.Put(o.bufp)
-	atomic.AddInt32(&o.s.activeIOCount, -1)
+func (o *odirectReader) Close() error {
+	if o.smallFile {
+		o.s.poolSmall.Put(o.bufp)
+	} else {
+		o.s.poolLarge.Put(o.bufp)
+	}
+	defer func() {
+		atomic.AddInt32(&o.s.activeIOCount, -1)
+	}()
 	return o.f.Close()
 }

@@ -1450,35 +1483,14 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		return nil, err
 	}

+	var file *os.File
+	// O_DIRECT only supported if offset is zero
 	if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
-		file, err := disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666)
-		if err != nil {
-			switch {
-			case osIsNotExist(err):
-				_, err = os.Stat(volumeDir)
-				if err != nil && osIsNotExist(err) {
-					return nil, errVolumeNotFound
-				}
-				return nil, errFileNotFound
-			case osIsPermission(err):
-				return nil, errFileAccessDenied
-			case isSysErrNotDir(err):
-				return nil, errFileAccessDenied
-			case isSysErrIO(err):
-				return nil, errFaultyDisk
-			case isSysErrTooManyFiles(err):
-				return nil, errTooManyOpenFiles
-			default:
-				return nil, err
-			}
-		}
-
-		atomic.AddInt32(&s.activeIOCount, 1)
-		return &odirectreader{file, nil, nil, true, s, nil}, nil
+		file, err = disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666)
+	} else {
+		// Open the file for reading.
+		file, err = os.Open(filePath)
 	}
-
-	// Open the file for reading.
-	file, err := os.Open(filePath)
 	if err != nil {
 		switch {
 		case osIsNotExist(err):
@@ -1511,16 +1523,27 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		return nil, errIsNotRegular
 	}

-	if _, err = file.Seek(offset, io.SeekStart); err != nil {
-		return nil, err
+	atomic.AddInt32(&s.activeIOCount, 1)
+	if offset == 0 && globalStorageClass.GetDMA() == storageclass.DMAReadWrite {
+		if length <= smallFileThreshold {
+			return &odirectReader{file, nil, nil, true, true, s, nil}, nil
+		}
+		return &odirectReader{file, nil, nil, true, false, s, nil}, nil
+	}
+
+	if offset > 0 {
+		if _, err = file.Seek(offset, io.SeekStart); err != nil {
+			return nil, err
+		}
 	}

-	atomic.AddInt32(&s.activeIOCount, 1)
 	r := struct {
 		io.Reader
 		io.Closer
 	}{Reader: io.LimitReader(file, length), Closer: closeWrapper(func() error {
-		atomic.AddInt32(&s.activeIOCount, -1)
+		defer func() {
+			atomic.AddInt32(&s.activeIOCount, -1)
+		}()
 		return file.Close()
 	})}

@@ -1562,6 +1585,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 	if err != nil {
 		return err
 	}
+
 	// Stat a volume entry.
 	_, err = os.Stat(volumeDir)
 	if err != nil {
@@ -1640,8 +1664,14 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		w.Close()
 	}()

-	bufp := s.pool.Get().(*[]byte)
-	defer s.pool.Put(bufp)
+	var bufp *[]byte
+	if fileSize <= smallFileThreshold {
+		bufp = s.poolSmall.Get().(*[]byte)
+		defer s.poolSmall.Put(bufp)
+	} else {
+		bufp = s.poolLarge.Get().(*[]byte)
+		defer s.poolLarge.Put(bufp)
+	}

 	written, err := xioutil.CopyAligned(w, r, *bufp, fileSize)
 	if err != nil {
@@ -2000,7 +2030,8 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
 		if !osIsNotExist(err) {
 			return osErrToFileErr(err)
 		}
-		err = s.renameLegacyMetadata(dstVolume, dstPath)
+		// errFileNotFound comes here.
+		err = s.renameLegacyMetadata(dstVolumeDir, dstPath)
 		if err != nil && err != errFileNotFound {
 			return err
 		}
@@ -2271,11 +2302,8 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg
 	defer file.Close()

 	if algo != HighwayHash256S {
-		bufp := s.pool.Get().(*[]byte)
-		defer s.pool.Put(bufp)
-
 		h := algo.New()
-		if _, err = io.CopyBuffer(h, file, *bufp); err != nil {
+		if _, err = io.Copy(h, file); err != nil {
 			// Premature failure in reading the object,file is corrupt.
 			return errFileCorrupt
 		}
@@ -253,6 +253,91 @@ func TestXLStorageIsDirEmpty(t *testing.T) {
 	}
 }

+// TestXLStorageReadVersion - TestXLStorages the functionality implemented by xlStorage ReadVersion storage API.
+func TestXLStorageReadVersion(t *testing.T) {
+	// create xlStorage test setup
+	xlStorage, path, err := newXLStorageTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create xlStorage test setup, %s", err)
+	}
+
+	defer os.RemoveAll(path)
+
+	xlMeta, _ := ioutil.ReadFile("testdata/xl.meta")
+
+	// Create files for the test cases.
+	if err = xlStorage.MakeVol(context.Background(), "exists"); err != nil {
+		t.Fatalf("Unable to create a volume \"exists\", %s", err)
+	}
+	if err = xlStorage.AppendFile(context.Background(), "exists", "as-directory/as-file/xl.meta", xlMeta); err != nil {
+		t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err)
+	}
+	if err = xlStorage.AppendFile(context.Background(), "exists", "as-file/xl.meta", xlMeta); err != nil {
+		t.Fatalf("Unable to create a file \"as-file\", %s", err)
+	}
+	if err = xlStorage.AppendFile(context.Background(), "exists", "as-file-parent/xl.meta", xlMeta); err != nil {
+		t.Fatalf("Unable to create a file \"as-file-parent\", %s", err)
+	}
+
+	// TestXLStoragecases to validate different conditions for ReadVersion API.
+	testCases := []struct {
+		volume string
+		path   string
+		err    error
+	}{
+		// TestXLStorage case - 1.
+		// Validate volume does not exist.
+		{
+			volume: "i-dont-exist",
+			path:   "",
+			err:    errVolumeNotFound,
+		},
+		// TestXLStorage case - 2.
+		// Validate bad condition file does not exist.
+		{
+			volume: "exists",
+			path:   "as-file-not-found",
+			err:    errFileNotFound,
+		},
+		// TestXLStorage case - 3.
+		// Validate bad condition file exists as prefix/directory and
+		// we are attempting to read it.
+		{
+			volume: "exists",
+			path:   "as-directory",
+			err:    errFileNotFound,
+		},
+		// TestXLStorage case - 4.
+		{
+			volume: "exists",
+			path:   "as-file-parent/as-file",
+			err:    errFileNotFound,
+		},
+		// TestXLStorage case - 5.
+		// Validate the good condition file exists and we are able to read it.
+		{
+			volume: "exists",
+			path:   "as-file",
+			err:    nil,
+		},
+		// TestXLStorage case - 6.
+		// TestXLStorage case with invalid volume name.
+		{
+			volume: "ab",
+			path:   "as-file",
+			err:    errVolumeNotFound,
+		},
+	}
+
+	// Run through all the test cases and validate for ReadVersion.
+	for i, testCase := range testCases {
+		_, err = xlStorage.ReadVersion(context.Background(), testCase.volume, testCase.path, "", false)
+		if err != testCase.err {
+			t.Fatalf("TestXLStorage %d: Expected err \"%s\", got err \"%s\"", i+1, testCase.err, err)
+		}
+	}
+}
+
 // TestXLStorageReadAll - TestXLStorages the functionality implemented by xlStorage ReadAll storage API.
 func TestXLStorageReadAll(t *testing.T) {
 	// create xlStorage test setup
@@ -703,8 +788,8 @@ func TestXLStorageListVols(t *testing.T) {
 	}
 }

-// TestXLStorageXlStorageListDir -  TestXLStorages validate the directory listing functionality provided by xlStorage.ListDir .
-func TestXLStorageXlStorageListDir(t *testing.T) {
+// TestXLStorageListDir -  TestXLStorages validate the directory listing functionality provided by xlStorage.ListDir .
+func TestXLStorageListDir(t *testing.T) {
 	// create xlStorage test setup
 	xlStorage, path, err := newXLStorageTestSetup()
 	if err != nil {
@@ -1692,8 +1777,9 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	w := newStreamingBitrotWriter(xlStorage, volName, fileName, size, algo, shardSize)
 	reader := bytes.NewReader(data)
 	for {
-		// Using io.CopyBuffer instead of this loop will not work for us as io.CopyBuffer
-		// will use bytes.Buffer.ReadConfig() which will not do shardSize'ed writes causing error.
+		// Using io.Copy instead of this loop will not work for us as io.Copy
+		// will use bytes.Reader.WriteTo() which will not do shardSize'ed writes
+		// causing error.
 		n, err := reader.Read(shard)
 		w.Write(shard[:n])
 		if err == nil {
@@ -23,7 +23,7 @@ if [ "${1}" != "minio" ]; then
 fi

 ## Look for docker secrets at given absolute path or in default documented location.
-docker_secrets_env() {
+docker_secrets_env_old() {
    if [ -f "$MINIO_ACCESS_KEY_FILE" ]; then
        ACCESS_KEY_FILE="$MINIO_ACCESS_KEY_FILE"
    else
@@ -47,6 +47,30 @@ docker_secrets_env() {
    fi
 }

+docker_secrets_env() {
+    if [ -f "$MINIO_ROOT_USER_FILE" ]; then
+        ROOT_USER_FILE="$MINIO_ROOT_USER_FILE"
+    else
+        ROOT_USER_FILE="/run/secrets/$MINIO_ROOT_USER_FILE"
+    fi
+    if [ -f "$MINIO_ROOT_PASSWORD_FILE" ]; then
+        SECRET_KEY_FILE="$MINIO_ROOT_PASSWORD_FILE"
+    else
+        SECRET_KEY_FILE="/run/secrets/$MINIO_ROOT_PASSWORD_FILE"
+    fi
+
+    if [ -f "$ROOT_USER_FILE" ] && [ -f "$SECRET_KEY_FILE" ]; then
+        if [ -f "$ROOT_USER_FILE" ]; then
+            MINIO_ROOT_USER="$(cat "$ROOT_USER_FILE")"
+            export MINIO_ROOT_USER
+        fi
+        if [ -f "$SECRET_KEY_FILE" ]; then
+            MINIO_ROOT_PASSWORD="$(cat "$SECRET_KEY_FILE")"
+            export MINIO_ROOT_PASSWORD
+        fi
+    fi
+}
+
 ## Set KMS_MASTER_KEY from docker secrets if provided
 docker_kms_encryption_env() {
    if [ -f "$MINIO_KMS_MASTER_KEY_FILE" ]; then
@@ -88,6 +112,9 @@ docker_switch_user() {
    fi
 }

+## Set access env from secrets if necessary.
+docker_secrets_env_old
+
 ## Set access env from secrets if necessary.
 docker_secrets_env

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Poorna Krishnamoorthy	7090bcc8e0	fix: doc links and delete replication permissions enforcement (#11285 )	2021-01-15 15:22:55 -08:00
Harshavardhana	c222bde14b	fix: use common logging implementation for DNSCache (#11284 )	2021-01-15 14:04:56 -08:00
Harshavardhana	4e06a72632	fix broken URL to k8s operator	2021-01-15 11:55:36 -08:00
Ravind Kumar	16040bc544	Update README for cleaner navigation, improved quickstart experience (#11277 )	2021-01-15 11:54:05 -08:00
Harshavardhana	cc2d887e0e	fix: whitespace and formatting in replication docs	2021-01-14 22:58:53 -08:00
Poorna Krishnamoorthy	c1b4b24236	Update replication docs (#11279 )	2021-01-15 10:22:57 +05:30
Poorna Krishnamoorthy	feaf8dfb9a	Fix replication status reported on completion (#11273 ) Fixes: #11272	2021-01-13 11:52:28 -08:00
Harshavardhana	628ef081d1	fix: preserve cache calculated previously while moving from v2 to v3 (#11269 ) This ensures that all the prometheus monitoring and usage trackers to avoid alerts configured, although we cannot support v1 to v2 here - we can v2 to v3.	2021-01-13 09:58:08 -08:00
Harshavardhana	44dff36ff7	listing with prefix prefixed with '/' should be ignored (#11268 ) fixes #11265	2021-01-13 09:44:11 -08:00
Poorna Krishnamoorthy	b97d53b29c	fix remote target healthcheck (#11267 )	2021-01-12 20:48:04 -08:00
Aditya Manthramurthy	00af9881b0	LDAP doc fix: remove repeated paragraph and add emphasis (#11266 )	2021-01-12 15:44:31 -08:00
Kanagaraj M	e09196d626	browser: fix file uploads with '#' in the name (#11261 ) When the file name or the prefix contains `#`, whatever comes after is ingored. This is fixed by encoding the object path using encodeURIComponent. Fixes #11112	2021-01-12 12:47:46 -08:00
Harshavardhana	1a5775e2e8	enable small and large file optimization (#11260 ) - for large objects we found that 1MiB block for r/w respectively. - for small objects we found that 128KiB block for r/w respectively.	2021-01-12 10:20:39 -08:00
Anis Elleuch	e2579b1f5a	azure: Use default upload parameters to avoid consuming too much memory (#11251 ) A lot of memory is consumed when uploading small files in parallel, use the default upload parameters and add MINIO_AZURE_UPLOAD_CONCURRENCY for users to tweak.	2021-01-11 22:48:09 -08:00
Poorna Krishnamoorthy	7824e19d20	Allow synchronous replication if enabled. (#11165 ) Synchronous replication can be enabled by setting the --sync flag while adding a remote replication target. This PR also adds proxying on GET/HEAD to another node in a active-active replication setup in the event of a 404 on the current node.	2021-01-11 22:36:51 -08:00
Harshavardhana	317305d5f9	fix: regression in adding new replication targets (#11257 )	2021-01-11 09:08:42 -08:00
Harshavardhana	e4e117faab	fix: enable xl.json to xl.meta only if legacy drive is found (#11255 ) another optimization is renameLegacyMetadata() never needs to validate bucket with os.Stat() again, leading to reduction in one extra syscall.	2021-01-11 02:27:04 -08:00
George Tsatsis	e8176fe978	Use `-new` during OpenSSL certificate generation (#11199 ) As per https://stackoverflow.com/a/3758443/8156177, OpenSSL expects a certificate via STDIN. `-new` will allow a new certificate to be generated instead.	2021-01-11 02:24:50 -08:00
Harshavardhana	828602d672	fix: nancy github URL has changed	2021-01-10 13:37:05 -08:00
Minio Trusted	d9224fbc65	Update yaml files to latest version RELEASE.2021-01-08T21-18-21Z	2021-01-08 21:37:35 +00:00
Klaus Post	51dad1d130	Fix missing GetObjectNInfo Closure (#11243 ) Review for missing Close of returned value from `GetObjectNInfo`. This was often obscured by the stuff that auto-unlocks when reaching EOF.	2021-01-08 10:12:26 -08:00
Harshavardhana	4593b146be	fix: print errors only when metacache status has errors (#11248 )	2021-01-08 16:52:19 +05:30
Harshavardhana	f21d650ed4	fix: readData in bulk call using messagepack byte wrappers (#11228 ) This PR refactors the way we use buffers for O_DIRECT and to re-use those buffers for messagepack reader writer. After some extensive benchmarking found that not all objects have this benefit, and only objects smaller than 64KiB see this benefit overall. Benefits are seen from almost all objects from 1KiB - 32KiB Beyond this no objects see benefit with bulk call approach as the latency of bytes sent over the wire v/s streaming content directly from disk negate each other with no remarkable benefits. All other optimizations include reuse of msgp.Reader, msgp.Writer using sync.Pool's for all internode calls.	2021-01-07 19:27:31 -08:00
Harshavardhana	a4f6705874	expire stale locks when owner is down (#11247 ) fixes #11246	2021-01-07 19:16:18 -08:00
Poorna Krishnamoorthy	b35b537e3f	Pass versionID to checkReplicateDelete in web handler (#11244 )	2021-01-07 15:28:27 -08:00
Harshavardhana	5c52d5ffc7	fix: treat errVolumeNotFound as EOF error in listPathRaw (#11238 )	2021-01-07 09:52:53 -08:00
Harshavardhana	f0808bb2e5	fix: getObject fd leaks in transition and replication code (#11237 )	2021-01-06 16:13:10 -08:00
Harshavardhana	a6dee21092	initialize IAM store before Init() to avoid any crash (#11236 )	2021-01-06 13:40:20 -08:00
Anis Elleuch	6f781c5e7a	heal: Reduce whitespace ticker to 5 seconds (#11234 ) 30 seconds white spaces is long for some setups which time out when no read activity in short time, reduce the subnet health white space ticker to 5 seconds, since it has no cost at all.	2021-01-06 13:29:50 -08:00
Harshavardhana	f8ca859790	fix: server/gateway banner formatting (#11230 )	2021-01-06 10:38:07 -08:00
Kanagaraj M	b78521cd69	update license verifier to use updated keys (#11197 )	2021-01-06 10:17:05 -08:00
Harshavardhana	76e2713ffe	fix: use buffers only when necessary for io.Copy() (#11229 ) Use separate sync.Pool for writes/reads Avoid passing buffers for io.CopyBuffer() if the writer or reader implement io.WriteTo or io.ReadFrom respectively then its useless for sync.Pool to allocate buffers on its own since that will be completely ignored by the io.CopyBuffer Go implementation. Improve this wherever we see this to be optimal. This allows us to be more efficient on memory usage. ``` 385 // copyBuffer is the actual implementation of Copy and CopyBuffer. 386 // if buf is nil, one is allocated. 387 func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) { 388 // If the reader has a WriteTo method, use it to do the copy. 389 // Avoids an allocation and a copy. 390 if wt, ok := src.(WriterTo); ok { 391 return wt.WriteTo(dst) 392 } 393 // Similarly, if the writer has a ReadFrom method, use it to do the copy. 394 if rt, ok := dst.(ReaderFrom); ok { 395 return rt.ReadFrom(src) 396 } ``` From readahead package ``` // WriteTo writes data to w until there's no more data to write or when an error occurs. // The return value n is the number of bytes written. // Any error encountered during the write is also returned. func (a *reader) WriteTo(w io.Writer) (n int64, err error) { if a.err != nil { return 0, a.err } n = 0 for { err = a.fill() if err != nil { return n, err } n2, err := w.Write(a.cur.buffer()) a.cur.inc(n2) n += int64(n2) if err != nil { return n, err } ```	2021-01-06 09:36:55 -08:00
Harshavardhana	b5d291ea88	fix: rename remaining zone -> pool (#11231 )	2021-01-06 09:35:47 -08:00
Klaus Post	eb9172eecb	Allow Compression + encryption (#11103 )	2021-01-05 20:08:35 -08:00
Klaus Post	97a4c120e9	Add Optimization as a type of change (#11221 ) * Add Optimization as a type of change. * Make the checkmark affirmative instead of disaffirming	2021-01-05 14:53:52 -08:00
Poorna Krishnamoorthy	64bddf47d8	Pass deletemarker correctly to replicate opts (#11227 ) fixes: #11180	2021-01-05 14:12:37 -08:00
Harshavardhana	4ed45ce543	fix: healing buckets during pool expansion (#11224 ) fixes #11209	2021-01-05 13:24:22 -08:00
Klaus Post	ad511b0eb8	tests: Fix occasional data race (#11223 ) CI tests could trigger a data race. Servers are generally not expected to reinitialize, so tests could trigger data races when reinitializing and async operations are running. We add the option to safely reset global vars instead of overwriting. Fixes races like: ``` WARNING: DATA RACE Read at 0x00000477ab18 by goroutine 1159: github.com/minio/minio/cmd.FileInfo.ToObjectInfo() /home/runner/work/minio/minio/cmd/erasure-metadata.go:105 +0x16d github.com/minio/minio/cmd.erasureObjects.putObject() /home/runner/work/minio/minio/cmd/erasure-object.go:748 +0x13f8 github.com/minio/minio/cmd.(erasureObjects).listPath.func3.2() /home/runner/work/minio/minio/cmd/metacache-set.go:682 +0x7d3 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1.2() /home/runner/work/minio/minio/cmd/metacache-stream.go:777 +0x1c4 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1() /home/runner/work/minio/minio/cmd/metacache-stream.go:806 +0x614 Previous write at 0x00000477ab18 by goroutine 1269: [failed to restore the stack] Goroutine 1159 (running) created at: github.com/minio/minio/cmd.newMetacacheBlockWriter() /home/runner/work/minio/minio/cmd/metacache-stream.go:760 +0x112 github.com/minio/minio/cmd.(erasureObjects).listPath.func3() /home/runner/work/minio/minio/cmd/metacache-set.go:672 +0xe22 Goroutine 1269 (running) created at: testing.(T).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1095 +0x537 testing.runTests.func1() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1339 +0xa6 testing.tRunner() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1050 +0x1eb testing.runTests() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1337 +0x594 testing.(M).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1252 +0x2ff github.com/minio/minio/cmd.TestMain() /home/runner/work/minio/minio/cmd/test-utils_test.go:120 +0x44e main.main() _testmain.go:1408 +0x223 ================== ================== WARNING: DATA RACE Read at 0x00000477aae8 by goroutine 1159: github.com/minio/minio/cmd.(BucketVersioningSys).Enabled() /home/runner/work/minio/minio/cmd/bucket-versioning.go:26 +0x52 github.com/minio/minio/cmd.FileInfo.ToObjectInfo() /home/runner/work/minio/minio/cmd/erasure-metadata.go:105 +0x197 github.com/minio/minio/cmd.erasureObjects.putObject() /home/runner/work/minio/minio/cmd/erasure-object.go:748 +0x13f8 github.com/minio/minio/cmd.(erasureObjects).listPath.func3.2() /home/runner/work/minio/minio/cmd/metacache-set.go:682 +0x7d3 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1.2() /home/runner/work/minio/minio/cmd/metacache-stream.go:777 +0x1c4 github.com/minio/minio/cmd.newMetacacheBlockWriter.func1() /home/runner/work/minio/minio/cmd/metacache-stream.go:806 +0x614 Previous write at 0x00000477aae8 by goroutine 1269: [failed to restore the stack] Goroutine 1159 (running) created at: github.com/minio/minio/cmd.newMetacacheBlockWriter() /home/runner/work/minio/minio/cmd/metacache-stream.go:760 +0x112 github.com/minio/minio/cmd.(erasureObjects).listPath.func3() /home/runner/work/minio/minio/cmd/metacache-set.go:672 +0xe22 Goroutine 1269 (running) created at: testing.(T).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1095 +0x537 testing.runTests.func1() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1339 +0xa6 testing.tRunner() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1050 +0x1eb testing.runTests() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1337 +0x594 testing.(*M).Run() /opt/hostedtoolcache/go/1.14.13/x64/src/testing/testing.go:1252 +0x2ff github.com/minio/minio/cmd.TestMain() /home/runner/work/minio/minio/cmd/test-utils_test.go:120 +0x44e main.main() _testmain.go:1408 +0x223 ================== ```	2021-01-05 10:45:26 -08:00
Harshavardhana	cb0eaeaad8	feat: migrate to ROOT_USER/PASSWORD from ACCESS/SECRET_KEY (#11185 )	2021-01-05 10:22:57 -08:00
Minio Trusted	f3f0041ad0	Update yaml files to latest version RELEASE.2021-01-05T05-22-38Z	2021-01-05 05:42:54 +00:00