add validation logs for configured Logger/Audit HTTP targets (#10274)

extra logs in-case of misconfiguration of audit/logger targets

.github/workflows/go.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.13.x]
+        go-version: [1.14.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

CREDITS

@@ -10293,25 +10293,6 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
       the Mozilla Public License, v. 2.0.
 
 
-================================================================
-
-github.com/inconshreveable/go-update
-https://github.com/inconshreveable/go-update
-----------------------------------------------------------------
-Copyright 2015 Alan Shreve
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
 ================================================================
 
 github.com/jcmturner/gofork
@@ -11470,8 +11451,216 @@ https://github.com/minio/md5-simd
 
 ================================================================
 
-github.com/minio/minio-go/v6
-https://github.com/minio/minio-go/v6
+github.com/minio/minio-go/v7
+https://github.com/minio/minio-go/v7
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
+github.com/minio/selfupdate
+https://github.com/minio/selfupdate
 ----------------------------------------------------------------
 
                                  Apache License

Dockerfile.arm.release

@@ -23,8 +23,7 @@ RUN \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS && \
-     /usr/bin/verify-minio.sh
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS
 
 EXPOSE 9000
 

Dockerfile.arm64.release

@@ -23,7 +23,7 @@ RUN \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
      /usr/bin/verify-minio.sh
 
 EXPOSE 9000

Dockerfile.ppc64le.release

@@ -22,7 +22,7 @@ RUN \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
      /usr/bin/verify-minio.sh
 
 EXPOSE 9000

Dockerfile.release

@@ -20,7 +20,7 @@ RUN \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
      /usr/bin/verify-minio.sh
 
 EXPOSE 9000

Dockerfile.s390x.release

@@ -22,7 +22,7 @@ RUN \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
      /usr/bin/verify-minio.sh
 
 EXPOSE 9000

README_zh_CN.md

@@ -7,29 +7,33 @@ MinIO是一个非常轻量的服务,可以很简单的和其他应用的结合
 ## Docker 容器
 ### 稳定版
 ```
-docker pull minio/minio
-docker run -p 9000:9000 minio/minio server /data
+docker run -p 9000:9000 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  minio/minio server /data
 ```
 
 ### 尝鲜版
 ```
-docker pull minio/minio:edge
-docker run -p 9000:9000 minio/minio:edge server /data
+docker run -p 9000:9000 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  minio/minio:edge server /data
 ```
-更多Docker部署信息请访问 [这里](https://docs.min.io/docs/minio-docker-quickstart-guide)
+
+> 提示：除非你通过`-it`(TTY交互)参数启动容器，否则Docker将不会显示默认的密钥。一般情况下，并不推荐使用容器的默认密钥，更多Docker部署信息请访问 [这里](https://docs.min.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
-### Homebrew
+### Homebrew（推荐）
 使用 [Homebrew](http://brew.sh/)安装minio
 
 ```sh
 brew install minio/stable/minio
 minio server /data
 ```
-#### Note
-如果你之前使用 `brew install minio`安装过minio, 可以用 `minio/stable/minio` 官方镜像进行重装. 由于golang 1.8的bug,homebrew版本不太稳定。
 
-```
+> 提示：如果你之前使用 `brew install minio`安装过minio, 可以用 `minio/stable/minio` 官方镜像进行重装. 由于golang 1.8的bug,homebrew版本不太稳定。
+```sh
 brew uninstall minio
 brew install minio/stable/minio
 ```
@@ -49,6 +53,16 @@ chmod 755 minio
 | ---------- | --------     | ------                                                     |
 | GNU/Linux  | 64-bit Intel | https://dl.min.io/server/minio/release/linux-amd64/minio |
 ```sh
+wget https://dl.min.io/server/minio/release/linux-amd64/minio
+chmod +x minio
+./minio server /data
+```
+
+| 操作系统    | CPU架构      | 地址                                                        |
+| ---------- | --------     | ------                                                     |
+| GNU/Linux  | ppc64le      | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
+```sh
+wget https://dl.min.io/server/minio/release/linux-ppc64le/minio
 chmod +x minio
 ./minio server /data
 ```
@@ -64,7 +78,7 @@ minio.exe server D:\Photos
 
 ## FreeBSD
 ### Port
-使用 [pkg](https://github.com/freebsd/pkg)进行安装。
+使用 [pkg](https://github.com/freebsd/pkg)进行安装，, MinIO官方并没有提供FreeBSD二进制文件， 它由FreeBSD上游维护，点击 [这里](https://www.freshports.org/www/minio)查看。
 
 ```sh
 pkg install minio
@@ -75,14 +89,68 @@ service minio start
 
 ## 使用源码安装
 
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install).
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.13](https://golang.org/dl/#stable)
 
 ```sh
-go get -u github.com/minio/minio
+GO111MODULE=on go get github.com/minio/minio
+```
+
+## 为防火墙设置允许访问的端口
+
+默认情况下，MinIO 使用端口9000来侦听传入的连接。如果你的平台默认阻止了该端口，则需要启用对该端口的访问。
+
+### ufw
+
+对于启用了ufw的主机（基于Debian的发行版）, 你可以通过`ufw`命令允许指定端口上的所有流量连接. 通过如下命令允许访问端口9000
+
+```sh
+ufw allow 9000
+```
+
+如下命令允许端口9000-9010上的所有传入流量。
+
+```sh
+ufw allow 9000:9010/tcp
+```
+
+### firewall-cmd
+
+对于启用了firewall-cmd的主机（CentOS）, 你可以通过`firewall-cmd`命令允许指定端口上的所有流量连接。 通过如下命令允许访问端口9000
+
+```sh
+firewall-cmd --get-active-zones
+```
+
+这个命令获取当前正在使用的区域。 现在，就可以为以上返回的区域应用端口规则了。 假如返回的区域是 `public`, 使用如下命令
+
+```sh
+firewall-cmd --zone=public --add-port=9000/tcp --permanent
+```
+
+这里的`permanent`参数表示持久化存储规则，可用于防火墙启动、重启和重新加载。 最后，需要防火墙重新加载，让我们刚刚的修改生效。
+
+```sh
+firewall-cmd --reload
+```
+
+### iptables
+
+对于启用了iptables的主机（RHEL, CentOS, etc）, 你可以通过`iptables`命令允许指定端口上的所有流量连接。 通过如下命令允许访问端口9000
+
+```sh
+iptables -A INPUT -p tcp --dport 9000 -j ACCEPT
+service iptables restart
+```
+
+如下命令允许端口9000-9010上的所有传入流量。
+
+```sh
+iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
+service iptables restart
 ```
 
 ## 使用MinIO浏览器进行验证
-安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
+MinIO Server带有一个嵌入的Web对象浏览器，安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
 
 ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 
@@ -94,6 +162,25 @@ go get -u github.com/minio/minio
 
 上述描述对所有网关后端同样有效。
 
+## 升级 MinIO
+MinIO 服务端支持滚动升级, 也就是说你可以一次更新分布式集群中的一个MinIO实例。 这样可以在不停机的情况下进行升级。可以通过将二进制文件替换为最新版本并以滚动方式重新启动所有服务器来手动完成升级。但是, 我们建议所有用户从客户端使用 [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) 命令升级。 这将同时更新集群中的所有节点并重新启动它们, 如下命令所示:
+
+```
+mc admin update <minio alias, e.g., myminio>
+```
+
+> 注意: 有些发行版可能不允许滚动升级，这通常在发行说明中提到，所以建议在升级之前阅读发行说明。在这种情况下，建议使用`mc admin update`升级机制来一次升级所有服务器。
+
+### MinIO升级时要记住的重要事项
+
+- `mc admin update` 命令仅当运行MinIO的用户对二进制文件所在的父目录具有写权限时才工作, 比如当前二进制文件位于`/usr/local/bin/minio`, 你需要具备`/usr/local/bin`目录的写权限.
+- `mc admin update` 命令同时更新并重新启动所有服务器，应用程序将在升级后重试并继续各自的操作。
+- `mc admin update` 命令在 kubernetes/container 环境下是不能用的, 容器环境提供了它自己的更新机制来更新。
+- 对于联盟部署模式，应分别针对每个群集运行`mc admin update`。 在成功更新所有群集之前，不要将`mc`更新为任何新版本。
+- 如果将`kes`用作MinIO的KMS，只需替换二进制文件并重新启动`kes`，可以在 [这里](https://github.com/minio/kes/wiki) 找到有关`kes`的更多信息。
+- 如果将Vault作为MinIO的KMS，请确保已遵循如下Vault升级过程的概述：https://www.vaultproject.io/docs/upgrading/index.html
+- 如果将MindIO与etcd配合使用, 请确保已遵循如下etcd升级过程的概述: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
+
 ## 了解更多
 - [MinIO纠删码入门](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [`mc`快速入门](https://docs.min.io/docs/minio-client-quickstart-guide)
@@ -104,3 +191,6 @@ go get -u github.com/minio/minio
 
 ## 如何参与到MinIO项目
 请参考 [贡献者指南](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)。欢迎各位中国程序员加到MinIO项目中。
+
+## 授权许可
+MinIO的使用受 Apache 2.0 License 约束，你可以在 [LICENSE](./LICENSE) 查看许可。

cmd/admin-bucket-handlers.go

@@ -20,20 +20,20 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 )
 
 const (
-	bucketQuotaConfigFile        = "quota.json"
-	bucketReplicationTargetsFile = "replication-targets.json"
+	bucketQuotaConfigFile = "quota.json"
+	bucketTargetsFile     = "bucket-targets.json"
 )
 
 // PutBucketQuotaConfigHandler - PUT Bucket quota configuration.
@@ -121,16 +121,16 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	writeSuccessResponseJSON(w, configData)
 }
 
-// SetBucketReplicationTargetHandler - sets a replication target for bucket
-func (a adminAPIHandlers) SetBucketReplicationTargetHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetBucketReplicationTarget")
+// SetRemoteTargetHandler - sets a remote target for bucket
+func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetBucketTarget")
 
-	defer logger.AuditLog(w, r, "SetBucketReplicationTarget", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketReplicationTargetAction)
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -139,11 +139,6 @@ func (a adminAPIHandlers) SetBucketReplicationTargetHandler(w http.ResponseWrite
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
-	// Turn off replication if disk crawl is unavailable.
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketReplicationDisabledError), r.URL)
-		return
-	}
 
 	// Check if bucket exists.
 	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
@@ -151,17 +146,11 @@ func (a adminAPIHandlers) SetBucketReplicationTargetHandler(w http.ResponseWrite
 		return
 	}
 
-	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrReplicationBucketNeedsVersioningError), r.URL)
-		return
-	}
-
 	cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
 	}
-
 	password := cred.SecretKey
 
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
@@ -169,84 +158,134 @@ func (a adminAPIHandlers) SetBucketReplicationTargetHandler(w http.ResponseWrite
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	var target madmin.BucketReplicationTarget
+	var target madmin.BucketTarget
 	if err = json.Unmarshal(reqBytes, &target); err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	target.Arn = globalBucketReplicationSys.getReplicationARN(target.URL())
-	tgtBytes, err := json.Marshal(&target)
+	host, port, err := net.SplitHostPort(target.Endpoint)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-	if err = globalBucketMetadataSys.Update(bucket, bucketReplicationTargetsFile, tgtBytes); err != nil {
+	sameTarget, _ := isLocalHost(host, port, globalMinioPort)
+	if sameTarget && bucket == target.TargetBucket {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
+		return
+	}
+	target.SourceBucket = bucket
+	target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
+	if target.Arn == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+
+	}
+	tgtBytes, err := json.Marshal(&targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if err = globalBucketReplicationSys.SetTarget(ctx, bucket, &target); err != nil {
+
+	data, err := json.Marshal(target.Arn)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}
+
+// ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target
+// for a particular ARN type
+func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListBucketTargets")
+
+	defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r))
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	arnType := vars["type"]
+
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketTargetAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if bucket != "" {
+		if _, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+	targets := globalBucketTargetSys.ListTargets(ctx, bucket, arnType)
+	data, err := json.Marshal(targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}
+
+// RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN
+func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveBucketTarget")
+
+	defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r))
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	arn := vars["arn"]
+
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
-}
-
-// GetBucketReplicationTargetsHandler - gets bucket replication targets for a particular bucket
-func (a adminAPIHandlers) GetBucketReplicationTargetsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetBucketReplicationTarget")
-
-	defer logger.AuditLog(w, r, "GetBucketReplicationTarget", mustGetClaimsFromToken(r))
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	// Get current object layer instance.
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketReplicationTargetAction)
-	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+	if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, arn); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	target, err := globalBucketMetadataSys.GetReplicationTargetConfig(bucket)
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
-	// remove secretKey from creds
-	var tgt madmin.BucketReplicationTarget
-	if !target.Empty() {
-		var creds auth.Credentials
-		creds.AccessKey = target.Credentials.AccessKey
-		tgt = madmin.BucketReplicationTarget{Endpoint: target.Endpoint, TargetBucket: target.TargetBucket, Credentials: &creds, Arn: target.Arn}
-	}
-	data, err := json.Marshal(tgt)
+	tgtBytes, err := json.Marshal(&targets)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	// Write success response.
-	writeSuccessResponseJSON(w, data)
-}
 
-// GetBucketReplicationARNHandler - gets replication ARN for a particular remote
-func (a adminAPIHandlers) GetBucketReplicationARNHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetBucketReplicationARN")
-
-	defer logger.AuditLog(w, r, "GetBucketReplicationARN", mustGetClaimsFromToken(r))
-	vars := mux.Vars(r)
-	rURL := vars["url"]
-	// Get current object layer instance.
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketReplicationTargetAction)
-	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-	data, err := json.Marshal(globalBucketReplicationSys.getARN(rURL))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 	// Write success response.
-	writeSuccessResponseJSON(w, data)
+	writeSuccessNoContent(w)
 }

cmd/admin-handlers-config-kv.go

@@ -136,7 +136,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -271,7 +271,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -383,7 +383,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}

cmd/admin-handlers.go

@@ -695,7 +695,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					// Start writing response to client
 					started = true
 					setCommonHeaders(w)
-					w.Header().Set(xhttp.ContentType, "text/event-stream")
+					setEventStreamHeaders(w)
 					// Set 200 OK status
 					w.WriteHeader(200)
 				}
@@ -799,6 +799,59 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, r, respCh)
 }
 
+func getAggregatedBackgroundHealState(ctx context.Context, failOnErr bool) (madmin.BgHealState, error) {
+	var bgHealStates []madmin.BgHealState
+
+	localHealState, ok := getLocalBackgroundHealStatus()
+	if !ok {
+		if failOnErr {
+			return madmin.BgHealState{}, errServerNotInitialized
+		}
+	}
+
+	// Get local heal status first
+	bgHealStates = append(bgHealStates, localHealState)
+
+	if globalIsDistErasure {
+		// Get heal status from other peers
+		peersHealStates, nerrs := globalNotificationSys.BackgroundHealStatus()
+		for _, nerr := range nerrs {
+			if nerr.Err != nil {
+				if failOnErr {
+					return madmin.BgHealState{}, nerr.Err
+				}
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+		bgHealStates = append(bgHealStates, peersHealStates...)
+	}
+
+	// Aggregate healing result
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+		HealDisks:         bgHealStates[0].HealDisks,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
+	for _, state := range bgHealStates {
+		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		aggregatedHealStateResult.HealDisks = append(aggregatedHealStateResult.HealDisks, state.HealDisks...)
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
+		}
+	}
+
+	return aggregatedHealStateResult, nil
+}
+
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
@@ -815,39 +868,8 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	var bgHealStates []madmin.BgHealState
-
-	// Get local heal status first
-	bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
-
-	if globalIsDistErasure {
-		// Get heal status from other peers
-		peersHealStates := globalNotificationSys.BackgroundHealStatus()
-		bgHealStates = append(bgHealStates, peersHealStates...)
-	}
-
-	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{
-		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
-		LastHealActivity:  bgHealStates[0].LastHealActivity,
-		NextHealRound:     bgHealStates[0].NextHealRound,
-	}
-
-	bgHealStates = bgHealStates[1:]
-
-	for _, state := range bgHealStates {
-		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
-			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
-			// The node which has the last heal activity means its
-			// is the node that is orchestrating self healing operations,
-			// which also means it is the same node which decides when
-			// the next self healing operation will be done.
-			aggregatedHealStateResult.NextHealRound = state.NextHealRound
-		}
-	}
-
-	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
+	aggregateHealStateResult, _ := getAggregatedBackgroundHealState(r.Context(), false)
+	if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -995,7 +1017,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
@@ -1064,7 +1086,8 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	// This is needed to make r.Context().Done() work as
 	// expected in case of read timeout
 	w.Header().Add("Connection", "close")
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+
+	setEventStreamHeaders(w)
 
 	logCh := make(chan interface{}, 4000)
 
@@ -1223,7 +1246,9 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	setCommonHeaders(w)
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+
+	setEventStreamHeaders(w)
+
 	w.WriteHeader(http.StatusOK)
 
 	errResp := func(err error) {
@@ -1628,9 +1653,6 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 
 // checkConnection - ping an endpoint , return err in case of no connection
 func checkConnection(endpointStr string, timeout time.Duration) error {
-	tr := newCustomHTTPTransport(&tls.Config{RootCAs: globalRootCAs}, timeout)()
-	defer tr.CloseIdleConnections()
-
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
 	defer cancel()
 
@@ -1639,7 +1661,20 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 		return err
 	}
 
-	client := &http.Client{Transport: tr}
+	client := &http.Client{Transport: &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           xhttp.NewCustomDialContext(timeout),
+		ResponseHeaderTimeout: 5 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
+		ExpectContinueTimeout: 5 * time.Second,
+		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
+		// Go net/http automatically unzip if content-type is
+		// gzip disable this feature, as we are always interested
+		// in raw stream.
+		DisableCompression: true,
+	}}
+	defer client.CloseIdleConnections()
+
 	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
 		return err

cmd/admin-heal-ops.go

@@ -88,7 +88,8 @@ type allHealState struct {
 	sync.Mutex
 
 	// map of heal path to heal sequence
-	healSeqMap map[string]*healSequence
+	healSeqMap     map[string]*healSequence
+	healLocalDisks []Endpoints
 }
 
 // newHealState - initialize global heal state management
@@ -102,6 +103,22 @@ func newHealState() *allHealState {
 	return healState
 }
 
+func (ahs *allHealState) getHealLocalDisks() []Endpoints {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	healLocalDisks := make([]Endpoints, len(ahs.healLocalDisks))
+	copy(healLocalDisks, ahs.healLocalDisks)
+	return healLocalDisks
+}
+
+func (ahs *allHealState) updateHealLocalDisks(healLocalDisks []Endpoints) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	ahs.healLocalDisks = healLocalDisks
+}
+
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
@@ -485,6 +502,10 @@ func (h *healSequence) isQuitting() bool {
 // check if the heal sequence has ended
 func (h *healSequence) hasEnded() bool {
 	h.mutex.RLock()
+	// background heal never ends
+	if h.clientToken == bgHealingUUID {
+		return false
+	}
 	ended := len(h.currentStatus.Items) == 0 || h.currentStatus.Summary == healStoppedStatus || h.currentStatus.Summary == healFinishedStatus
 	h.mutex.RUnlock()
 	return ended

cmd/admin-router.go

@@ -182,15 +182,15 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
 			}
 			// Bucket replication operations
-			// GetBucketReplicationTargetHandler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-replication-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.GetBucketReplicationTargetsHandler)).Queries("bucket", "{bucket:.*}")
-			// GetBucketReplicationARN	Handler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-replication-arn").HandlerFunc(
-				httpTraceHdrs(adminAPI.GetBucketReplicationARNHandler)).Queries("url", "{url:.*}")
-			// SetBucketReplicationTargetHandler
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-replication-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.SetBucketReplicationTargetHandler)).Queries("bucket", "{bucket:.*}")
+			// GetBucketTargetHandler
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+			// SetRemoteTargetHandler
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+			// SetRemoteTargetHandler
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 		}
 
 		// -- Top APIs --

cmd/api-errors.go

@@ -107,8 +107,15 @@ const (
 	ErrNoSuchWebsiteConfiguration
 	ErrReplicationConfigurationNotFoundError
 	ErrReplicationDestinationNotFoundError
+	ErrReplicationDestinationMissingLock
 	ErrReplicationTargetNotFoundError
+	ErrBucketRemoteIdenticalToSource
+	ErrBucketRemoteAlreadyExists
+	ErrBucketRemoteArnTypeInvalid
+	ErrBucketRemoteArnInvalid
+	ErrBucketRemoteRemoveDisallowed
 	ErrReplicationTargetNotVersionedError
+	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrBucketReplicationDisabledError
@@ -825,15 +832,50 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication destination bucket does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrReplicationDestinationMissingLock: {
+		Code:           "ReplicationDestinationMissingLockError",
+		Description:    "The replication destination bucket does not have object locking enabled",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrReplicationTargetNotFoundError: {
-		Code:           "ReplicationTargetNotFoundError",
+		Code:           "XminioAdminReplicationTargetNotFoundError",
 		Description:    "The replication target does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrBucketRemoteIdenticalToSource: {
+		Code:           "XminioAdminRemoteIdenticalToSource",
+		Description:    "The remote target cannot be identical to source",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteAlreadyExists: {
+		Code:           "XminioAdminBucketRemoteAlreadyExists",
+		Description:    "The remote target already exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteRemoveDisallowed: {
+		Code:           "XMinioAdminRemoteRemoveDisallowed",
+		Description:    "Replication configuration exists with this ARN.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteArnTypeInvalid: {
+		Code:           "XMinioAdminRemoteARNTypeInvalid",
+		Description:    "The bucket remote ARN type is not valid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteArnInvalid: {
+		Code:           "XMinioAdminRemoteArnInvalid",
+		Description:    "The bucket remote ARN does not have correct format",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrReplicationTargetNotVersionedError: {
 		Code:           "ReplicationTargetNotVersionedError",
 		Description:    "The replication target does not have versioning enabled",
-		HTTPStatusCode: http.StatusNotFound,
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrReplicationSourceNotVersionedError: {
+		Code:           "ReplicationSourceNotVersionedError",
+		Description:    "The replication source does not have versioning enabled",
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationNeedsVersioningError: {
 		Code:           "InvalidRequest",
@@ -1879,10 +1921,22 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrReplicationConfigurationNotFoundError
 	case BucketReplicationDestinationNotFound:
 		apiErr = ErrReplicationDestinationNotFoundError
-	case BucketReplicationTargetNotFound:
+	case BucketReplicationDestinationMissingLock:
+		apiErr = ErrReplicationDestinationMissingLock
+	case BucketRemoteTargetNotFound:
 		apiErr = ErrReplicationTargetNotFoundError
+	case BucketRemoteAlreadyExists:
+		apiErr = ErrBucketRemoteAlreadyExists
+	case BucketRemoteArnTypeInvalid:
+		apiErr = ErrBucketRemoteArnTypeInvalid
+	case BucketRemoteArnInvalid:
+		apiErr = ErrBucketRemoteArnInvalid
+	case BucketRemoteRemoveDisallowed:
+		apiErr = ErrBucketRemoteRemoveDisallowed
 	case BucketReplicationTargetNotVersioned:
 		apiErr = ErrReplicationTargetNotVersionedError
+	case BucketReplicationSourceNotVersioned:
+		apiErr = ErrReplicationSourceNotVersionedError
 	case BucketQuotaExceeded:
 		apiErr = ErrAdminBucketQuotaExceeded
 	case *event.ErrInvalidEventName:

cmd/api-headers.go

@@ -38,6 +38,13 @@ func mustGetRequestID(t time.Time) string {
 	return fmt.Sprintf("%X", t.UnixNano())
 }
 
+// setEventStreamHeaders to allow proxies to avoid buffering proxy responses
+func setEventStreamHeaders(w http.ResponseWriter) {
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	w.Header().Set(xhttp.CacheControl, "no-cache") // nginx to turn off buffering
+	w.Header().Set("X-Accel-Buffering", "no")      // nginx to turn off buffering
+}
+
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
@@ -121,6 +128,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 			// values to client.
 			continue
 		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			continue
+		}
 		var isSet bool
 		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
 			if !strings.HasPrefix(k, userMetadataPrefix) {

cmd/api-response.go

@@ -564,6 +564,10 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					// values to client.
 					continue
 				}
+				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+					continue
+				}
 				content.UserMetadata[k] = v
 			}
 		}

cmd/background-heal-ops.go

@@ -116,24 +116,6 @@ func newHealRoutine() *healRoutine {
 
 }
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
-	go globalBackgroundHealRoutine.run(ctx, objAPI)
-
-	nh := newBgHealSequence()
-	// Heal any disk format and metadata early, if possible.
-	if err := nh.healDiskMeta(); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(nh.ctx, err)
-		}
-	}
-
-	globalBackgroundHealState.LaunchNewHealSequence(nh)
-}
-
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) {

cmd/background-newdisks-heal-ops.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -26,19 +27,23 @@ import (
 
 const defaultMonitorNewDiskInterval = time.Minute * 3
 
-func initLocalDisksAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	go monitorLocalDisksAndHeal(ctx, objAPI)
-}
-
-// monitorLocalDisksAndHeal - ensures that detected new disks are healed
-//  1. Only the concerned erasure set will be listed and healed
-//  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
+func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	z, ok := objAPI.(*erasureZones)
 	if !ok {
 		return
 	}
 
+	initBackgroundHealing(ctx, objAPI) // start quick background healing
+
+	localDisksInZoneHeal := getLocalDisksToHeal(objAPI)
+	globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
+
+	drivesToHeal := getDrivesToHealCount(localDisksInZoneHeal)
+	if drivesToHeal != 0 {
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+			drivesToHeal, defaultMonitorNewDiskInterval))
+	}
+
 	var bgSeq *healSequence
 	var found bool
 
@@ -50,53 +55,99 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 		time.Sleep(time.Second)
 	}
 
+	if drivesToHeal != 0 {
+		// Heal any disk format and metadata early, if possible.
+		if err := bgSeq.healDiskMeta(); err != nil {
+			if newObjectLayerFn() != nil {
+				// log only in situations, when object layer
+				// has fully initialized.
+				logger.LogIf(bgSeq.ctx, err)
+			}
+		}
+	}
+
+	go monitorLocalDisksAndHeal(ctx, z, drivesToHeal, localDisksInZoneHeal, bgSeq)
+}
+
+func getLocalDisksToHeal(objAPI ObjectLayer) []Endpoints {
+	z, ok := objAPI.(*erasureZones)
+	if !ok {
+		return nil
+	}
+
+	// Attempt a heal as the server starts-up first.
+	localDisksInZoneHeal := make([]Endpoints, len(z.zones))
+	for i, ep := range globalEndpoints {
+		localDisksToHeal := Endpoints{}
+		for _, endpoint := range ep.Endpoints {
+			if !endpoint.IsLocal {
+				continue
+			}
+			// Try to connect to the current endpoint
+			// and reformat if the current disk is not formatted
+			_, _, err := connectEndpoint(endpoint)
+			if err == errUnformattedDisk {
+				localDisksToHeal = append(localDisksToHeal, endpoint)
+			}
+		}
+		if len(localDisksToHeal) == 0 {
+			continue
+		}
+		localDisksInZoneHeal[i] = localDisksToHeal
+	}
+	return localDisksInZoneHeal
+
+}
+
+func getDrivesToHealCount(localDisksInZoneHeal []Endpoints) int {
+	var drivesToHeal int
+	for _, eps := range localDisksInZoneHeal {
+		for range eps {
+			drivesToHeal++
+		}
+	}
+	return drivesToHeal
+}
+
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	// Run the background healer
+	globalBackgroundHealRoutine = newHealRoutine()
+	go globalBackgroundHealRoutine.run(ctx, objAPI)
+
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence())
+}
+
+// monitorLocalDisksAndHeal - ensures that detected new disks are healed
+//  1. Only the concerned erasure set will be listed and healed
+//  2. Only the node hosting the disk is responsible to perform the heal
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, drivesToHeal int, localDisksInZoneHeal []Endpoints, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-time.After(defaultMonitorNewDiskInterval):
-			// Attempt a heal as the server starts-up first.
-			localDisksInZoneHeal := make([]Endpoints, len(z.zones))
-			var healNewDisks bool
-			for i, ep := range globalEndpoints {
-				localDisksToHeal := Endpoints{}
-				for _, endpoint := range ep.Endpoints {
-					if !endpoint.IsLocal {
-						continue
-					}
-					// Try to connect to the current endpoint
-					// and reformat if the current disk is not formatted
-					_, _, err := connectEndpoint(endpoint)
-					if err == errUnformattedDisk {
-						localDisksToHeal = append(localDisksToHeal, endpoint)
-					}
-				}
-				if len(localDisksToHeal) == 0 {
+			// heal only if new disks found.
+			if drivesToHeal == 0 {
+				localDisksInZoneHeal = getLocalDisksToHeal(z)
+				drivesToHeal = getDrivesToHealCount(localDisksInZoneHeal)
+				if drivesToHeal == 0 {
+					// No drives to heal.
+					globalBackgroundHealState.updateHealLocalDisks(nil)
 					continue
 				}
-				localDisksInZoneHeal[i] = localDisksToHeal
-				healNewDisks = true
+				globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
+
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+					drivesToHeal))
+
+				// Reformat disks
+				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
+
+				// Ensure that reformatting disks is finished
+				bgSeq.sourceCh <- healSource{bucket: nopHeal}
 			}
 
-			// Reformat disks only if needed.
-			if !healNewDisks {
-				continue
-			}
-
-			logger.Info("New unformatted drives detected attempting to heal...")
-			for i, disks := range localDisksInZoneHeal {
-				for _, disk := range disks {
-					logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
-				}
-			}
-
-			// Reformat disks
-			bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
-
-			// Ensure that reformatting disks is finished
-			bgSeq.sourceCh <- healSource{bucket: nopHeal}
-
 			var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
 			// Compute the list of erasure set to heal
 			for i, localDisksToHeal := range localDisksInZoneHeal {
@@ -108,6 +159,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 						printEndpointError(endpoint, err, true)
 						continue
 					}
+
 					// Calculate the set index where the current endpoint belongs
 					setIndex, _, err := findDiskIndex(z.zones[i].format, format)
 					if err != nil {
@@ -120,6 +172,13 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 				erasureSetInZoneToHeal[i] = erasureSetToHeal
 			}
 
+			logger.Info("New unformatted drives detected attempting to heal the content...")
+			for i, disks := range localDisksInZoneHeal {
+				for _, disk := range disks {
+					logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+				}
+			}
+
 			// Heal all erasure sets that need
 			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
 				for _, setIndex := range erasureSetToHeal {
@@ -127,6 +186,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 					if err != nil {
 						logger.LogIf(ctx, err)
 					}
+
+					// Only upon success reduce the counter
+					if err == nil {
+						drivesToHeal--
+					}
 				}
 			}
 		}

cmd/bootstrap-peer-server.go

@@ -228,7 +228,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		}
 	}
 
-	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
+	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
 	restClient.HealthCheckFn = func() bool {
 		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)

cmd/bucket-handlers.go

@@ -1271,7 +1271,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
-	sameTarget, err := globalBucketReplicationSys.validateDestination(ctx, bucket, replicationConfig)
+	sameTarget, err := validateReplicationDestination(ctx, bucket, replicationConfig)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-listobjects-handlers.go

@@ -24,12 +24,36 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
+	inParallel := func(objects []ObjectInfo) {
+		g := errgroup.WithNErrs(len(objects))
+		for index := range objects {
+			index := index
+			g.Go(func() error {
+				objects[index].ETag = objects[index].GetActualETag(nil)
+				objects[index].Size, _ = objects[index].GetActualSize()
+				return nil
+			}, index)
+		}
+		g.Wait()
+	}
+	const maxConcurrent = 500
+	for {
+		if len(objects) < maxConcurrent {
+			inParallel(objects)
+			return
+		}
+		inParallel(objects[:maxConcurrent])
+		objects = objects[maxConcurrent:]
+	}
+}
+
 // Validate all the ListObjects query arguments, returns an APIErrorCode
 // if one of the args do not meet the required conditions.
 // Special conditions required by MinIO server are as below
@@ -89,6 +113,10 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if proxyRequestByBucket(ctx, w, r, bucket) {
+		return
+	}
+
 	listObjectVersions := objectAPI.ListObjectVersions
 
 	// Inititate a list object versions operation based on the input params.
@@ -100,16 +128,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	for i := range listObjectVersionsInfo.Objects {
-		if crypto.IsEncrypted(listObjectVersionsInfo.Objects[i].UserDefined) {
-			listObjectVersionsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectVersionsInfo.Objects[i], false)
-		}
-		listObjectVersionsInfo.Objects[i].Size, err = listObjectVersionsInfo.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects)
 
 	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 
@@ -178,16 +197,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-		}
-		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
@@ -264,16 +274,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-		}
-		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
@@ -396,16 +397,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	for i := range listObjectsInfo.Objects {
-		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
-			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-		}
-		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsInfo.Objects)
 
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 

cmd/bucket-metadata-sys.go

@@ -156,8 +156,8 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		meta.QuotaConfigJSON = configData
 	case bucketReplicationConfig:
 		meta.ReplicationConfigXML = configData
-	case bucketReplicationTargetsFile:
-		meta.ReplicationTargetsConfigJSON = configData
+	case bucketTargetsFile:
+		meta.BucketTargetsConfigJSON = configData
 	default:
 		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
@@ -340,17 +340,17 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 	return meta.replicationConfig, nil
 }
 
-// GetReplicationTargetConfig returns configured bucket replication target for this bucket
+// GetBucketTargetsConfig returns configured bucket targets for this bucket
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetReplicationTargetConfig(bucket string) (*madmin.BucketReplicationTarget, error) {
+func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
 	meta, err := sys.GetConfig(bucket)
 	if err != nil {
 		return nil, err
 	}
-	if meta.replicationTargetConfig == nil {
-		return nil, BucketReplicationTargetNotFound{Bucket: bucket}
+	if meta.bucketTargetConfig == nil {
+		return nil, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
-	return meta.replicationTargetConfig, nil
+	return meta.bucketTargetConfig, nil
 }
 
 // GetConfig returns a specific configuration from the bucket metadata.

cmd/bucket-metadata.go

@@ -61,31 +61,31 @@ var (
 // bucketMetadataFormat refers to the format.
 // bucketMetadataVersion can be used to track a rolling upgrade of a field.
 type BucketMetadata struct {
-	Name                         string
-	Created                      time.Time
-	LockEnabled                  bool // legacy not used anymore.
-	PolicyConfigJSON             []byte
-	NotificationConfigXML        []byte
-	LifecycleConfigXML           []byte
-	ObjectLockConfigXML          []byte
-	VersioningConfigXML          []byte
-	EncryptionConfigXML          []byte
-	TaggingConfigXML             []byte
-	QuotaConfigJSON              []byte
-	ReplicationConfigXML         []byte
-	ReplicationTargetsConfigJSON []byte
+	Name                    string
+	Created                 time.Time
+	LockEnabled             bool // legacy not used anymore.
+	PolicyConfigJSON        []byte
+	NotificationConfigXML   []byte
+	LifecycleConfigXML      []byte
+	ObjectLockConfigXML     []byte
+	VersioningConfigXML     []byte
+	EncryptionConfigXML     []byte
+	TaggingConfigXML        []byte
+	QuotaConfigJSON         []byte
+	ReplicationConfigXML    []byte
+	BucketTargetsConfigJSON []byte
 
 	// Unexported fields. Must be updated atomically.
-	policyConfig            *policy.Policy
-	notificationConfig      *event.Config
-	lifecycleConfig         *lifecycle.Lifecycle
-	objectLockConfig        *objectlock.Config
-	versioningConfig        *versioning.Versioning
-	sseConfig               *bucketsse.BucketSSEConfig
-	taggingConfig           *tags.Tags
-	quotaConfig             *madmin.BucketQuota
-	replicationConfig       *replication.Config
-	replicationTargetConfig *madmin.BucketReplicationTarget
+	policyConfig       *policy.Policy
+	notificationConfig *event.Config
+	lifecycleConfig    *lifecycle.Lifecycle
+	objectLockConfig   *objectlock.Config
+	versioningConfig   *versioning.Versioning
+	sseConfig          *bucketsse.BucketSSEConfig
+	taggingConfig      *tags.Tags
+	quotaConfig        *madmin.BucketQuota
+	replicationConfig  *replication.Config
+	bucketTargetConfig *madmin.BucketTargets
 }
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
@@ -100,7 +100,7 @@ func newBucketMetadata(name string) BucketMetadata {
 		versioningConfig: &versioning.Versioning{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
-		replicationTargetConfig: &madmin.BucketReplicationTarget{},
+		bucketTargetConfig: &madmin.BucketTargets{},
 	}
 }
 
@@ -227,12 +227,12 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 		b.replicationConfig = nil
 	}
 
-	if len(b.ReplicationTargetsConfigJSON) != 0 {
-		if err = json.Unmarshal(b.ReplicationTargetsConfigJSON, b.replicationTargetConfig); err != nil {
+	if len(b.BucketTargetsConfigJSON) != 0 {
+		if err = json.Unmarshal(b.BucketTargetsConfigJSON, b.bucketTargetConfig); err != nil {
 			return err
 		}
 	} else {
-		b.replicationTargetConfig = &madmin.BucketReplicationTarget{}
+		b.bucketTargetConfig = &madmin.BucketTargets{}
 	}
 	return nil
 }
@@ -247,7 +247,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		bucketSSEConfig,
 		bucketTaggingConfig,
 		bucketReplicationConfig,
-		bucketReplicationTargetsFile,
+		bucketTargetsFile,
 		objectLockConfig,
 	}
 
@@ -265,6 +265,12 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 
 		configData, err := readConfig(ctx, objectAPI, configFile)
 		if err != nil {
+			switch err.(type) {
+			case ObjectExistsAsDirectory:
+				// in FS mode it possible that we have actual
+				// files in this folder with `.minio.sys/buckets/bucket/configFile`
+				continue
+			}
 			if errors.Is(err, errConfigNotFound) {
 				// legacy file config not found, proceed to look for new metadata.
 				continue
@@ -306,8 +312,8 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.QuotaConfigJSON = configData
 		case bucketReplicationConfig:
 			b.ReplicationConfigXML = configData
-		case bucketReplicationTargetsFile:
-			b.ReplicationTargetsConfigJSON = configData
+		case bucketTargetsFile:
+			b.BucketTargetsConfigJSON = configData
 		}
 	}
 

cmd/bucket-metadata_gen.go

@@ -96,10 +96,10 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ReplicationConfigXML")
 				return
 			}
-		case "ReplicationTargetsConfigJSON":
-			z.ReplicationTargetsConfigJSON, err = dc.ReadBytes(z.ReplicationTargetsConfigJSON)
+		case "BucketTargetsConfigJSON":
+			z.BucketTargetsConfigJSON, err = dc.ReadBytes(z.BucketTargetsConfigJSON)
 			if err != nil {
-				err = msgp.WrapError(err, "ReplicationTargetsConfigJSON")
+				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
 		default:
@@ -236,14 +236,14 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ReplicationConfigXML")
 		return
 	}
-	// write "ReplicationTargetsConfigJSON"
-	err = en.Append(0xbc, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	// write "BucketTargetsConfigJSON"
+	err = en.Append(0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
 	if err != nil {
 		return
 	}
-	err = en.WriteBytes(z.ReplicationTargetsConfigJSON)
+	err = en.WriteBytes(z.BucketTargetsConfigJSON)
 	if err != nil {
-		err = msgp.WrapError(err, "ReplicationTargetsConfigJSON")
+		err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 		return
 	}
 	return
@@ -289,9 +289,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ReplicationConfigXML"
 	o = append(o, 0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
 	o = msgp.AppendBytes(o, z.ReplicationConfigXML)
-	// string "ReplicationTargetsConfigJSON"
-	o = append(o, 0xbc, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
-	o = msgp.AppendBytes(o, z.ReplicationTargetsConfigJSON)
+	// string "BucketTargetsConfigJSON"
+	o = append(o, 0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.BucketTargetsConfigJSON)
 	return
 }
 
@@ -385,10 +385,10 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ReplicationConfigXML")
 				return
 			}
-		case "ReplicationTargetsConfigJSON":
-			z.ReplicationTargetsConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.ReplicationTargetsConfigJSON)
+		case "BucketTargetsConfigJSON":
+			z.BucketTargetsConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigJSON)
 			if err != nil {
-				err = msgp.WrapError(err, "ReplicationTargetsConfigJSON")
+				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
 				return
 			}
 		default:
@@ -405,6 +405,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 29 + msgp.BytesPrefixSize + len(z.ReplicationTargetsConfigJSON)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON)
 	return
 }

cmd/bucket-replication.go

@@ -18,35 +18,23 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strings"
-	"sync"
 	"time"
 
 	miniogo "github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/madmin"
 )
 
-// BucketReplicationSys represents replication subsystem
-type BucketReplicationSys struct {
-	sync.RWMutex
-	targetsMap    map[string]*miniogo.Core
-	targetsARNMap map[string]string
-}
-
-// GetConfig - gets replication config associated to a given bucket name.
-func (sys *BucketReplicationSys) GetConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
+// gets replication config associated to a given bucket name.
+func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
@@ -59,153 +47,52 @@ func (sys *BucketReplicationSys) GetConfig(ctx context.Context, bucketName strin
 	return globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
 }
 
-// SetTarget - sets a new minio-go client replication target for this bucket.
-func (sys *BucketReplicationSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketReplicationTarget) error {
-	if globalIsGateway {
-		return nil
-	}
-	// delete replication targets that were removed
-	if tgt.Empty() {
-		sys.Lock()
-		if currTgt, ok := sys.targetsMap[bucket]; ok {
-			delete(sys.targetsARNMap, currTgt.EndpointURL().String())
-		}
-		delete(sys.targetsMap, bucket)
-		sys.Unlock()
-		return nil
-	}
-	clnt, err := getReplicationTargetClient(tgt)
-	if err != nil {
-		return BucketReplicationTargetNotFound{Bucket: tgt.TargetBucket}
-	}
-	ok, err := clnt.BucketExists(ctx, tgt.TargetBucket)
-	if err != nil {
-		return err
-	}
-	if !ok {
-		return BucketReplicationDestinationNotFound{Bucket: tgt.TargetBucket}
-	}
-	vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
-	if err != nil || vcfg.Status != string(versioning.Enabled) {
-		return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
-	}
-	sys.Lock()
-	sys.targetsMap[bucket] = clnt
-	sys.targetsARNMap[tgt.URL()] = tgt.Arn
-	sys.Unlock()
-	return nil
-}
-
-// GetTargetClient returns minio-go client for target instance
-func (sys *BucketReplicationSys) GetTargetClient(ctx context.Context, bucket string) *miniogo.Core {
-	var clnt *miniogo.Core
-	sys.RLock()
-	if c, ok := sys.targetsMap[bucket]; ok {
-		clnt = c
-	}
-	sys.RUnlock()
-	return clnt
-}
-
-// validateDestination returns error if replication destination bucket missing or not configured
+// validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
-func (sys *BucketReplicationSys) validateDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
-	clnt := sys.GetTargetClient(ctx, bucket)
+func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
+	clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.RoleArn)
 	if clnt == nil {
-		return false, BucketReplicationTargetNotFound{Bucket: bucket}
+		return false, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	if found, _ := clnt.BucketExists(ctx, rCfg.GetDestination().Bucket); !found {
 		return false, BucketReplicationDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
 	}
-	// validate replication ARN against target endpoint
-	for k, v := range sys.targetsARNMap {
-		if v == rCfg.ReplicationArn {
-			if k == clnt.EndpointURL().String() {
-				sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
-				return sameTarget, nil
+	if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil {
+		if ret.LockEnabled {
+			lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, rCfg.GetDestination().Bucket)
+			if err != nil || lock != "Enabled" {
+				return false, BucketReplicationDestinationMissingLock{Bucket: rCfg.GetDestination().Bucket}
 			}
 		}
 	}
-	return false, BucketReplicationTargetNotFound{Bucket: bucket}
-}
-
-// NewBucketReplicationSys - creates new replication system.
-func NewBucketReplicationSys() *BucketReplicationSys {
-	return &BucketReplicationSys{
-		targetsMap:    make(map[string]*miniogo.Core),
-		targetsARNMap: make(map[string]string),
-	}
-}
-
-// Init initializes the bucket replication subsystem for buckets with replication config
-func (sys *BucketReplicationSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// In gateway mode, replication is not supported.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load bucket replication targets once during boot.
-	sys.load(ctx, buckets, objAPI)
-	return nil
-}
-
-// create minio-go clients for buckets having replication targets
-func (sys *BucketReplicationSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
-	for _, bucket := range buckets {
-		tgt, err := globalBucketMetadataSys.GetReplicationTargetConfig(bucket.Name)
-		if err != nil {
-			continue
+	// validate replication ARN against target endpoint
+	c, ok := globalBucketTargetSys.arnRemotesMap[rCfg.RoleArn]
+	if ok {
+		if c.EndpointURL().String() == clnt.EndpointURL().String() {
+			sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+			return sameTarget, nil
 		}
-		if tgt == nil || tgt.Empty() {
-			continue
-		}
-		tgtClient, err := getReplicationTargetClient(tgt)
-		if err != nil {
-			continue
-		}
-		sys.Lock()
-		sys.targetsMap[bucket.Name] = tgtClient
-		sys.targetsARNMap[tgt.URL()] = tgt.Arn
-		sys.Unlock()
 	}
+	return false, BucketRemoteTargetNotFound{Bucket: bucket}
 }
 
-// GetARN returns the ARN associated with replication target URL
-func (sys *BucketReplicationSys) getARN(endpoint string) string {
-	return sys.targetsARNMap[endpoint]
-}
-
-// getReplicationTargetInstanceTransport contains a singleton roundtripper.
-var getReplicationTargetInstanceTransport http.RoundTripper
-var getReplicationTargetInstanceTransportOnce sync.Once
-
-// Returns a minio-go Client configured to access remote host described in replication target config.
-var getReplicationTargetClient = func(tcfg *madmin.BucketReplicationTarget) (*miniogo.Core, error) {
-	config := tcfg.Credentials
-	// if Signature version '4' use NewV4 directly.
-	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
-	// if Signature version '2' use NewV2 directly.
-	if strings.ToUpper(tcfg.API) == "S3V2" {
-		creds = credentials.NewStaticV2(config.AccessKey, config.SecretKey, "")
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) bool {
+	if permErr != ErrNone {
+		return false
 	}
-
-	getReplicationTargetInstanceTransportOnce.Do(func() {
-		getReplicationTargetInstanceTransport = NewGatewayHTTPTransport()
-	})
-	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{
-		Creds:     creds,
-		Secure:    tcfg.IsSSL,
-		Transport: getReplicationTargetInstanceTransport,
-	})
-	return core, err
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
 }
 
 // mustReplicate returns true if object meets replication criteria.
-func (sys *BucketReplicationSys) mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+		return false
+	}
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+}
+
+// mustReplicater returns true if object meets replication criteria.
+func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
 	if globalIsGateway {
 		return false
 	}
@@ -215,10 +102,7 @@ func (sys *BucketReplicationSys) mustReplicate(ctx context.Context, r *http.Requ
 	if replication.StatusType(replStatus) == replication.Replica {
 		return false
 	}
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
-		return false
-	}
-	cfg, err := globalBucketReplicationSys.GetConfig(ctx, bucket)
+	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
 		return false
 	}
@@ -239,22 +123,29 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 		if k == xhttp.AmzBucketReplicationStatus {
 			continue
 		}
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			continue
+		}
 		meta[k] = v
 	}
-
 	tag, err := tags.ParseObjectTags(objInfo.UserTags)
 	if err != nil {
 		return
 	}
+	sc := dest.StorageClass
+	if sc == "" {
+		sc = objInfo.StorageClass
+	}
 	putOpts = miniogo.PutObjectOptions{
 		UserMetadata:         meta,
 		UserTags:             tag.ToMap(),
 		ContentType:          objInfo.ContentType,
 		ContentEncoding:      objInfo.ContentEncoding,
-		StorageClass:         dest.StorageClass,
+		StorageClass:         sc,
 		ReplicationVersionID: objInfo.VersionID,
 		ReplicationStatus:    miniogo.ReplicationStatusReplica,
 		ReplicationMTime:     objInfo.ModTime,
+		ReplicationETag:      objInfo.ETag,
 	}
 	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
 		rmode := miniogo.RetentionMode(mode)
@@ -279,12 +170,12 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
 func replicateObject(ctx context.Context, bucket, object, versionID string, objectAPI ObjectLayer, eventArg *eventArgs, healPending bool) {
-	cfg, err := globalBucketReplicationSys.GetConfig(ctx, bucket)
+	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
-	tgt := globalBucketReplicationSys.GetTargetClient(ctx, bucket)
+	tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
 		return
 	}
@@ -345,11 +236,25 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 	}
 }
 
-// getReplicationARN gets existing ARN for an endpoint or generates a new one.
-func (sys *BucketReplicationSys) getReplicationARN(endpoint string) string {
-	arn, ok := sys.targetsARNMap[endpoint]
-	if ok {
-		return arn
+// filterReplicationStatusMetadata filters replication status metadata for COPY
+func filterReplicationStatusMetadata(metadata map[string]string) map[string]string {
+	// Copy on write
+	dst := metadata
+	var copied bool
+	delKey := func(key string) {
+		if _, ok := metadata[key]; !ok {
+			return
+		}
+		if !copied {
+			dst = make(map[string]string, len(metadata))
+			for k, v := range metadata {
+				dst[k] = v
+			}
+			copied = true
+		}
+		delete(dst, key)
 	}
-	return fmt.Sprintf("arn:minio:s3::%s:*", mustGetUUID())
+
+	delKey(xhttp.AmzBucketReplicationStatus)
+	return dst
 }

cmd/bucket-targets.go

@@ -0,0 +1,312 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"net/http"
+	"sync"
+
+	miniogo "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/pkg/bucket/versioning"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+// BucketTargetSys represents bucket targets subsystem
+type BucketTargetSys struct {
+	sync.RWMutex
+	arnRemotesMap map[string]*miniogo.Core
+	targetsMap    map[string][]madmin.BucketTarget
+	clientsCache  map[string]*miniogo.Core
+}
+
+// ListTargets lists bucket targets across tenant or for individual bucket, and returns
+// results filtered by arnType
+func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) {
+	if bucket != "" {
+		if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil {
+			for _, t := range ts.Targets {
+				if string(t.Type) == arnType || arnType == "" {
+					targets = append(targets, t.Clone())
+				}
+			}
+		}
+		return targets
+	}
+	sys.RLock()
+	defer sys.RUnlock()
+	for _, tgts := range sys.targetsMap {
+		for _, t := range tgts {
+			if string(t.Type) == arnType || arnType == "" {
+				targets = append(targets, t.Clone())
+			}
+		}
+	}
+	return
+}
+
+// ListBucketTargets - gets list of bucket targets for this bucket.
+func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	tgts, ok := sys.targetsMap[bucket]
+	if ok {
+		return &madmin.BucketTargets{Targets: tgts}, nil
+	}
+	return nil, BucketRemoteTargetNotFound{Bucket: bucket}
+}
+
+// SetTarget - sets a new minio-go client target for this bucket.
+func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget) error {
+	if globalIsGateway {
+		return nil
+	}
+	if !tgt.Type.IsValid() {
+		return BucketRemoteArnTypeInvalid{Bucket: bucket}
+	}
+	clnt, err := sys.getRemoteTargetClient(tgt)
+	if err != nil {
+		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+	}
+	if tgt.Type == madmin.ReplicationService {
+		if !globalBucketVersioningSys.Enabled(bucket) {
+			return BucketReplicationSourceNotVersioned{Bucket: bucket}
+		}
+		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
+		if err != nil {
+			if isErrBucketNotFound(err) {
+				return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+			}
+			if vcfg.Status != string(versioning.Enabled) {
+				return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			}
+			return err
+		}
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	tgts := sys.targetsMap[bucket]
+	newtgts := make([]madmin.BucketTarget, len(tgts))
+	found := false
+	for idx, t := range tgts {
+		if t.Type == tgt.Type {
+			if t.Arn == tgt.Arn {
+				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
+			}
+			newtgts[idx] = *tgt
+			found = true
+			continue
+		}
+		newtgts[idx] = t
+	}
+	if !found {
+		newtgts = append(newtgts, *tgt)
+	}
+
+	sys.targetsMap[bucket] = newtgts
+	sys.arnRemotesMap[tgt.Arn] = clnt
+	if _, ok := sys.clientsCache[clnt.EndpointURL().String()]; !ok {
+		sys.clientsCache[clnt.EndpointURL().String()] = clnt
+	}
+	return nil
+}
+
+// RemoveTarget - removes a remote bucket target for this source bucket.
+func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
+	if globalIsGateway {
+		return nil
+	}
+	if arnStr == "" {
+		return BucketRemoteArnInvalid{Bucket: bucket}
+	}
+	arn, err := madmin.ParseARN(arnStr)
+	if err != nil {
+		return BucketRemoteArnInvalid{Bucket: bucket}
+	}
+	if arn.Type == madmin.ReplicationService {
+		// reject removal of remote target if replication configuration is present
+		rcfg, err := getReplicationConfig(ctx, bucket)
+		if err == nil && rcfg.RoleArn == arnStr {
+			if _, ok := sys.arnRemotesMap[arnStr]; ok {
+				return BucketRemoteRemoveDisallowed{Bucket: bucket}
+			}
+		}
+	}
+	// delete ARN type from list of matching targets
+	sys.Lock()
+	defer sys.Unlock()
+	targets := make([]madmin.BucketTarget, 0)
+	found := false
+	tgts := sys.targetsMap[bucket]
+	for _, tgt := range tgts {
+		if tgt.Arn != arnStr {
+			targets = append(targets, tgt)
+			continue
+		}
+		found = true
+	}
+	if !found {
+		return BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	sys.targetsMap[bucket] = targets
+	delete(sys.arnRemotesMap, arnStr)
+	return nil
+}
+
+// GetReplicationTargetClient returns minio-go client for replication target instance
+func (sys *BucketTargetSys) GetReplicationTargetClient(ctx context.Context, arn string) *miniogo.Core {
+	sys.RLock()
+	defer sys.RUnlock()
+	return sys.arnRemotesMap[arn]
+}
+
+// NewBucketTargetSys - creates new replication system.
+func NewBucketTargetSys() *BucketTargetSys {
+	return &BucketTargetSys{
+		arnRemotesMap: make(map[string]*miniogo.Core),
+		targetsMap:    make(map[string][]madmin.BucketTarget),
+		clientsCache:  make(map[string]*miniogo.Core),
+	}
+}
+
+// Init initializes the bucket targets subsystem for buckets which have targets configured.
+func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// In gateway mode, bucket targets is not supported.
+	if globalIsGateway {
+		return nil
+	}
+
+	// Load bucket targets once during boot.
+	sys.load(ctx, buckets, objAPI)
+	return nil
+}
+
+// UpdateTarget updates target to reflect metadata updates
+func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTargets) {
+	if sys == nil {
+		return
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	if cfg == nil || cfg.Empty() {
+		// remove target and arn association
+		if tgts, ok := sys.targetsMap[bucket]; ok {
+			for _, t := range tgts {
+				delete(sys.arnRemotesMap, t.Arn)
+			}
+		}
+		delete(sys.targetsMap, bucket)
+		return
+	}
+
+	if len(cfg.Targets) > 0 {
+		sys.targetsMap[bucket] = cfg.Targets
+	}
+	for _, tgt := range cfg.Targets {
+		tgtClient, err := sys.getRemoteTargetClient(&tgt)
+		if err != nil {
+			continue
+		}
+		sys.arnRemotesMap[tgt.Arn] = tgtClient
+		if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
+			sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
+		}
+	}
+	sys.targetsMap[bucket] = cfg.Targets
+}
+
+// create minio-go clients for buckets having remote targets
+func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+	for _, bucket := range buckets {
+		cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket.Name)
+		if err != nil {
+			continue
+		}
+		if cfg == nil || cfg.Empty() {
+			continue
+		}
+		if len(cfg.Targets) > 0 {
+			sys.targetsMap[bucket.Name] = cfg.Targets
+		}
+		for _, tgt := range cfg.Targets {
+			tgtClient, err := sys.getRemoteTargetClient(&tgt)
+			if err != nil {
+				continue
+			}
+			sys.arnRemotesMap[tgt.Arn] = tgtClient
+			if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
+				sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
+			}
+		}
+		sys.targetsMap[bucket.Name] = cfg.Targets
+	}
+}
+
+// getRemoteTargetInstanceTransport contains a singleton roundtripper.
+var getRemoteTargetInstanceTransport http.RoundTripper
+var getRemoteTargetInstanceTransportOnce sync.Once
+
+// Returns a minio-go Client configured to access remote host described in replication target config.
+func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
+	if clnt, ok := sys.clientsCache[tcfg.Endpoint]; ok {
+		return clnt, nil
+	}
+	config := tcfg.Credentials
+	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
+
+	getRemoteTargetInstanceTransportOnce.Do(func() {
+		getRemoteTargetInstanceTransport = NewGatewayHTTPTransport()
+	})
+	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{
+		Creds:     creds,
+		Secure:    tcfg.Secure,
+		Transport: getRemoteTargetInstanceTransport,
+	})
+	return core, err
+}
+
+// getRemoteARN gets existing ARN for an endpoint or generates a new one.
+func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) string {
+	if target == nil {
+		return ""
+	}
+	tgts := sys.targetsMap[bucket]
+	for _, tgt := range tgts {
+		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL() == tgt.URL() {
+			return tgt.Arn
+		}
+	}
+	if !madmin.ServiceType(target.Type).IsValid() {
+		return ""
+	}
+	arn := madmin.ARN{
+		Type:   target.Type,
+		ID:     mustGetUUID(),
+		Region: target.Region,
+		Bucket: target.TargetBucket,
+	}
+	return arn.String()
+}

cmd/config-common.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 )
 
@@ -36,8 +35,6 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 			return nil, errConfigNotFound
 		}
 
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
 		return nil, err
 	}
 
@@ -74,8 +71,6 @@ func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) err
 			return errConfigNotFound
 		}
 
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
 		return err
 	}
 	return nil

cmd/config-current.go

@@ -92,7 +92,7 @@ func initHelp() {
 		},
 		config.HelpKV{
 			Key:         config.PolicyOPASubSys,
-			Description: "enable external OPA for policy enforcement",
+			Description: "[DEPRECATED] enable external OPA for policy enforcement",
 		},
 		config.HelpKV{
 			Key:         config.KmsVaultSubSys,
@@ -213,7 +213,7 @@ var (
 	globalServerConfigMu sync.RWMutex
 )
 
-func validateConfig(s config.Config) error {
+func validateConfig(s config.Config, setDriveCount int) error {
 	// Disable merging env values with config for validation.
 	env.SetEnvOff()
 
@@ -233,8 +233,7 @@ func validateConfig(s config.Config) error {
 	}
 
 	if globalIsErasure {
-		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
-			globalErasureSetDriveCount); err != nil {
+		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
 			return err
 		}
 	}
@@ -311,7 +310,7 @@ func validateConfig(s config.Config) error {
 		globalNotificationSys.ConfiguredTargetIDs())
 }
 
-func lookupConfigs(s config.Config) {
+func lookupConfigs(s config.Config, setDriveCount int) {
 	ctx := GlobalContext
 
 	var err error
@@ -382,8 +381,7 @@ func lookupConfigs(s config.Config) {
 	globalAPIConfig.init(apiConfig)
 
 	if globalIsErasure {
-		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
-			globalErasureSetDriveCount)
+		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
 		}
@@ -457,28 +455,32 @@ func lookupConfigs(s config.Config) {
 	for _, l := range loggerCfg.HTTP {
 		if l.Enabled {
 			// Enable http logging
-			logger.AddTarget(
+			if err = logger.AddTarget(
 				http.New(http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
 					http.WithTransport(NewGatewayHTTPTransport()),
 				),
-			)
+			); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
+			}
 		}
 	}
 
 	for _, l := range loggerCfg.Audit {
 		if l.Enabled {
 			// Enable http audit logging
-			logger.AddAuditTarget(
+			if err = logger.AddAuditTarget(
 				http.New(http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
 					http.WithTransport(NewGatewayHTTPTransport()),
 				),
-			)
+			); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
+			}
 		}
 	}
 
@@ -601,7 +603,7 @@ func loadConfig(objAPI ObjectLayer) error {
 	}
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg)
+	lookupConfigs(srvCfg, objAPI.SetDriveCount())
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/config/certs.go

@@ -24,8 +24,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"io/ioutil"
-	"os"
-	"path"
 
 	"github.com/minio/minio/pkg/env"
 )
@@ -69,38 +67,6 @@ func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 	return x509Certs, nil
 }
 
-// GetRootCAs - returns all the root CAs into certPool
-// at the input certsCADir
-func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
-	rootCAs, _ := x509.SystemCertPool()
-	if rootCAs == nil {
-		// In some systems (like Windows) system cert pool is
-		// not supported or no certificates are present on the
-		// system - so we create a new cert pool.
-		rootCAs = x509.NewCertPool()
-	}
-
-	fis, err := ioutil.ReadDir(certsCAsDir)
-	if err != nil {
-		if os.IsNotExist(err) || os.IsPermission(err) {
-			// Return success if CA's directory is missing or permission denied.
-			err = nil
-		}
-		return rootCAs, err
-	}
-
-	// Load all custom CA files.
-	for _, fi := range fis {
-		caCert, err := ioutil.ReadFile(path.Join(certsCAsDir, fi.Name()))
-		if err != nil {
-			// ignore files which are not readable.
-			continue
-		}
-		rootCAs.AppendCertsFromPEM(caCert)
-	}
-	return rootCAs, nil
-}
-
 // LoadX509KeyPair - load an X509 key pair (private key , certificate)
 // from the provided paths. The private key may be encrypted and is
 // decrypted using the ENV_VAR: MINIO_CERT_PASSWD.

cmd/config/certs_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"runtime"
 	"testing"
 )
@@ -194,60 +193,6 @@ M9ofSEt/bdRD
 	}
 }
 
-func TestGetRootCAs(t *testing.T) {
-	emptydir, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", emptydir)
-	}
-	defer os.RemoveAll(emptydir)
-
-	dir1, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir1)
-	}
-	defer os.RemoveAll(dir1)
-	if err = os.Mkdir(filepath.Join(dir1, "empty-dir"), 0755); err != nil {
-		t.Fatalf("Unable create empty dir. %v", err)
-	}
-
-	dir2, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir2)
-	}
-	defer os.RemoveAll(dir2)
-	if err = ioutil.WriteFile(filepath.Join(dir2, "empty-file"), []byte{}, 0644); err != nil {
-		t.Fatalf("Unable create test file. %v", err)
-	}
-
-	testCases := []struct {
-		certCAsDir  string
-		expectedErr error
-	}{
-		// ignores non-existent directories.
-		{"nonexistent-dir", nil},
-		// Ignores directories.
-		{dir1, nil},
-		// Ignore empty directory.
-		{emptydir, nil},
-		// Loads the cert properly.
-		{dir2, nil},
-	}
-
-	for _, testCase := range testCases {
-		_, err := GetRootCAs(testCase.certCAsDir)
-
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-	}
-}
-
 func TestLoadX509KeyPair(t *testing.T) {
 	for i, testCase := range loadX509KeyPairTests {
 		privateKey, err := createTempFile("private.key", testCase.privateKey)

cmd/config/constants.go

@@ -31,11 +31,12 @@ const (
 	EnvDomain       = "MINIO_DOMAIN"
 	EnvRegionName   = "MINIO_REGION_NAME"
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
-	EnvEndpoints    = "MINIO_ENDPOINTS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
+	EnvArgs         = "MINIO_ARGS"
 
 	EnvUpdate = "MINIO_UPDATE"
 
-	EnvWorm   = "MINIO_WORM"   // legacy
-	EnvRegion = "MINIO_REGION" // legacy
+	EnvEndpoints = "MINIO_ENDPOINTS" // legacy
+	EnvWorm      = "MINIO_WORM"      // legacy
+	EnvRegion    = "MINIO_REGION"    // legacy
 )

cmd/config/errors.go

@@ -187,12 +187,6 @@ Example 1:
 		"",
 	)
 
-	ErrCorruptedBackend = newErrFn(
-		"Unable to use the specified backend, pre-existing content detected",
-		"Please ensure your disk mount does not have any pre-existing content",
-		"",
-	)
-
 	ErrUnableToWriteInBackend = newErrFn(
 		"Unable to write to the backend",
 		"Please ensure MinIO binary has write permissions for the backend",

cmd/config/notify/legacy.go

@@ -331,7 +331,7 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg
 		},
 		config.KV{
 			Key:   target.PostgresUsername,
-			Value: cfg.User,
+			Value: cfg.Username,
 		},
 		config.KV{
 			Key:   target.PostgresPassword,

cmd/config/policy/opa/help.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019,2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,12 +23,12 @@ var (
 	Help = config.HelpKVS{
 		config.HelpKV{
 			Key:         URL,
-			Description: `OPA HTTP(s) endpoint e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"`,
+			Description: `[DEPRECATED] OPA HTTP(s) endpoint e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"`,
 			Type:        "url",
 		},
 		config.HelpKV{
 			Key:         AuthToken,
-			Description: "authorization token for OPA endpoint",
+			Description: "[DEPRECATED] authorization token for OPA endpoint",
 			Optional:    true,
 			Type:        "string",
 		},

cmd/config/storageclass/storage-class.go

@@ -226,7 +226,7 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 	cfg.RRS.Parity = defaultRRSParity
 
 	if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS); err != nil {
-		return cfg, err
+		return Config{}, err
 	}
 
 	ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
@@ -235,7 +235,7 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 	if ssc != "" {
 		cfg.Standard, err = parseStorageClass(ssc)
 		if err != nil {
-			return cfg, err
+			return Config{}, err
 		}
 	}
 	if cfg.Standard.Parity == 0 {
@@ -245,7 +245,7 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 	if rrsc != "" {
 		cfg.RRS, err = parseStorageClass(rrsc)
 		if err != nil {
-			return cfg, err
+			return Config{}, err
 		}
 	}
 	if cfg.RRS.Parity == 0 {
@@ -255,7 +255,7 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
 	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, drivesPerSet); err != nil {
-		return cfg, err
+		return Config{}, err
 	}
 
 	return cfg, nil

cmd/consolelogger.go

@@ -117,6 +117,11 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
 	sys.pubsub.Subscribe(subCh, doneCh, filter)
 }
 
+// Validate if HTTPConsoleLoggerSys is valid, always returns nil right now
+func (sys *HTTPConsoleLoggerSys) Validate() error {
+	return nil
+}
+
 // Send log message 'e' to console and publish to console
 // log pubsub system
 func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {

cmd/crypto/header.go

@@ -18,9 +18,11 @@ import (
 	"bytes"
 	"crypto/md5"
 	"encoding/base64"
-	"encoding/json"
 	"net/http"
 	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // SSEHeader is the general AWS SSE HTTP header key.
@@ -81,6 +83,8 @@ const (
 func RemoveSensitiveHeaders(h http.Header) {
 	h.Del(SSECKey)
 	h.Del(SSECopyKey)
+	h.Del(xhttp.AmzMetaUnencryptedContentLength)
+	h.Del(xhttp.AmzMetaUnencryptedContentMD5)
 }
 
 // IsRequested returns true if the HTTP headers indicates
@@ -144,6 +148,7 @@ func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
 	contextStr, ok := h[SSEKmsContext]
 	if ok {
 		var context map[string]interface{}
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
 		if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
 			return "", nil, err
 		}

cmd/crypto/header_test.go

@@ -457,6 +457,16 @@ var removeSensitiveHeadersTests = []struct {
 			"X-Amz-Meta-Test-1": []string{"Test-1"},
 		},
 	},
+	{ // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		Header: http.Header{
+			"X-Amz-Meta-X-Amz-Unencrypted-Content-Md5":    []string{"value"},
+			"X-Amz-Meta-X-Amz-Unencrypted-Content-Length": []string{"value"},
+			"X-Amz-Meta-Test-1":                           []string{"Test-1"},
+		},
+		ExpectedHeader: http.Header{
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+	},
 }
 
 func TestRemoveSensitiveHeaders(t *testing.T) {

cmd/crypto/kes.go

@@ -18,7 +18,6 @@ import (
 	"bytes"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -30,10 +29,13 @@ import (
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	xhttp "github.com/minio/minio/cmd/http"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
 // ErrKESKeyExists is the error returned a KES server
 // when a master key does exist.
 var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist")

cmd/crypto/metadata.go

@@ -19,6 +19,7 @@ import (
 	"encoding/base64"
 	"errors"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -38,6 +39,8 @@ func IsMultiPart(metadata map[string]string) bool {
 func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
 	delete(metadata, SSECKey)
 	delete(metadata, SSECopyKey)
+	delete(metadata, xhttp.AmzMetaUnencryptedContentLength)
+	delete(metadata, xhttp.AmzMetaUnencryptedContentMD5)
 }
 
 // RemoveSSEHeaders removes all crypto-specific SSE

cmd/data-crawler.go

@@ -510,8 +510,7 @@ func (i *crawlItem) transformMetaDir() {
 // actionMeta contains information used to apply actions.
 type actionMeta struct {
 	oi          ObjectInfo
-	trustOI     bool // Set true if oi can be trusted and has been read with quorum.
-	numVersions int  // The number of versions of this object
+	numVersions int // The number of versions of this object
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -539,7 +538,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			NumVersions:  meta.numVersions,
 		})
 	if i.debug {
-		logger.Info(color.Green("applyActions:")+" lifecycle: %q, Initial scan: %v", i.objectPath(), action)
+		logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
 	}
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
@@ -548,63 +547,71 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		return size
 	}
 
-	// These (expensive) operations should only run on items we are likely to delete.
-	// Load to ensure that we have the correct version and not an unsynced version.
-	if !meta.trustOI {
-		obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{
-			VersionID: versionID,
-		})
-		if err != nil {
-			switch err.(type) {
-			case MethodNotAllowed: // This happens usually for a delete marker
-				if !obj.DeleteMarker { // if this is not a delete marker log and return
-					// Do nothing - heal in the future.
-					logger.LogIf(ctx, err)
-					return size
-				}
-			case ObjectNotFound:
-				// object not found return 0
-				return 0
-			default:
-				// All other errors proceed.
+	obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{
+		VersionID: versionID,
+	})
+	if err != nil {
+		switch err.(type) {
+		case MethodNotAllowed: // This happens usually for a delete marker
+			if !obj.DeleteMarker { // if this is not a delete marker log and return
+				// Do nothing - heal in the future.
 				logger.LogIf(ctx, err)
 				return size
 			}
-		}
-		size = obj.Size
-
-		// Recalculate action.
-		action = i.lifeCycle.ComputeAction(
-			lifecycle.ObjectOpts{
-				Name:         i.objectPath(),
-				UserTags:     obj.UserTags,
-				ModTime:      obj.ModTime,
-				VersionID:    obj.VersionID,
-				DeleteMarker: obj.DeleteMarker,
-				IsLatest:     obj.IsLatest,
-				NumVersions:  meta.numVersions,
-			})
-		if i.debug {
-			logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
-		}
-		versionID = obj.VersionID
-		switch action {
-		case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+		case ObjectNotFound:
+			// object not found return 0
+			return 0
 		default:
-			// No action.
+			// All other errors proceed.
+			logger.LogIf(ctx, err)
 			return size
 		}
 	}
+	size = obj.Size
+
+	// Recalculate action.
+	action = i.lifeCycle.ComputeAction(
+		lifecycle.ObjectOpts{
+			Name:         i.objectPath(),
+			UserTags:     obj.UserTags,
+			ModTime:      obj.ModTime,
+			VersionID:    obj.VersionID,
+			DeleteMarker: obj.DeleteMarker,
+			IsLatest:     obj.IsLatest,
+			NumVersions:  meta.numVersions,
+		})
+	if i.debug {
+		logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
+	}
+	switch action {
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
+	default:
+		// No action.
+		return size
+	}
 
 	opts := ObjectOptions{}
 	switch action {
 	case lifecycle.DeleteVersionAction:
-		opts.VersionID = versionID
+		// Defensive code, should never happen
+		if obj.VersionID == "" {
+			return size
+		}
+		if rcfg, _ := globalBucketObjectLockSys.Get(i.bucket); rcfg.LockEnabled {
+			locked := enforceRetentionForDeletion(ctx, obj)
+			if locked {
+				if i.debug {
+					logger.Info(color.Green("applyActions:")+" lifecycle: %s is locked, not deleting", i.objectPath())
+				}
+				return size
+			}
+		}
+		opts.VersionID = obj.VersionID
 	case lifecycle.DeleteAction:
 		opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
 	}
 
-	obj, err := o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
+	obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
 	if err != nil {
 		// Assume it is still there.
 		logger.LogIf(ctx, err)

cmd/data-update-tracker_test.go

@@ -42,6 +42,10 @@ type testingLogger struct {
 	t  testLoggerI
 }
 
+func (t *testingLogger) Validate() error {
+	return nil
+}
+
 func (t *testingLogger) Send(entry interface{}, errKind string) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()

cmd/disk-cache.go

@@ -326,7 +326,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
 				// use a new context to avoid locker prematurely timing out operation when the GetObjectNInfo returns.
-				dcache.Put(context.Background(), bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{
+					UserDefined: getMetadata(bReader.ObjInfo),
+				}, false)
 				return
 			}
 		}()
@@ -337,7 +339,11 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	go func() {
-		putErr := dcache.Put(ctx, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)}, false)
+		putErr := dcache.Put(ctx, bucket, object,
+			io.LimitReader(pipeReader, bkReader.ObjInfo.Size),
+			bkReader.ObjInfo.Size, nil, ObjectOptions{
+				UserDefined: getMetadata(bkReader.ObjInfo),
+			}, false)
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
@@ -684,11 +690,13 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 	c.cacheStats.GetDiskStats = func() []CacheDiskStats {
 		cacheDiskStats := make([]CacheDiskStats, len(c.cache))
 		for i := range c.cache {
-			cacheDiskStats[i] = CacheDiskStats{
-				Dir: c.cache[i].stats.Dir,
+			dcache := c.cache[i]
+			cacheDiskStats[i] = CacheDiskStats{}
+			if dcache != nil {
+				cacheDiskStats[i].Dir = dcache.stats.Dir
+				atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&dcache.stats.UsageState))
+				atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&dcache.stats.UsagePercent))
 			}
-			atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&c.cache[i].stats.UsageState))
-			atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&c.cache[i].stats.UsagePercent))
 		}
 		return cacheDiskStats
 	}
@@ -712,7 +720,9 @@ func (c *cacheObjects) gc(ctx context.Context) {
 				continue
 			}
 			for _, dcache := range c.cache {
-				dcache.triggerGC <- struct{}{}
+				if dcache != nil {
+					dcache.triggerGC <- struct{}{}
+				}
 			}
 		}
 	}

cmd/encryption-v1.go

@@ -605,12 +605,14 @@ func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool)
 	if crypto.IsMultiPart(objInfo.UserDefined) {
 		return objInfo.ETag
 	}
+
 	if crypto.SSECopy.IsRequested(headers) {
 		key, err = crypto.SSECopy.ParseHTTP(headers)
 		if err != nil {
 			return objInfo.ETag
 		}
 	}
+
 	// As per AWS S3 Spec, ETag for SSE-C encrypted objects need not be MD5Sum of the data.
 	// Since server side copy with same source and dest just replaces the ETag, we save
 	// encrypted content MD5Sum as ETag for both SSE-C and SSE-S3, we standardize the ETag

cmd/endpoint-ellipses.go

@@ -329,28 +329,28 @@ var (
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
 func createServerEndpoints(serverAddr string, args ...string) (
-	endpointZones EndpointZones, setDriveCount int,
-	setupType SetupType, err error) {
+	endpointZones EndpointZones, setupType SetupType, err error) {
 
 	if len(args) == 0 {
-		return nil, -1, -1, errInvalidArgument
+		return nil, -1, errInvalidArgument
 	}
 
+	var setDriveCount int
 	if v := env.Get(EnvErasureSetDriveCount, ""); v != "" {
 		setDriveCount, err = strconv.Atoi(v)
 		if err != nil {
-			return nil, -1, -1, config.ErrInvalidErasureSetSize(err)
+			return nil, -1, config.ErrInvalidErasureSetSize(err)
 		}
 	}
 
 	if !ellipses.HasEllipses(args...) {
 		setArgs, err := GetAllSets(uint64(setDriveCount), args...)
 		if err != nil {
-			return nil, -1, -1, err
+			return nil, -1, err
 		}
 		endpointList, newSetupType, err := CreateEndpoints(serverAddr, false, setArgs...)
 		if err != nil {
-			return nil, -1, -1, err
+			return nil, -1, err
 		}
 		endpointZones = append(endpointZones, ZoneEndpoints{
 			SetCount:     len(setArgs),
@@ -358,7 +358,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			Endpoints:    endpointList,
 		})
 		setupType = newSetupType
-		return endpointZones, len(setArgs[0]), setupType, nil
+		return endpointZones, setupType, nil
 	}
 
 	var prevSetupType SetupType
@@ -366,25 +366,25 @@ func createServerEndpoints(serverAddr string, args ...string) (
 	for _, arg := range args {
 		setArgs, err := GetAllSets(uint64(setDriveCount), arg)
 		if err != nil {
-			return nil, -1, -1, err
+			return nil, -1, err
 		}
 		var endpointList Endpoints
 		endpointList, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
 		if err != nil {
-			return nil, -1, -1, err
+			return nil, -1, err
 		}
 		if setDriveCount != 0 && setDriveCount != len(setArgs[0]) {
-			return nil, -1, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
+			return nil, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
 		}
 		if prevSetupType != UnknownSetupType && prevSetupType != setupType {
-			return nil, -1, -1, fmt.Errorf("All zones should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
+			return nil, -1, fmt.Errorf("All zones should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
 		}
 		if err = endpointZones.Add(ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
 		}); err != nil {
-			return nil, -1, -1, err
+			return nil, -1, err
 		}
 		foundPrevLocal = endpointList.atleastOneEndpointLocal()
 		if setDriveCount == 0 {
@@ -393,5 +393,5 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		prevSetupType = setupType
 	}
 
-	return endpointZones, setDriveCount, setupType, nil
+	return endpointZones, setupType, nil
 }

cmd/endpoint-ellipses_test.go

@@ -56,7 +56,7 @@ func TestCreateServerEndpoints(t *testing.T) {
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			_, _, _, err := createServerEndpoints(testCase.serverAddr, testCase.args...)
+			_, _, err := createServerEndpoints(testCase.serverAddr, testCase.args...)
 			if err != nil && testCase.success {
 				t.Errorf("Expected success but failed instead %s", err)
 			}

cmd/erasure-healing.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"sync"
@@ -197,10 +198,10 @@ func listAllBuckets(storageDisks []StorageAPI, healBuckets map[string]VolInfo) (
 // Only heal on disks where we are sure that healing is needed. We can expand
 // this list as and when we figure out more errors can be added to this list safely.
 func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, quorumModTime time.Time) bool {
-	switch erErr {
-	case errFileNotFound, errFileVersionNotFound:
+	switch {
+	case errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound):
 		return true
-	case errCorruptedFormat:
+	case errors.Is(erErr, errCorruptedFormat):
 		return true
 	}
 	if erErr == nil {
@@ -686,9 +687,9 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	// or when er.meta is not readable in read quorum disks.
 	var notFoundErasureMeta, corruptedErasureMeta int
 	for _, readErr := range errs {
-		if readErr == errFileNotFound || readErr == errFileVersionNotFound {
+		if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
 			notFoundErasureMeta++
-		} else if readErr == errCorruptedFormat {
+		} else if errors.Is(readErr, errCorruptedFormat) {
 			corruptedErasureMeta++
 		}
 	}
@@ -699,7 +700,10 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 		// double counting when both parts and er.meta
 		// are not available.
 		if errs[i] != dataErrs[i] {
-			if dataErrs[i] == errFileNotFound || dataErrs[i] == errFileVersionNotFound {
+			if IsErr(dataErrs[i], []error{
+				errFileNotFound,
+				errFileVersionNotFound,
+			}...) {
 				notFoundParts++
 			}
 		}

cmd/erasure-metadata.go

@@ -88,9 +88,10 @@ func (fi FileInfo) IsValid() bool {
 		// for erasure coding information
 		return true
 	}
-	data := fi.Erasure.DataBlocks
-	parity := fi.Erasure.ParityBlocks
-	return ((data >= parity) && (data != 0) && (parity != 0))
+	dataBlocks := fi.Erasure.DataBlocks
+	parityBlocks := fi.Erasure.ParityBlocks
+	return ((dataBlocks >= parityBlocks) &&
+		(dataBlocks != 0) && (parityBlocks != 0))
 }
 
 // ToObjectInfo - Converts metadata to object info.
@@ -111,6 +112,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 		DeleteMarker:    fi.Deleted,
 		Size:            fi.Size,
 		ModTime:         fi.ModTime,
+		Legacy:          fi.XLV1,
 		ContentType:     fi.Metadata["content-type"],
 		ContentEncoding: fi.Metadata["content-encoding"],
 	}
@@ -323,7 +325,18 @@ func objectQuorumFromMeta(ctx context.Context, er erasureObjects, partsMetaData
 		return 0, 0, err
 	}
 
+	dataBlocks := latestFileInfo.Erasure.DataBlocks
+	parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass])
+	if parityBlocks == 0 {
+		parityBlocks = dataBlocks
+	}
+
+	writeQuorum := dataBlocks
+	if dataBlocks == parityBlocks {
+		writeQuorum = dataBlocks + 1
+	}
+
 	// Since all the valid erasure code meta updated at the same time are equivalent, pass dataBlocks
 	// from latestFileInfo to get the quorum
-	return latestFileInfo.Erasure.DataBlocks, latestFileInfo.Erasure.DataBlocks + 1, nil
+	return dataBlocks, writeQuorum, nil
 }

cmd/erasure-multipart.go

@@ -367,7 +367,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
-	n, err := erasure.Encode(ctx, data, writers, buffer, fi.Erasure.DataBlocks+1)
+	n, err := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
@@ -702,6 +702,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	// Save successfully calculated md5sum.
 	fi.Metadata["etag"] = s3MD5
+	if opts.UserDefined["etag"] != "" { // preserve ETag if set
+		fi.Metadata["etag"] = opts.UserDefined["etag"]
+	}
 
 	// Save the consolidated actual size.
 	fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)

cmd/erasure-object.go

@@ -65,7 +65,7 @@ func (er erasureObjects) putObjectDir(ctx context.Context, bucket, object string
 // if source object and destination object are same we only
 // update metadata.
 func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
-	// This call shouldn't be used for anything other than metadata updates.
+	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
 	if !srcInfo.metadataOnly {
 		return oi, NotImplemented{}
 	}
@@ -97,8 +97,23 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return fi.ToObjectInfo(srcBucket, srcObject), toObjectErr(errMethodNotAllowed, srcBucket, srcObject)
 	}
 
+	versionID := srcInfo.VersionID
+	if srcInfo.versionOnly {
+		versionID = dstOpts.VersionID
+		// preserve destination versionId if specified.
+		if versionID == "" {
+			versionID = mustGetUUID()
+		}
+		modTime = UTCNow()
+	}
+
+	fi.VersionID = versionID // set any new versionID we might have created
+	fi.ModTime = modTime     // set modTime for the new versionID
+
 	// Update `xl.meta` content on each disks.
 	for index := range metaArr {
+		metaArr[index].ModTime = modTime
+		metaArr[index].VersionID = versionID
 		metaArr[index].Metadata = srcInfo.UserDefined
 		metaArr[index].Metadata["etag"] = srcInfo.ETag
 	}
@@ -670,7 +685,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
-	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, fi.Erasure.DataBlocks+1)
+	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if erasureErr != nil {
 		return ObjectInfo{}, toObjectErr(erasureErr, minioMetaTmpBucket, tempErasureObj)
@@ -695,8 +710,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			Hash:       bitrotWriterSum(w),
 		})
 	}
-
-	opts.UserDefined["etag"] = r.MD5CurrentHexString()
+	if opts.UserDefined["etag"] == "" {
+		opts.UserDefined["etag"] = r.MD5CurrentHexString()
+	}
 
 	// Guess content-type from the extension if possible.
 	if opts.UserDefined["content-type"] == "" {

cmd/erasure-object_test.go

@@ -420,6 +420,8 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 
 	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "")
 
+	parts1SC := globalStorageClass
+
 	// Object for test case 2 - No StorageClass defined, MetaData in PutObject requesting RRS Class
 	object2 := "object2"
 	metadata2 := make(map[string]string)
@@ -430,6 +432,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "")
+	parts2SC := globalStorageClass
 
 	// Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class
 	object3 := "object3"
@@ -441,6 +444,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "")
+	parts3SC := globalStorageClass
 
 	// Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class
 	object4 := "object4"
@@ -458,6 +462,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "")
+	parts4SC := storageclass.Config{
+		Standard: storageclass.StorageClass{
+			Parity: 6,
+		},
+	}
 
 	// Object for test case 5 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting RRS Class
 	// Reset global storage class flags
@@ -476,6 +485,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "")
+	parts5SC := storageclass.Config{
+		RRS: storageclass.StorageClass{
+			Parity: 2,
+		},
+	}
 
 	// Object for test case 6 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting Standard Storage Class
 	object6 := "object6"
@@ -493,6 +507,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "")
+	parts6SC := storageclass.Config{
+		RRS: storageclass.StorageClass{
+			Parity: 2,
+		},
+	}
 
 	// Object for test case 7 - Standard StorageClass defined as Parity 5, MetaData in PutObject requesting RRS Class
 	// Reset global storage class flags
@@ -511,25 +530,32 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "")
+	parts7SC := storageclass.Config{
+		Standard: storageclass.StorageClass{
+			Parity: 5,
+		},
+	}
 
 	tests := []struct {
 		parts               []FileInfo
 		errs                []error
 		expectedReadQuorum  int
 		expectedWriteQuorum int
+		storageClassCfg     storageclass.Config
 		expectedError       error
 	}{
-		{parts1, errs1, 8, 9, nil},
-		{parts2, errs2, 14, 15, nil},
-		{parts3, errs3, 8, 9, nil},
-		{parts4, errs4, 10, 11, nil},
-		{parts5, errs5, 14, 15, nil},
-		{parts6, errs6, 8, 9, nil},
-		{parts7, errs7, 14, 15, nil},
+		{parts1, errs1, 8, 9, parts1SC, nil},
+		{parts2, errs2, 14, 14, parts2SC, nil},
+		{parts3, errs3, 8, 9, parts3SC, nil},
+		{parts4, errs4, 10, 10, parts4SC, nil},
+		{parts5, errs5, 14, 14, parts5SC, nil},
+		{parts6, errs6, 8, 9, parts6SC, nil},
+		{parts7, errs7, 14, 14, parts7SC, nil},
 	}
 	for _, tt := range tests {
 		tt := tt
 		t.(*testing.T).Run("", func(t *testing.T) {
+			globalStorageClass = tt.storageClassCfg
 			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, *xl, tt.parts, tt.errs)
 			if tt.expectedError != nil && err == nil {
 				t.Errorf("Expected %s, got %s", tt.expectedError, err)

cmd/erasure-sets.go

@@ -371,6 +371,11 @@ func (s *erasureSets) NewNSLock(ctx context.Context, bucket string, objects ...s
 	return s.getHashedSet("").NewNSLock(ctx, bucket, objects...)
 }
 
+// SetDriveCount returns the current drives per set.
+func (s *erasureSets) SetDriveCount() int {
+	return s.drivesPerSet
+}
+
 // StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets.
 // This only returns disk usage info for Zones to perform placement decision, this call
 // is not implemented in Object interface and is not meant to be used by other object
@@ -738,14 +743,24 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 	srcSet := s.getHashedSet(srcObject)
 	dstSet := s.getHashedSet(dstObject)
 
+	cpSrcDstSame := srcSet == dstSet
+
 	// Check if this request is only metadata update.
-	if srcSet == dstSet && srcInfo.metadataOnly {
+	if cpSrcDstSame && srcInfo.metadataOnly {
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		// CopyObject optimization where we don't create an entire copy
+		// of the content, instead we add a reference, we disallow legacy
+		// objects to be self referenced in this manner so make sure
+		// that we actually create a new dataDir for legacy objects.
+		if dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID && !srcInfo.Legacy {
+			srcInfo.versionOnly = true
+			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+		}
 	}
 
 	putOpts := ObjectOptions{
@@ -1189,21 +1204,11 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 		}
 	}(storageDisks)
 
-	formats, sErrs := loadFormatErasureAll(storageDisks, false)
+	formats, _ := loadFormatErasureAll(storageDisks, false)
 	if err = checkFormatErasureValues(formats, s.drivesPerSet); err != nil {
 		return err
 	}
 
-	for index, sErr := range sErrs {
-		if sErr != nil {
-			// Look for acceptable heal errors, for any other
-			// errors we should simply quit and return.
-			if _, ok := formatHealErrors[sErr]; !ok {
-				return fmt.Errorf("Disk %s: %w", s.endpoints[index], sErr)
-			}
-		}
-	}
-
 	refFormat, err := getFormatErasureInQuorum(formats)
 	if err != nil {
 		return err
@@ -1347,16 +1352,6 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		res.After.Drives[k] = madmin.HealDriveInfo(v)
 	}
 
-	for index, sErr := range sErrs {
-		if sErr != nil {
-			// Look for acceptable heal errors, for any other
-			// errors we should simply quit and return.
-			if _, ok := formatHealErrors[sErr]; !ok {
-				return res, fmt.Errorf("Disk %s: %w", s.endpoints[index], sErr)
-			}
-		}
-	}
-
 	if countErrs(sErrs, errUnformattedDisk) == 0 {
 		// No unformatted disks found disks are either offline
 		// or online, no healing is required.
@@ -1419,6 +1414,11 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			return madmin.HealResultItem{}, err
 		}
 
+		refFormat, err = getFormatErasureInQuorum(tmpNewFormats)
+		if err != nil {
+			return madmin.HealResultItem{}, err
+		}
+
 		// kill the monitoring loop such that we stop writing
 		// to indicate that we will re-initialize everything
 		// with new format.

cmd/erasure-zones.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -87,6 +88,10 @@ func (z *erasureZones) NewNSLock(ctx context.Context, bucket string, objects ...
 	return z.zones[0].NewNSLock(ctx, bucket, objects...)
 }
 
+func (z *erasureZones) SetDriveCount() int {
+	return z.zones[0].SetDriveCount()
+}
+
 type zonesAvailableSpace []zoneAvailableSpace
 
 type zoneAvailableSpace struct {
@@ -626,22 +631,24 @@ func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dst
 		defer lk.Unlock()
 	}
 
-	if z.SingleZone() {
-		return z.zones[0].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
-	}
-
 	zoneIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
 
-	if cpSrcDstSame && srcInfo.metadataOnly && srcOpts.VersionID == dstOpts.VersionID {
+	if cpSrcDstSame && srcInfo.metadataOnly {
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
 			return z.zones[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
 			return z.zones[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		if dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID && !srcInfo.Legacy {
+			// CopyObject optimization where we don't create an entire copy
+			// of the content, instead we add a reference.
+			srcInfo.versionOnly = true
+			return z.zones[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+		}
 	}
 
 	putOpts := ObjectOptions{
@@ -2026,6 +2033,7 @@ type HealthOptions struct {
 // was queried
 type HealthResult struct {
 	Healthy       bool
+	HealingDrives int
 	ZoneID, SetID int
 	WriteQuorum   int
 }
@@ -2056,6 +2064,8 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 		}
 	}
 
+	reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance))
+
 	for zoneIdx := range erasureSetUpCount {
 		parityDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
 		diskCount := z.zones[zoneIdx].drivesPerSet
@@ -2069,8 +2079,9 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 		}
 		for setIdx := range erasureSetUpCount[zoneIdx] {
 			if erasureSetUpCount[zoneIdx][setIdx] < writeQuorum {
-				logger.LogIf(ctx, fmt.Errorf("Write quorum lost on zone: %d, set: %d, expected write quorum: %d",
-					zoneIdx, setIdx, writeQuorum))
+				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+					fmt.Errorf("Write quorum may be lost on zone: %d, set: %d, expected write quorum: %d",
+						zoneIdx, setIdx, writeQuorum))
 				return HealthResult{
 					Healthy:     false,
 					ZoneID:      zoneIdx,
@@ -2080,8 +2091,35 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 			}
 		}
 	}
+
+	// when maintenance is not specified we don't have
+	// to look at the healing side of the code.
+	if !opts.Maintenance {
+		return HealthResult{
+			Healthy: true,
+		}
+	}
+
+	// check if local disks are being healed, if they are being healed
+	// we need to tell healthy status as 'false' so that this server
+	// is not taken down for maintenance
+	aggHealStateResult, err := getAggregatedBackgroundHealState(ctx, true)
+	if err != nil {
+		logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Unable to verify global heal status: %w", err))
+		return HealthResult{
+			Healthy: false,
+		}
+	}
+
+	if len(aggHealStateResult.HealDisks) > 0 {
+		logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", len(aggHealStateResult.HealDisks)))
+	}
+
+	healthy := len(aggHealStateResult.HealDisks) == 0
+
 	return HealthResult{
-		Healthy: true,
+		Healthy:       healthy,
+		HealingDrives: len(aggHealStateResult.HealDisks),
 	}
 }
 

cmd/erasure.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sort"
 	"sync"
@@ -71,6 +72,11 @@ func (er erasureObjects) NewNSLock(ctx context.Context, bucket string, objects .
 	return er.nsMutex.NewNSLock(ctx, er.getLockers, bucket, objects...)
 }
 
+// SetDriveCount returns the current drives per set.
+func (er erasureObjects) SetDriveCount() int {
+	return len(er.getDisks())
+}
+
 // Shutdown function for object storage interface.
 func (er erasureObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
@@ -89,18 +95,18 @@ func (d byDiskTotal) Less(i, j int) bool {
 
 func diskErrToDriveState(err error) (state string) {
 	state = madmin.DriveStateUnknown
-	switch err {
-	case errDiskNotFound:
+	switch {
+	case errors.Is(err, errDiskNotFound):
 		state = madmin.DriveStateOffline
-	case errCorruptedFormat:
+	case errors.Is(err, errCorruptedFormat):
 		state = madmin.DriveStateCorrupt
-	case errUnformattedDisk:
+	case errors.Is(err, errUnformattedDisk):
 		state = madmin.DriveStateUnformatted
-	case errDiskAccessDenied:
+	case errors.Is(err, errDiskAccessDenied):
 		state = madmin.DriveStatePermission
-	case errFaultyDisk:
+	case errors.Is(err, errFaultyDisk):
 		state = madmin.DriveStateFaulty
-	case nil:
+	case err == nil:
 		state = madmin.DriveStateOk
 	}
 	return
@@ -161,7 +167,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 	errs = g.Wait()
 	// Wait for the routines.
 	for i, diskInfoErr := range errs {
-		ep := endpoints[i]
+		ep := disksInfo[i].Endpoint
 		if diskInfoErr != nil {
 			offlineDisks[ep]++
 			continue

cmd/format-erasure.go

@@ -27,7 +27,6 @@ import (
 	"sync"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/color"
@@ -58,18 +57,6 @@ const (
 // Offline disk UUID represents an offline disk.
 const offlineDiskUUID = "ffffffff-ffff-ffff-ffff-ffffffffffff"
 
-// Healing is only supported for the list of errors mentioned here.
-var formatHealErrors = map[error]struct{}{
-	errUnformattedDisk: {},
-	errDiskNotFound:    {},
-}
-
-// List of errors considered critical for disk formatting.
-var formatCriticalErrors = map[error]struct{}{
-	errCorruptedFormat: {},
-	errFaultyDisk:      {},
-}
-
 // Used to detect the version of "xl" format.
 type formatErasureVersionDetect struct {
 	Erasure struct {
@@ -415,7 +402,8 @@ func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
 			}
 			if !isHiddenDirectories(vols...) {
 				// 'format.json' not found, but we found user data, reject such disks.
-				return nil, errCorruptedFormat
+				return nil, fmt.Errorf("some unexpected files '%v' found on %s: %w",
+					vols, disk, errCorruptedFormat)
 			}
 			// No other data found, its a fresh disk.
 			return nil, errUnformattedDisk
@@ -490,7 +478,8 @@ func formatErasureGetDeploymentID(refFormat *formatErasureV3, formats []*formatE
 			} else if deploymentID != format.ID {
 				// DeploymentID found earlier doesn't match with the
 				// current format.json's ID.
-				return "", errCorruptedFormat
+				return "", fmt.Errorf("Deployment IDs do not match expected %s, got %s: %w",
+					deploymentID, format.ID, errCorruptedFormat)
 			}
 		}
 	}
@@ -500,14 +489,7 @@ func formatErasureGetDeploymentID(refFormat *formatErasureV3, formats []*formatE
 // formatErasureFixDeploymentID - Add deployment id if it is not present.
 func formatErasureFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatErasureV3) (err error) {
 	// Attempt to load all `format.json` from all disks.
-	var sErrs []error
-	formats, sErrs := loadFormatErasureAll(storageDisks, false)
-	for i, sErr := range sErrs {
-		if _, ok := formatCriticalErrors[sErr]; ok {
-			return config.ErrCorruptedBackend(err).Hint(fmt.Sprintf("Clear any pre-existing content on %s", endpoints[i]))
-		}
-	}
-
+	formats, _ := loadFormatErasureAll(storageDisks, false)
 	for index := range formats {
 		// If the Erasure sets do not match, set those formats to nil,
 		// We do not have to update the ID on those format.json file.
@@ -515,6 +497,7 @@ func formatErasureFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI
 			formats[index] = nil
 		}
 	}
+
 	refFormat.ID, err = formatErasureGetDeploymentID(refFormat, formats)
 	if err != nil {
 		return err

cmd/format-erasure_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -436,8 +437,8 @@ func TestGetErasureID(t *testing.T) {
 	}
 
 	formats[2].ID = "bad-id"
-	if _, err = formatErasureGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
-		t.Fatal("Unexpected Success")
+	if _, err = formatErasureGetDeploymentID(quorumFormat, formats); !errors.Is(err, errCorruptedFormat) {
+		t.Fatalf("Unexpect error %s", err)
 	}
 }
 

cmd/fs-v1-helpers.go

@@ -256,7 +256,6 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 
 	// Verify if its not a regular file, since subsequent Seek is undefined.
 	if !st.Mode().IsRegular() {
-		logger.LogIf(ctx, errIsNotRegular)
 		return nil, 0, errIsNotRegular
 	}
 
@@ -286,12 +285,18 @@ func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []
 	}
 
 	if err := mkdirAll(pathutil.Dir(filePath), 0777); err != nil {
-		logger.LogIf(ctx, err)
-		return 0, err
-	}
-
-	if err := checkDiskFree(pathutil.Dir(filePath), fallocSize); err != nil {
-		logger.LogIf(ctx, err)
+		switch {
+		case os.IsPermission(err):
+			return 0, errFileAccessDenied
+		case os.IsExist(err):
+			return 0, errFileAccessDenied
+		case isSysErrIO(err):
+			return 0, errFaultyDisk
+		case isSysErrInvalidArg(err):
+			return 0, errUnsupportedDisk
+		case isSysErrNoSpace(err):
+			return 0, errDiskFull
+		}
 		return 0, err
 	}
 

cmd/fs-v1.go

@@ -191,6 +191,11 @@ func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, objects ...st
 	return fs.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
 }
 
+// SetDriveCount no-op
+func (fs *FSObjects) SetDriveCount() int {
+	return 0
+}
+
 // Shutdown - should be called when process shuts down.
 func (fs *FSObjects) Shutdown(ctx context.Context) error {
 	fs.fsFormatRlk.Close()
@@ -298,6 +303,7 @@ func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter,
 		if intDataUpdateTracker.debug {
 			logger.Info(color.Green("CrawlAndGetDataUsage:")+" Saving totals cache with %d entries", len(totalCache.Cache))
 		}
+		totalCache.Info.LastUpdate = time.Now()
 		logger.LogIf(ctx, totalCache.save(ctx, fs, dataUsageCacheName))
 		cloned := totalCache.clone()
 		updates <- cloned.dui(dataUsageRoot, buckets)
@@ -900,6 +906,56 @@ func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 {
 	return fsMeta
 }
 
+func (fs *FSObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+	fsMeta := fsMetaV1{}
+	if HasSuffix(object, SlashSeparator) {
+		fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object))
+		if err != nil {
+			return oi, err
+		}
+		return fsMeta.ToObjectInfo(bucket, object, fi), nil
+	}
+
+	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+	// Read `fs.json` to perhaps contend with
+	// parallel Put() operations.
+
+	rc, _, err := fsOpenFile(ctx, fsMetaPath, 0)
+	if err == nil {
+		fsMetaBuf, rerr := ioutil.ReadAll(rc)
+		rc.Close()
+		if rerr == nil {
+			var json = jsoniter.ConfigCompatibleWithStandardLibrary
+			if rerr = json.Unmarshal(fsMetaBuf, &fsMeta); rerr != nil {
+				// For any error to read fsMeta, set default ETag and proceed.
+				fsMeta = fs.defaultFsJSON(object)
+			}
+		} else {
+			// For any error to read fsMeta, set default ETag and proceed.
+			fsMeta = fs.defaultFsJSON(object)
+		}
+	}
+
+	// Return a default etag and content-type based on the object's extension.
+	if err == errFileNotFound {
+		fsMeta = fs.defaultFsJSON(object)
+	}
+
+	// Ignore if `fs.json` is not available, this is true for pre-existing data.
+	if err != nil && err != errFileNotFound {
+		logger.LogIf(ctx, err)
+		return oi, err
+	}
+
+	// Stat the file to get file size.
+	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
+	if err != nil {
+		return oi, err
+	}
+
+	return fsMeta.ToObjectInfo(bucket, object, fi), nil
+}
+
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
@@ -1385,14 +1441,13 @@ func (fs *FSObjects) ListObjectVersions(ctx context.Context, bucket, prefix, mar
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
 func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt64(&fs.activeIOCount, -1)
 	}()
 
 	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool,
-		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
+		fs.listDirFactory(), fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
 }
 
 // GetObjectTags - get object tags from an existing object
@@ -1489,7 +1544,7 @@ func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remo
 // error walker returns error. Optionally if context.Done() is received
 // then Walk() stops the walker.
 func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, opts ObjectOptions) error {
-	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfo, fs.getObjectInfo)
+	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
 }
 
 // HealObjects - no-op for fs. Valid only for Erasure.

cmd/gateway-main.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
-	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/certs"
@@ -172,6 +171,23 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
 
+	// Check and load TLS certificates.
+	var err error
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
+	logger.FatalIf(err, "Invalid TLS certificate file")
+
+	// Check and load Root CAs.
+	globalRootCAs, err = certs.GetRootCAs(globalCertsCADir.Get())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Add the global public crts as part of global root CAs
+	for _, publicCrt := range globalPublicCerts {
+		globalRootCAs.AddCert(publicCrt)
+	}
+
+	// Register root CAs for remote ENVs
+	env.RegisterGlobalCAs(globalRootCAs)
+
 	// Initialize all help
 	initHelp()
 
@@ -184,15 +200,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// To avoid this error situation we check for port availability.
 	logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the gateway")
 
-	// Check and load TLS certificates.
-	var err error
-	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
-	logger.FatalIf(err, "Invalid TLS certificate file")
-
-	// Check and load Root CAs.
-	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
-	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
-
 	globalMinioEndpoint = func() string {
 		host := globalMinioHost
 		if host == "" {
@@ -220,7 +227,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	srvCfg := newServerConfig()
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg)
+	lookupConfigs(srvCfg, 0)
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/gateway-unsupported.go

@@ -46,6 +46,11 @@ func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, object
 	return nil
 }
 
+// SetDriveCount no-op
+func (a GatewayUnsupported) SetDriveCount() int {
+	return 0
+}
+
 // ListMultipartUploads lists all multipart uploads.
 func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
 	return lmi, NotImplemented{}

cmd/global-heal.go

@@ -53,7 +53,9 @@ func newBgHealSequence() *healSequence {
 		respCh:      make(chan healResult),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
-		settings:    hs,
+		// run-background heal with reserved bucket
+		bucket:   minioReservedBucket,
+		settings: hs,
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
@@ -67,17 +69,25 @@ func newBgHealSequence() *healSequence {
 	}
 }
 
-func getLocalBackgroundHealStatus() madmin.BgHealState {
+func getLocalBackgroundHealStatus() (madmin.BgHealState, bool) {
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if !ok {
-		return madmin.BgHealState{}
+		return madmin.BgHealState{}, false
+	}
+
+	var healDisks []string
+	for _, eps := range globalBackgroundHealState.getHealLocalDisks() {
+		for _, ep := range eps {
+			healDisks = append(healDisks, ep.String())
+		}
 	}
 
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.getScannedItemsCount(),
 		LastHealActivity:  bgSeq.lastHealActivity,
+		HealDisks:         healDisks,
 		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
-	}
+	}, true
 }
 
 // healErasureSet lists and heals all objects in a specific erasure set
@@ -164,13 +174,14 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, dr
 // deepHealObject heals given object path in deep to fix bitrot.
 func deepHealObject(bucket, object, versionID string) {
 	// Get background heal sequence to send elements to heal
-	bgSeq, _ := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-
-	bgSeq.sourceCh <- healSource{
-		bucket:    bucket,
-		object:    object,
-		versionID: versionID,
-		opts:      &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
+	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+	if ok {
+		bgSeq.sourceCh <- healSource{
+			bucket:    bucket,
+			object:    object,
+			versionID: versionID,
+			opts:      &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
+		}
 	}
 }
 

cmd/globals.go

@@ -58,7 +58,6 @@ const (
 	globalMinioDefaultOwnerID      = "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4"
 	globalMinioDefaultStorageClass = "STANDARD"
 	globalWindowsOSName            = "windows"
-	globalNetBSDOSName             = "netbsd"
 	globalMacOSName                = "darwin"
 	globalMinioModeFS              = "mode-server-fs"
 	globalMinioModeErasure         = "mode-server-xl"
@@ -109,9 +108,6 @@ var globalCLIContext = struct {
 }{}
 
 var (
-	// Indicates set drive count.
-	globalErasureSetDriveCount int
-
 	// Indicates if the running minio server is distributed setup.
 	globalIsDistErasure = false
 
@@ -154,9 +150,9 @@ var (
 	globalPolicySys         *PolicySys
 	globalIAMSys            *IAMSys
 
-	globalLifecycleSys         *LifecycleSys
-	globalBucketSSEConfigSys   *BucketSSEConfigSys
-	globalBucketReplicationSys *BucketReplicationSys
+	globalLifecycleSys       *LifecycleSys
+	globalBucketSSEConfigSys *BucketSSEConfigSys
+	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
 	// healthcheck readiness deadlines and cors settings.
 	globalAPIConfig apiConfig

cmd/handler-utils.go

@@ -131,6 +131,13 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 		metadata[strings.ToLower(xhttp.ContentType)] = "application/octet-stream"
 	}
 
+	// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+	for k := range metadata {
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			delete(metadata, k)
+		}
+	}
+
 	if contentEncoding, ok := metadata[strings.ToLower(xhttp.ContentEncoding)]; ok {
 		contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
 		if contentEncoding != "" {
@@ -480,6 +487,12 @@ func getHostName(r *http.Request) (hostName string) {
 func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, ep ProxyEndpoint) (success bool) {
 	success = true
 
+	// Make sure we remove any existing headers before
+	// proxying the request to another node.
+	for k := range w.Header() {
+		w.Header().Del(k)
+	}
+
 	f := handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
 		RoundTripper: ep.Transport,

cmd/healthcheck-handler.go

@@ -23,7 +23,7 @@ import (
 
 // ClusterCheckHandler returns if the server is ready for requests.
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ClusterCheckCheckHandler")
+	ctx := newContext(r, w, "ClusterCheckHandler")
 
 	objLayer := newObjectLayerFn()
 	// Service not initialized yet

cmd/http-tracer.go

@@ -129,6 +129,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 
 	rq := trace.RequestInfo{
 		Time:     time.Now().UTC(),
+		Proto:    r.Proto,
 		Method:   r.Method,
 		Path:     r.URL.Path,
 		RawQuery: r.URL.RawQuery,

cmd/http/dial_linux.go

@@ -49,20 +49,17 @@ func setInternalTCPParameters(c syscall.RawConn) error {
 		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 5)
 
 		// Wait time after successful probe in seconds.
-		// ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 2 secs)
+		// ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 3 secs)
 		// 75
-		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 2)
-
-		// Set TCP_USER_TIMEOUT to TCP_KEEPIDLE + TCP_KEEPINTVL * TCP_KEEPCNT.
-		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, 15)
+		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 3)
 	})
 }
 
 // DialContext is a function to make custom Dial for internode communications
 type DialContext func(ctx context.Context, network, address string) (net.Conn, error)
 
-// NewCustomDialContext setups a custom dialer for internode communications
-func NewCustomDialContext(dialTimeout time.Duration) DialContext {
+// NewInternodeDialContext setups a custom dialer for internode communication
+func NewInternodeDialContext(dialTimeout time.Duration) DialContext {
 	return func(ctx context.Context, network, addr string) (net.Conn, error) {
 		dialer := &net.Dialer{
 			Timeout: dialTimeout,
@@ -73,3 +70,25 @@ func NewCustomDialContext(dialTimeout time.Duration) DialContext {
 		return dialer.DialContext(ctx, network, addr)
 	}
 }
+
+// NewCustomDialContext setups a custom dialer for any external communication and proxies.
+func NewCustomDialContext(dialTimeout time.Duration) DialContext {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		dialer := &net.Dialer{
+			Timeout: dialTimeout,
+			Control: func(network, address string, c syscall.RawConn) error {
+				return c.Control(func(fdPtr uintptr) {
+					// got socket file descriptor to set parameters.
+					fd := int(fdPtr)
+
+					// Enable TCP fast connect
+					// TCPFastOpenConnect sets the underlying socket to use
+					// the TCP fast open connect. This feature is supported
+					// since Linux 4.11.
+					_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
+				})
+			},
+		}
+		return dialer.DialContext(ctx, network, addr)
+	}
+}

cmd/http/dial_others.go

@@ -33,6 +33,9 @@ func setInternalTCPParameters(c syscall.RawConn) error {
 // DialContext is a function to make custom Dial for internode communications
 type DialContext func(ctx context.Context, network, address string) (net.Conn, error)
 
+// NewInternodeDialContext setups a custom dialer for internode communication
+var NewInternodeDialContext = NewCustomDialContext
+
 // NewCustomDialContext configures a custom dialer for internode communications
 func NewCustomDialContext(dialTimeout time.Duration) DialContext {
 	return func(ctx context.Context, network, addr string) (net.Conn, error) {

cmd/http/headers.go

@@ -102,6 +102,9 @@ const (
 	AmzSecurityToken        = "X-Amz-Security-Token"
 	AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
 
+	AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length"
+	AmzMetaUnencryptedContentMD5    = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5"
+
 	// Signature v2 related constants
 	AmzSignatureV2 = "Signature"
 	AmzAccessKeyID = "AWSAccessKeyId"
@@ -120,6 +123,9 @@ const (
 
 	// Header indicates if the mtime should be preserved by client
 	MinIOSourceMTime = "x-minio-source-mtime"
+
+	// Header indicates if the etag should be preserved by client
+	MinIOSourceETag = "x-minio-source-etag"
 )
 
 // Common http query params S3 API

cmd/http/server.go

@@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			// TLS hardening
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
-			// Do not edit the next line, protos priority is kept
-			// on purpose in this manner for HTTP 2.0, we would
-			// still like HTTP 2.0 clients to negotiate connection
-			// to server if needed but by default HTTP 1.1 is
-			// expected. We need to change this in future
-			// when we wish to go back to HTTP 2.0 as default
-			// priority for HTTP protocol negotiation.
-			NextProtos: []string{"http/1.1", "h2"},
+			NextProtos:               []string{"h2", "http/1.1"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/iam-etcd-store.go

@@ -516,6 +516,28 @@ func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
 		sys.iamUserPolicyMap[k] = v
 	}
 
+	// purge any expired entries which became expired now.
+	var expiredEntries []string
+	for k, v := range sys.iamUsersMap {
+		if v.IsExpired() {
+			delete(sys.iamUsersMap, k)
+			delete(sys.iamUserPolicyMap, k)
+			expiredEntries = append(expiredEntries, k)
+			// Deleting on the disk is taken care of in the next cycle
+		}
+	}
+
+	for _, v := range sys.iamUsersMap {
+		if v.IsServiceAccount() {
+			for _, accessKey := range expiredEntries {
+				if v.ParentUser == accessKey {
+					_ = ies.deleteUserIdentity(v.AccessKey, srvAccUser)
+					delete(sys.iamUsersMap, v.AccessKey)
+				}
+			}
+		}
+	}
+
 	// purge any expired entries which became expired now.
 	for k, v := range sys.iamUsersMap {
 		if v.IsExpired() {

cmd/iam-object-store.go

@@ -151,7 +151,7 @@ func (iamOS *IAMObjectStore) migrateUsersConfigToV1(ctx context.Context, isSTS b
 		cred.AccessKey = user
 		u := newUserIdentity(cred)
 		if err := iamOS.saveIAMConfig(u, identityPath); err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(ctx, err)
 			return err
 		}
 
@@ -216,7 +216,7 @@ func (iamOS *IAMObjectStore) saveIAMConfig(item interface{}, path string) error
 			return err
 		}
 	}
-	return saveConfig(context.Background(), iamOS.objAPI, path, data)
+	return saveConfig(GlobalContext, iamOS.objAPI, path, data)
 }
 
 func (iamOS *IAMObjectStore) loadIAMConfig(item interface{}, path string) error {
@@ -304,6 +304,7 @@ func (iamOS *IAMObjectStore) loadUser(user string, userType IAMUserType, m map[s
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
+
 	m[user] = u.Credentials
 	return nil
 }
@@ -474,14 +475,27 @@ func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	}
 
 	// purge any expired entries which became expired now.
+	var expiredEntries []string
 	for k, v := range sys.iamUsersMap {
 		if v.IsExpired() {
 			delete(sys.iamUsersMap, k)
 			delete(sys.iamUserPolicyMap, k)
+			expiredEntries = append(expiredEntries, k)
 			// Deleting on the disk is taken care of in the next cycle
 		}
 	}
 
+	for _, v := range sys.iamUsersMap {
+		if v.IsServiceAccount() {
+			for _, accessKey := range expiredEntries {
+				if v.ParentUser == accessKey {
+					_ = iamOS.deleteUserIdentity(v.AccessKey, srvAccUser)
+					delete(sys.iamUsersMap, v.AccessKey)
+				}
+			}
+		}
+	}
+
 	for k, v := range iamGroupPolicyMap {
 		sys.iamGroupPolicyMap[k] = v
 	}
@@ -572,7 +586,7 @@ func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix stri
 
 		marker := ""
 		for {
-			lo, err := objAPI.ListObjects(context.Background(),
+			lo, err := objAPI.ListObjects(ctx,
 				minioMetaBucket, pathPrefix, marker, SlashSeparator, maxObjectList)
 			if err != nil {
 				select {

cmd/iam.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
@@ -451,7 +452,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
-		if err := txnLk.GetLock(newDynamicTimeout(1*time.Second, 5*time.Second)); err != nil {
+		if err := txnLk.GetLock(newDynamicTimeout(3*time.Second, 5*time.Second)); err != nil {
 			logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. trying to acquire lock")
 			continue
 		}
@@ -929,7 +930,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, ses
 		if err != nil {
 			return auth.Credentials{}, err
 		}
-		if len(policyBuf) > 16*1024 {
+		if len(policyBuf) > 16*humanize.KiByte {
 			return auth.Credentials{}, fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
 	}
@@ -951,14 +952,6 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, ses
 		return auth.Credentials{}, errIAMActionNotAllowed
 	}
 
-	// FIXME: Disallow temporary users with no parent, this is most
-	// probably with OpenID which we don't support to provide
-	// any parent user, LDAPUsersType and MinIOUsersType are
-	// currently supported.
-	if cr.ParentUser == "" && cr.IsTemp() {
-		return auth.Credentials{}, errIAMActionNotAllowed
-	}
-
 	m := make(map[string]interface{})
 	m[parentClaim] = parentUser
 
@@ -1647,7 +1640,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 	if err != nil {
 		// Log any error in input session policy config.
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return false
 	}
 
@@ -1790,7 +1783,7 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
 		subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 		if err != nil {
 			// Log any error in input session policy config.
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 			return false
 		}
 
@@ -1814,7 +1807,7 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	if globalPolicyOPA != nil {
 		ok, err := globalPolicyOPA.IsAllowed(args)
 		if err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 		}
 		return ok
 	}

cmd/listen-notification-handlers.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	policy "github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
@@ -119,7 +118,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()})
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()

cmd/local-locker.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"time"
@@ -71,30 +72,35 @@ func (l *localLocker) canTakeLock(resources ...string) bool {
 	return noLkCnt == len(resources)
 }
 
-func (l *localLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
 
-	if !l.canTakeLock(args.Resources...) {
-		// Not all locks can be taken on resources,
-		// reject it completely.
-		return false, nil
-	}
-
-	// No locks held on the all resources, so claim write
-	// lock on all resources at once.
-	for _, resource := range args.Resources {
-		l.lockMap[resource] = []lockRequesterInfo{
-			{
-				Writer:        true,
-				Source:        args.Source,
-				UID:           args.UID,
-				Timestamp:     UTCNow(),
-				TimeLastCheck: UTCNow(),
-			},
+		if !l.canTakeLock(args.Resources...) {
+			// Not all locks can be taken on resources,
+			// reject it completely.
+			return false, nil
 		}
+
+		// No locks held on the all resources, so claim write
+		// lock on all resources at once.
+		for _, resource := range args.Resources {
+			l.lockMap[resource] = []lockRequesterInfo{
+				{
+					Writer:        true,
+					Source:        args.Source,
+					UID:           args.UID,
+					Timestamp:     UTCNow(),
+					TimeLastCheck: UTCNow(),
+				},
+			}
+		}
+		return true, nil
 	}
-	return true, nil
 }
 
 func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
@@ -138,28 +144,33 @@ func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bo
 	return false
 }
 
-func (l *localLocker) RLock(args dsync.LockArgs) (reply bool, err error) {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
-	lrInfo := lockRequesterInfo{
-		Writer:        false,
-		Source:        args.Source,
-		UID:           args.UID,
-		Timestamp:     UTCNow(),
-		TimeLastCheck: UTCNow(),
-	}
-	resource := args.Resources[0]
-	if lri, ok := l.lockMap[resource]; ok {
-		if reply = !isWriteLock(lri); reply {
-			// Unless there is a write lock
-			l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
+func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
+		lrInfo := lockRequesterInfo{
+			Writer:        false,
+			Source:        args.Source,
+			UID:           args.UID,
+			Timestamp:     UTCNow(),
+			TimeLastCheck: UTCNow(),
 		}
-	} else {
-		// No locks held on the given name, so claim (first) read lock
-		l.lockMap[resource] = []lockRequesterInfo{lrInfo}
-		reply = true
+		resource := args.Resources[0]
+		if lri, ok := l.lockMap[resource]; ok {
+			if reply = !isWriteLock(lri); reply {
+				// Unless there is a write lock
+				l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
+			}
+		} else {
+			// No locks held on the given name, so claim (first) read lock
+			l.lockMap[resource] = []lockRequesterInfo{lrInfo}
+			reply = true
+		}
+		return reply, nil
 	}
-	return reply, nil
 }
 
 func (l *localLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
@@ -202,22 +213,27 @@ func (l *localLocker) IsOnline() bool {
 	return true
 }
 
-func (l *localLocker) Expired(args dsync.LockArgs) (expired bool, err error) {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired bool, err error) {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
 
-	// Lock found, proceed to verify if belongs to given uid.
-	for _, resource := range args.Resources {
-		if lri, ok := l.lockMap[resource]; ok {
-			// Check whether uid is still active
-			for _, entry := range lri {
-				if entry.UID == args.UID {
-					return false, nil
+		// Lock found, proceed to verify if belongs to given uid.
+		for _, resource := range args.Resources {
+			if lri, ok := l.lockMap[resource]; ok {
+				// Check whether uid is still active
+				for _, entry := range lri {
+					if entry.UID == args.UID {
+						return false, nil
+					}
 				}
 			}
 		}
+		return true, nil
 	}
-	return true, nil
 }
 
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.

cmd/lock-rest-client.go

@@ -58,7 +58,7 @@ func (client *lockRESTClient) String() string {
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
-func (client *lockRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+func (client *lockRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
 	if values == nil {
 		values = make(url.Values)
 	}
@@ -83,7 +83,7 @@ func (client *lockRESTClient) Close() error {
 }
 
 // restCall makes a call to the lock REST server.
-func (client *lockRESTClient) restCall(call string, args dsync.LockArgs) (reply bool, err error) {
+func (client *lockRESTClient) restCall(ctx context.Context, call string, args dsync.LockArgs) (reply bool, err error) {
 	values := url.Values{}
 	values.Set(lockRESTUID, args.UID)
 	values.Set(lockRESTSource, args.Source)
@@ -92,7 +92,7 @@ func (client *lockRESTClient) restCall(call string, args dsync.LockArgs) (reply
 		buffer.WriteString(resource)
 		buffer.WriteString("\n")
 	}
-	respBody, err := client.call(call, values, &buffer, -1)
+	respBody, err := client.callWithContext(ctx, call, values, &buffer, -1)
 	defer http.DrainBody(respBody)
 	switch err {
 	case nil:
@@ -105,28 +105,28 @@ func (client *lockRESTClient) restCall(call string, args dsync.LockArgs) (reply
 }
 
 // RLock calls read lock REST API.
-func (client *lockRESTClient) RLock(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(lockRESTMethodRLock, args)
+func (client *lockRESTClient) RLock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(ctx, lockRESTMethodRLock, args)
 }
 
 // Lock calls lock REST API.
-func (client *lockRESTClient) Lock(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(lockRESTMethodLock, args)
+func (client *lockRESTClient) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(ctx, lockRESTMethodLock, args)
 }
 
 // RUnlock calls read unlock REST API.
 func (client *lockRESTClient) RUnlock(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(lockRESTMethodRUnlock, args)
+	return client.restCall(context.Background(), lockRESTMethodRUnlock, args)
 }
 
 // Unlock calls write unlock RPC.
 func (client *lockRESTClient) Unlock(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(lockRESTMethodUnlock, args)
+	return client.restCall(context.Background(), lockRESTMethodUnlock, args)
 }
 
 // Expired calls expired handler to check if lock args have expired.
-func (client *lockRESTClient) Expired(args dsync.LockArgs) (expired bool, err error) {
-	return client.restCall(lockRESTMethodExpired, args)
+func (client *lockRESTClient) Expired(ctx context.Context, args dsync.LockArgs) (expired bool, err error) {
+	return client.restCall(ctx, lockRESTMethodExpired, args)
 }
 
 func newLockAPI(endpoint Endpoint) dsync.NetLocker {
@@ -149,11 +149,10 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 
-	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
+	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
 	restClient.HealthCheckFn = func() bool {
 		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)

cmd/lock-rest-client_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"testing"
 
 	"github.com/minio/minio/pkg/dsync"
@@ -34,12 +35,12 @@ func TestLockRESTlient(t *testing.T) {
 	}
 
 	// Attempt all calls.
-	_, err = lkClient.RLock(dsync.LockArgs{})
+	_, err = lkClient.RLock(context.Background(), dsync.LockArgs{})
 	if err == nil {
 		t.Fatal("Expected for Rlock to fail")
 	}
 
-	_, err = lkClient.Lock(dsync.LockArgs{})
+	_, err = lkClient.Lock(context.Background(), dsync.LockArgs{})
 	if err == nil {
 		t.Fatal("Expected for Lock to fail")
 	}

cmd/lock-rest-server.go

@@ -96,7 +96,7 @@ func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	success, err := l.ll.Lock(args)
+	success, err := l.ll.Lock(r.Context(), args)
 	if err == nil && !success {
 		err = errLockConflict
 	}
@@ -141,7 +141,7 @@ func (l *lockRESTServer) RLockHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	success, err := l.ll.RLock(args)
+	success, err := l.ll.RLock(r.Context(), args)
 	if err == nil && !success {
 		err = errLockConflict
 	}
@@ -185,20 +185,14 @@ func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	l.ll.mutex.Lock()
-	defer l.ll.mutex.Unlock()
-
-	// Lock found, proceed to verify if belongs to given uid.
-	for _, resource := range args.Resources {
-		if lri, ok := l.ll.lockMap[resource]; ok {
-			// Check whether uid is still active
-			for _, entry := range lri {
-				if entry.UID == args.UID {
-					l.writeErrorResponse(w, errLockNotExpired)
-					return
-				}
-			}
-		}
+	expired, err := l.ll.Expired(r.Context(), args)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+	if !expired {
+		l.writeErrorResponse(w, errLockNotExpired)
+		return
 	}
 }
 
@@ -219,7 +213,10 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 			for idx := range lriArray {
 				// Check whether enough time has gone by since last check
 				if time.Since(lriArray[idx].TimeLastCheck) >= interval {
-					rslt = append(rslt, nameLockRequesterInfoPair{name: name, lri: lriArray[idx]})
+					rslt = append(rslt, nameLockRequesterInfoPair{
+						name: name,
+						lri:  lriArray[idx],
+					})
 					lriArray[idx].TimeLastCheck = UTCNow()
 				}
 			}
@@ -239,6 +236,11 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
 func lockMaintenance(ctx context.Context, interval time.Duration) error {
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI == nil {
+		return nil
+	}
+
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
 	for lendpoint, nlrips := range getLongLivedLocks(interval) {
@@ -254,12 +256,15 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 					continue
 				}
 
+				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
+
 				// Call back to original server verify whether the lock is
 				// still active (based on name & uid)
-				expired, err := c.Expired(dsync.LockArgs{
+				expired, err := c.Expired(ctx, dsync.LockArgs{
 					UID:       nlrip.lri.UID,
 					Resources: []string{nlrip.name},
 				})
+				cancel()
 				if err != nil {
 					nlripsMap[nlrip.name]++
 					c.Close()
@@ -275,10 +280,10 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 			}
 
 			// Read locks we assume quorum for be N/2 success
-			quorum := globalErasureSetDriveCount / 2
+			quorum := objAPI.SetDriveCount() / 2
 			if nlrip.lri.Writer {
 				// For write locks we need N/2+1 success
-				quorum = globalErasureSetDriveCount/2 + 1
+				quorum = objAPI.SetDriveCount()/2 + 1
 			}
 
 			// less than the quorum, we have locks expired.

cmd/logger/audit.go

@@ -125,15 +125,6 @@ func (lrw *ResponseWriter) Size() int {
 	return lrw.bytesWritten
 }
 
-// AuditTargets is the list of enabled audit loggers
-var AuditTargets = []Target{}
-
-// AddAuditTarget adds a new audit logger target to the
-// list of enabled loggers
-func AddAuditTarget(t Target) {
-	AuditTargets = append(AuditTargets, t)
-}
-
 // AuditLog - logs audit logs to all audit targets.
 func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}, filterKeys ...string) {
 	// Fast exit if there is not audit target configured

cmd/logger/target/console/console.go

@@ -32,6 +32,11 @@ import (
 // in plain or json format to the standard output.
 type Target struct{}
 
+// Validate - validate if the tty can be written to
+func (c *Target) Validate() error {
+	return nil
+}
+
 // Send log message 'e' to console
 func (c *Target) Send(e interface{}, logKind string) error {
 	entry, ok := e.(log.Entry)

cmd/logger/target/http/http.go

@@ -18,12 +18,16 @@ package http
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"strings"
+	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // Target implements logger.Target and sends the json
@@ -45,6 +49,47 @@ type Target struct {
 	client    http.Client
 }
 
+// Validate validate the http target
+func (h *Target) Validate() error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.endpoint, strings.NewReader(`{}`))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set(xhttp.ContentType, "application/json")
+
+	// Set user-agent to indicate MinIO release
+	// version to the configured log endpoint
+	req.Header.Set("User-Agent", h.userAgent)
+
+	if h.authToken != "" {
+		req.Header.Set("Authorization", h.authToken)
+	}
+
+	resp, err := h.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	// Drain any response.
+	xhttp.DrainBody(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		switch resp.StatusCode {
+		case http.StatusForbidden:
+			return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set",
+				h.endpoint, resp.Status)
+		}
+		return fmt.Errorf("%s returned '%s', please check your endpoint configuration",
+			h.endpoint, resp.Status)
+	}
+
+	return nil
+}
+
 func (h *Target) startHTTPLogger() {
 	// Create a routine which sends json logs received
 	// from an internal channel.
@@ -55,8 +100,11 @@ func (h *Target) startHTTPLogger() {
 				continue
 			}
 
-			req, err := http.NewRequest(http.MethodPost, h.endpoint, bytes.NewReader(logJSON))
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+				h.endpoint, bytes.NewReader(logJSON))
 			if err != nil {
+				cancel()
 				continue
 			}
 			req.Header.Set(xhttp.ContentType, "application/json")
@@ -70,13 +118,26 @@ func (h *Target) startHTTPLogger() {
 			}
 
 			resp, err := h.client.Do(req)
+			cancel()
 			if err != nil {
-				h.client.CloseIdleConnections()
+				logger.LogIf(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration\n",
+					h.endpoint, err))
 				continue
 			}
 
 			// Drain any response.
 			xhttp.DrainBody(resp.Body)
+
+			if resp.StatusCode != http.StatusOK {
+				switch resp.StatusCode {
+				case http.StatusForbidden:
+					logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set",
+						h.endpoint, resp.Status))
+				default:
+					logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration",
+						h.endpoint, resp.Status))
+				}
+			}
 		}
 	}()
 }

cmd/logger/targets.go

@@ -20,14 +20,33 @@ package logger
 // a single log entry and Send it to the log target
 //   e.g. Send the log to a http server
 type Target interface {
+	Validate() error
 	Send(entry interface{}, errKind string) error
 }
 
 // Targets is the set of enabled loggers
 var Targets = []Target{}
 
+// AuditTargets is the list of enabled audit loggers
+var AuditTargets = []Target{}
+
+// AddAuditTarget adds a new audit logger target to the
+// list of enabled loggers
+func AddAuditTarget(t Target) error {
+	if err := t.Validate(); err != nil {
+		return err
+	}
+
+	AuditTargets = append(AuditTargets, t)
+	return nil
+}
+
 // AddTarget adds a new logger target to the
 // list of enabled loggers
-func AddTarget(t Target) {
+func AddTarget(t Target) error {
+	if err := t.Validate(); err != nil {
+		return err
+	}
 	Targets = append(Targets, t)
+	return nil
 }

cmd/metrics.go

@@ -378,6 +378,10 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		return
 	}
 
+	if globalIsGateway {
+		return
+	}
+
 	// Crawler disabled, nothing to do.
 	if env.Get(envDataUsageCrawlConf, config.EnableOn) != config.EnableOn {
 		return

cmd/namespace-lock.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/lsync"
@@ -147,7 +148,18 @@ func (di *distLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error)
 	lockSource := getSource(2)
 	start := UTCNow()
 
-	if !di.rwMutex.GetLock(di.ctx, di.opsID, lockSource, timeout.Timeout()) {
+	// Lockers default to standard storage class always, why because
+	// we always dictate storage tolerance in terms of standard
+	// storage class be it number of drives or a multiplicative
+	// of number of nodes, defaulting lockers to this value
+	// simply means that locking is always similar in behavior
+	// and effect with erasure coded drive tolerance.
+	tolerance := globalStorageClass.GetParityForSC(storageclass.STANDARD)
+
+	if !di.rwMutex.GetLock(di.ctx, di.opsID, lockSource, dsync.Options{
+		Timeout:   timeout.Timeout(),
+		Tolerance: tolerance,
+	}) {
 		timeout.LogFailure()
 		return OperationTimedOut{}
 	}
@@ -164,7 +176,14 @@ func (di *distLockInstance) Unlock() {
 func (di *distLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr error) {
 	lockSource := getSource(2)
 	start := UTCNow()
-	if !di.rwMutex.GetRLock(di.ctx, di.opsID, lockSource, timeout.Timeout()) {
+
+	// Lockers default to standard storage class always.
+	tolerance := globalStorageClass.GetParityForSC(storageclass.STANDARD)
+
+	if !di.rwMutex.GetRLock(di.ctx, di.opsID, lockSource, dsync.Options{
+		Timeout:   timeout.Timeout(),
+		Tolerance: tolerance,
+	}) {
 		timeout.LogFailure()
 		return OperationTimedOut{}
 	}

cmd/notification.go

@@ -269,21 +269,26 @@ func (sys *NotificationSys) LoadServiceAccount(accessKey string) []NotificationP
 }
 
 // BackgroundHealStatus - returns background heal status of all peers
-func (sys *NotificationSys) BackgroundHealStatus() []madmin.BgHealState {
+func (sys *NotificationSys) BackgroundHealStatus() ([]madmin.BgHealState, []NotificationPeerErr) {
+	ng := WithNPeers(len(sys.peerClients))
 	states := make([]madmin.BgHealState, len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
 			continue
 		}
-		st, err := client.BackgroundHealStatus()
-		if err != nil {
-			logger.LogIf(GlobalContext, err)
-		} else {
+		idx := idx
+		client := client
+		ng.Go(GlobalContext, func() error {
+			st, err := client.BackgroundHealStatus()
+			if err != nil {
+				return err
+			}
 			states[idx] = st
-		}
+			return nil
+		}, idx, *client.host)
 	}
 
-	return states
+	return states, ng.Wait()
 }
 
 // StartProfiling - start profiling on remote peers, by initiating a remote RPC.

cmd/object-api-datatypes.go

@@ -198,11 +198,14 @@ type ObjectInfo struct {
 	PutObjReader *PutObjReader  `json:"-"`
 
 	metadataOnly bool
+	versionOnly  bool // adds a new version, only used by CopyObject
 	keyRotation  bool
 
 	// Date and time when the object was last accessed.
 	AccTime time.Time
 
+	Legacy bool // indicates object on disk is in legacy data format
+
 	// backendType indicates which backend filled this structure
 	backendType BackendType
 }

cmd/object-api-errors.go

@@ -362,11 +362,46 @@ func (e BucketReplicationDestinationNotFound) Error() string {
 	return "Destination bucket does not exist: " + e.Bucket
 }
 
-// BucketReplicationTargetNotFound replication target does not exist.
-type BucketReplicationTargetNotFound GenericError
+// BucketReplicationDestinationMissingLock bucket does not have object lock enabled.
+type BucketReplicationDestinationMissingLock GenericError
 
-func (e BucketReplicationTargetNotFound) Error() string {
-	return "Replication target not found: " + e.Bucket
+func (e BucketReplicationDestinationMissingLock) Error() string {
+	return "Destination bucket does not have object lock enabled: " + e.Bucket
+}
+
+// BucketRemoteTargetNotFound remote target does not exist.
+type BucketRemoteTargetNotFound GenericError
+
+func (e BucketRemoteTargetNotFound) Error() string {
+	return "Remote target not found: " + e.Bucket
+}
+
+// BucketRemoteAlreadyExists remote already exists for this target type.
+type BucketRemoteAlreadyExists GenericError
+
+func (e BucketRemoteAlreadyExists) Error() string {
+	return "Remote already exists for this bucket: " + e.Bucket
+}
+
+// BucketRemoteArnTypeInvalid arn type for remote is not valid.
+type BucketRemoteArnTypeInvalid GenericError
+
+func (e BucketRemoteArnTypeInvalid) Error() string {
+	return "Remote ARN type not valid: " + e.Bucket
+}
+
+// BucketRemoteArnInvalid arn needs to be specified.
+type BucketRemoteArnInvalid GenericError
+
+func (e BucketRemoteArnInvalid) Error() string {
+	return "Remote ARN has invalid format: " + e.Bucket
+}
+
+// BucketRemoteRemoveDisallowed when replication configuration exists
+type BucketRemoteRemoveDisallowed GenericError
+
+func (e BucketRemoteRemoveDisallowed) Error() string {
+	return "Replication configuration exists with this ARN:" + e.Bucket
 }
 
 // BucketReplicationTargetNotVersioned replication target does not have versioning enabled.
@@ -376,6 +411,13 @@ func (e BucketReplicationTargetNotVersioned) Error() string {
 	return "Replication target does not have versioning enabled: " + e.Bucket
 }
 
+// BucketReplicationSourceNotVersioned replication source does not have versioning enabled.
+type BucketReplicationSourceNotVersioned GenericError
+
+func (e BucketReplicationSourceNotVersioned) Error() string {
+	return "Replication source does not have versioning enabled: " + e.Bucket
+}
+
 /// Bucket related errors.
 
 // BucketNameInvalid - bucketname provided is invalid.

cmd/object-api-interface.go

@@ -64,6 +64,8 @@ const (
 
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
+	SetDriveCount() int // Only implemented by erasure layer
+
 	// Locking operations on object.
 	NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker
 

cmd/object-api-listobjects_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015-2016, 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -448,7 +448,9 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 				{Name: "temporary/0/"},
 			},
 		},
-		// ListObjectsResult-34 Listing with marker > last object should return empty
+		// ListObjectsResult-34:
+		//    * Listing with marker > last object should return empty
+		//    * Listing an object with a trailing slash and '/' delimiter
 		{
 			IsTruncated: false,
 			Objects:     []ObjectInfo{},
@@ -569,6 +571,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		{"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true},
 		// Test listing with marker > last object such that response should be empty (65)
 		{"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true},
+		// Test listing an object with a trailing slash and a slash delimiter (66)
+		{"test-bucket-list-object", "Asia-maps.png/", "", "/", 1000, resultCases[34], nil, true},
 	}
 
 	for i, testCase := range testCases {

cmd/object-api-options.go

@@ -164,7 +164,13 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 	} else {
 		opts.MTime = UTCNow()
 	}
-
+	etag := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceETag))
+	if etag != "" {
+		if metadata == nil {
+			metadata = make(map[string]string)
+		}
+		metadata["etag"] = etag
+	}
 	// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
 	// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
 	if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {

cmd/object-api-utils.go

@@ -400,6 +400,15 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
 	return true, fmt.Errorf("unknown compression scheme: %s", scheme)
 }
 
+// GetActualETag - returns the actual etag of the stored object
+// decrypts SSE objects.
+func (o ObjectInfo) GetActualETag(h http.Header) string {
+	if !crypto.IsEncrypted(o.UserDefined) {
+		return o.ETag
+	}
+	return getDecryptedETag(h, o, false)
+}
+
 // GetActualSize - returns the actual size of the stored object
 func (o ObjectInfo) GetActualSize() (int64, error) {
 	if crypto.IsEncrypted(o.UserDefined) {

cmd/object-handlers.go

@@ -1155,6 +1155,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if objTags != "" {
 		srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
 	}
+	srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
+
 	srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
 	retPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectLegalHoldAction)
@@ -1177,7 +1179,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if globalBucketReplicationSys.mustReplicate(ctx, r, dstBucket, dstObject, srcInfo.UserDefined, srcInfo.ReplicationStatus.String()) {
+	if mustReplicate(ctx, r, dstBucket, dstObject, srcInfo.UserDefined, srcInfo.ReplicationStatus.String()) {
 		srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 
@@ -1258,8 +1260,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
-	if globalBucketReplicationSys.mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		defer replicateObject(ctx, dstBucket, dstObject, objInfo.VersionID, objectAPI, &eventArgs{
+	if mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
+		defer replicateObject(GlobalContext, dstBucket, dstObject, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedCopy,
 			BucketName:   dstBucket,
 			Object:       objInfo,
@@ -1511,7 +1513,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if globalBucketReplicationSys.mustReplicate(ctx, r, bucket, object, metadata, "") {
+	if mustReplicate(ctx, r, bucket, object, metadata, "") {
 		metadata[xhttp.AmzBucketReplicationStatus] = string(replication.Pending)
 	}
 	if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
@@ -1574,8 +1576,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			}
 		}
 	}
-	if globalBucketReplicationSys.mustReplicate(ctx, r, bucket, object, metadata, "") {
-		defer replicateObject(ctx, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+	if mustReplicate(ctx, r, bucket, object, metadata, "") {
+		defer replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedPut,
 			BucketName:   bucket,
 			Object:       objInfo,
@@ -1696,7 +1698,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if globalBucketReplicationSys.mustReplicate(ctx, r, bucket, object, metadata, "") {
+	if mustReplicate(ctx, r, bucket, object, metadata, "") {
 		metadata[xhttp.AmzBucketReplicationStatus] = string(replication.Pending)
 	}
 	// We need to preserve the encryption headers set in EncryptRequest,
@@ -2615,7 +2617,9 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		w.Write(encodedErrorResponse)
 		w.(http.Flusher).Flush()
 	}
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+
+	setEventStreamHeaders(w)
+
 	w = &whiteSpaceWriter{ResponseWriter: w, Flusher: w.(http.Flusher)}
 	completeDoneCh := sendWhiteSpace(w)
 	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, opts)
@@ -2647,8 +2651,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	setPutObjHeaders(w, objInfo, false)
-	if globalBucketReplicationSys.mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		defer replicateObject(ctx, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+	if mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
+		defer replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedCompleteMultipartUpload,
 			BucketName:   bucket,
 			Object:       objInfo,

cmd/peer-rest-client.go

@@ -23,7 +23,6 @@ import (
 	"encoding/gob"
 	"errors"
 	"io"
-	"io/ioutil"
 	"math"
 	"net/url"
 	"strconv"
@@ -122,19 +121,22 @@ type progressReader struct {
 
 func (p *progressReader) Read(b []byte) (int, error) {
 	n, err := p.r.Read(b)
-	if err != nil && err != io.EOF {
-		return n, err
+	if n >= 0 {
+		p.progressChan <- int64(n)
 	}
-	p.progressChan <- int64(n)
 	return n, err
 }
 
+type nullReader struct{}
+
+func (r *nullReader) Read(b []byte) (int, error) {
+	return len(b), nil
+}
+
 func (client *peerRESTClient) doNetOBDTest(ctx context.Context, dataSize int64, threadCount uint) (info madmin.NetOBDInfo, err error) {
 	latencies := []float64{}
 	throughputs := []float64{}
 
-	buf := make([]byte, dataSize)
-
 	buflimiter := make(chan struct{}, threadCount)
 	errChan := make(chan error, threadCount)
 
@@ -163,7 +165,7 @@ func (client *peerRESTClient) doNetOBDTest(ctx context.Context, dataSize int64,
 		}
 	}
 
-	wg := sync.WaitGroup{}
+	var wg sync.WaitGroup
 	finish := func() {
 		<-buflimiter
 		wg.Done()
@@ -183,27 +185,26 @@ func (client *peerRESTClient) doNetOBDTest(ctx context.Context, dataSize int64,
 			}
 
 			go func(i int) {
-				bufReader := bytes.NewReader(buf)
-				bufReadCloser := ioutil.NopCloser(&progressReader{
-					r:            bufReader,
+				progress := &progressReader{
+					r:            io.LimitReader(&nullReader{}, dataSize),
 					progressChan: transferChan,
-				})
+				}
 				start := time.Now()
 				before := atomic.LoadInt64(&totalTransferred)
 
 				ctx, cancel := context.WithTimeout(innerCtx, 10*time.Second)
 				defer cancel()
-				respBody, err := client.callWithContext(ctx, peerRESTMethodNetOBDInfo, nil, bufReadCloser, dataSize)
-				if err != nil {
 
-					if netErr, ok := err.(*rest.NetworkError); ok {
-						if urlErr, ok := netErr.Err.(*url.Error); ok {
-							if urlErr.Err.Error() == context.DeadlineExceeded.Error() {
-								slowSample()
-								finish()
-								return
-							}
-						}
+				// Turn off healthCheckFn for OBD tests to cater for higher load on the peers.
+				clnt := newPeerRESTClient(client.host)
+				clnt.restClient.HealthCheckFn = nil
+
+				respBody, err := clnt.callWithContext(ctx, peerRESTMethodNetOBDInfo, nil, progress, dataSize)
+				if err != nil {
+					if errors.Is(err, context.DeadlineExceeded) {
+						slowSample()
+						finish()
+						return
 					}
 
 					errChan <- err
@@ -329,15 +330,8 @@ func (client *peerRESTClient) NetOBDInfo(ctx context.Context) (info madmin.NetOB
 				continue
 			}
 
-			if netErr, ok := err.(*rest.NetworkError); ok {
-				if urlErr, ok := netErr.Err.(*url.Error); ok {
-					if urlErr.Err.Error() == context.Canceled.Error() {
-						continue
-					}
-					if urlErr.Err.Error() == context.DeadlineExceeded.Error() {
-						continue
-					}
-				}
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				continue
 			}
 		}
 		return info, err
@@ -880,7 +874,7 @@ func newPeerRESTClient(peer *xnet.Host) *peerRESTClient {
 		}
 	}
 
-	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
+	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
 
 	// Construct a new health function.

cmd/peer-rest-server.go

@@ -182,7 +182,7 @@ func (s *peerRESTServer) DeleteServiceAccountHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err := globalIAMSys.DeleteServiceAccount(context.Background(), accessKey); err != nil {
+	if err := globalIAMSys.DeleteServiceAccount(r.Context(), accessKey); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -610,6 +610,10 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
 	if meta.notificationConfig != nil {
 		globalNotificationSys.AddRulesMap(bucketName, meta.notificationConfig.ToRulesMap())
 	}
+
+	if meta.bucketTargetConfig != nil {
+		globalBucketTargetSys.UpdateTarget(bucketName, meta.bucketTargetConfig)
+	}
 }
 
 // ReloadFormatHandler - Reload Format.
@@ -972,7 +976,11 @@ func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *h
 
 	ctx := newContext(r, w, "BackgroundHealStatus")
 
-	state := getLocalBackgroundHealStatus()
+	state, ok := getLocalBackgroundHealStatus()
+	if !ok {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
 
 	defer w.(http.Flusher).Flush()
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(state))

cmd/prepare-storage.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net/http"
@@ -27,7 +28,6 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -185,7 +185,6 @@ func IsServerResolvable(endpoint Endpoint) error {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 
@@ -213,7 +212,10 @@ func IsServerResolvable(endpoint Endpoint) error {
 	}
 	defer httpClient.CloseIdleConnections()
 
-	resp, err := httpClient.Do(req)
+	ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
+	defer cancel()
+
+	resp, err := httpClient.Do(req.WithContext(ctx))
 	if err != nil {
 		return err
 	}
@@ -253,10 +255,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 	formatConfigs, sErrs := loadFormatErasureAll(storageDisks, false)
 	// Check if we have
 	for i, sErr := range sErrs {
-		if _, ok := formatCriticalErrors[sErr]; ok {
-			return nil, nil, config.ErrCorruptedBackend(err).Hint(fmt.Sprintf("Clear any pre-existing content on %s", endpoints[i]))
-		}
-		// not critical error but still print the error, nonetheless, which is perhaps unhandled
+		// print the error, nonetheless, which is perhaps unhandled
 		if sErr != errUnformattedDisk && sErr != errDiskNotFound && retryCount >= 5 {
 			if sErr != nil {
 				logger.Info("Unable to read 'format.json' from %s: %v\n", endpoints[i], sErr)

cmd/rest/client.go

@@ -85,10 +85,20 @@ const (
 	querySep = "?"
 )
 
+type restError string
+
+func (e restError) Error() string {
+	return string(e)
+}
+
+func (e restError) Timeout() bool {
+	return true
+}
+
 // CallWithContext - make a REST call with context.
 func (c *Client) CallWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
 	if !c.IsOnline() {
-		return nil, &NetworkError{Err: errors.New("remote server offline")}
+		return nil, &NetworkError{Err: &url.Error{Op: method, URL: c.url.String(), Err: restError("remote server offline")}}
 	}
 	req, err := http.NewRequest(http.MethodPost, c.url.String()+method+querySep+values.Encode(), body)
 	if err != nil {

cmd/server-main.go

@@ -93,11 +93,16 @@ EXAMPLES:
 `,
 }
 
-// Checks if endpoints are either available through environment
-// or command line, returns false if both fails.
-func endpointsPresent(ctx *cli.Context) bool {
-	endpoints := env.Get(config.EnvEndpoints, strings.Join(ctx.Args(), config.ValueSeparator))
-	return len(endpoints) != 0
+func serverCmdArgs(ctx *cli.Context) []string {
+	v := env.Get(config.EnvArgs, "")
+	if v == "" {
+		// Fall back to older ENV MINIO_ENDPOINTS
+		v = env.Get(config.EnvEndpoints, "")
+	}
+	if v == "" {
+		return ctx.Args()
+	}
+	return strings.Fields(v)
 }
 
 func serverHandleCmdArgs(ctx *cli.Context) {
@@ -106,18 +111,29 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 
 	logger.FatalIf(CheckLocalServerAddr(globalCLIContext.Addr), "Unable to validate passed arguments")
 
-	var setupType SetupType
 	var err error
+	var setupType SetupType
+
+	// Check and load TLS certificates.
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
+	logger.FatalIf(err, "Unable to load the TLS configuration")
+
+	// Check and load Root CAs.
+	globalRootCAs, err = certs.GetRootCAs(globalCertsCADir.Get())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Add the global public crts as part of global root CAs
+	for _, publicCrt := range globalPublicCerts {
+		globalRootCAs.AddCert(publicCrt)
+	}
+
+	// Register root CAs for remote ENVs
+	env.RegisterGlobalCAs(globalRootCAs)
 
 	globalMinioAddr = globalCLIContext.Addr
 
 	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
-	endpoints := strings.Fields(env.Get(config.EnvEndpoints, ""))
-	if len(endpoints) > 0 {
-		globalEndpoints, globalErasureSetDriveCount, setupType, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
-	} else {
-		globalEndpoints, globalErasureSetDriveCount, setupType, err = createServerEndpoints(globalCLIContext.Addr, ctx.Args()...)
-	}
+	globalEndpoints, setupType, err = createServerEndpoints(globalCLIContext.Addr, serverCmdArgs(ctx)...)
 	logger.FatalIf(err, "Invalid command line arguments")
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
@@ -170,7 +186,7 @@ func newAllSubsystems() {
 	globalBucketVersioningSys = NewBucketVersioningSys()
 
 	// Create new bucket replication subsytem
-	globalBucketReplicationSys = NewBucketReplicationSys()
+	globalBucketTargetSys = NewBucketTargetSys()
 }
 
 func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
@@ -210,10 +226,11 @@ func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
 
 	// Enable healing to heal drives if possible
 	if globalIsErasure {
-		initBackgroundHealing(ctx, newObject)
-		initLocalDisksAutoHeal(ctx, newObject)
+		initAutoHeal(ctx, newObject)
 	}
 
+	timeout := newDynamicTimeout(3*time.Second, 3*time.Second)
+
 	// ****  WARNING ****
 	// Migrating to encrypted backend should happen before initialization of any
 	// sub-systems, make sure that we do not move the above codeblock elsewhere.
@@ -229,7 +246,7 @@ func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
 	for range retry.NewTimer(retryCtx) {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
-		if err = txnLk.GetLock(newDynamicTimeout(1*time.Second, 3*time.Second)); err != nil {
+		if err = txnLk.GetLock(timeout); err != nil {
 			logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock")
 			continue
 		}
@@ -340,10 +357,11 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 		return fmt.Errorf("Unable to initialize notification system: %w", err)
 	}
 
-	// Initialize bucket replication sub-system.
-	if err = globalBucketReplicationSys.Init(GlobalContext, buckets, newObject); err != nil {
-		return fmt.Errorf("Unable to initialize bucket replication sub-system: %w", err)
+	// Initialize bucket targets sub-system.
+	if err = globalBucketTargetSys.Init(GlobalContext, buckets, newObject); err != nil {
+		return fmt.Errorf("Unable to initialize bucket target sub-system: %w", err)
 	}
+
 	return nil
 }
 
@@ -369,9 +387,6 @@ func startBackgroundOps(ctx context.Context, objAPI ObjectLayer) {
 
 // serverMain handler called for 'minio server' command.
 func serverMain(ctx *cli.Context) {
-	if ctx.Args().First() == "help" || !endpointsPresent(ctx) {
-		cli.ShowCommandHelpAndExit(ctx, "server", 1)
-	}
 	setDefaultProfilerRates()
 
 	// Initialize globalConsoleSys system
@@ -391,15 +406,7 @@ func serverMain(ctx *cli.Context) {
 	// Initialize all help
 	initHelp()
 
-	// Check and load TLS certificates.
 	var err error
-	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
-	logger.FatalIf(err, "Unable to load the TLS configuration")
-
-	// Check and load Root CAs.
-	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
-	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
-
 	globalProxyEndpoints, err = GetProxyEndpoints(globalEndpoints)
 	logger.FatalIf(err, "Invalid command line arguments")
 

cmd/signals.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"net/http"
 	"os"
 	"strings"
 
@@ -73,7 +75,7 @@ func handleSignals() {
 			if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
 				objAPI.Shutdown(context.Background())
 			}
-			if err != nil {
+			if err != nil && !errors.Is(err, http.ErrServerClosed) {
 				logger.Fatal(err, "Unable to start MinIO server")
 			}
 			exit(true)

cmd/storage-errors.go

@@ -19,10 +19,10 @@ package cmd
 import "os"
 
 // errUnexpected - unexpected error, requires manual intervention.
-var errUnexpected = StorageErr("Unexpected error, please report this issue at https://github.com/minio/minio/issues")
+var errUnexpected = StorageErr("unexpected error, please report this issue at https://github.com/minio/minio/issues")
 
 // errCorruptedFormat - corrupted backend format.
-var errCorruptedFormat = StorageErr("corrupted backend format, please join https://slack.min.io for assistance")
+var errCorruptedFormat = StorageErr("corrupted backend format, specified disk mount has unexpected previous content")
 
 // errUnformattedDisk - unformatted disk found.
 var errUnformattedDisk = StorageErr("unformatted disk found")
@@ -142,5 +142,14 @@ func osErrToFileErr(err error) error {
 	if isSysErrHandleInvalid(err) {
 		return errFileNotFound
 	}
+	if isSysErrIO(err) {
+		return errFaultyDisk
+	}
+	if isSysErrInvalidArg(err) {
+		return errUnsupportedDisk
+	}
+	if isSysErrNoSpace(err) {
+		return errDiskFull
+	}
 	return err
 }

cmd/storage-rest-client.go

@@ -478,7 +478,7 @@ func (client *storageRESTClient) WalkVersions(volume, dirPath, marker string, re
 			if gerr := decoder.Decode(&fi); gerr != nil {
 				// Upon error return
 				if gerr != io.EOF {
-					logger.LogIf(context.Background(), gerr)
+					logger.LogIf(GlobalContext, gerr)
 				}
 				return
 			}
@@ -663,11 +663,10 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 
-	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
+	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
 	restClient.HealthCheckInterval = 500 * time.Millisecond
 	restClient.HealthCheckFn = func() bool {

cmd/storage-rest-server.go

@@ -151,7 +151,8 @@ func (s *storageRESTServer) CrawlAndGetDataUsageHandler(w http.ResponseWriter, r
 		return
 	}
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
+
 	var cache dataUsageCache
 	err := cache.deserialize(r.Body)
 	if err != nil {
@@ -520,7 +521,7 @@ func (s *storageRESTServer) WalkSplunkHandler(w http.ResponseWriter, r *http.Req
 	dirPath := vars[storageRESTDirPath]
 	markerPath := vars[storageRESTMarkerPath]
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 	encoder := gob.NewEncoder(w)
 
 	fch, err := s.storage.WalkSplunk(volume, dirPath, markerPath, r.Context().Done())
@@ -548,7 +549,7 @@ func (s *storageRESTServer) WalkVersionsHandler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 	encoder := gob.NewEncoder(w)
 
 	fch, err := s.storage.WalkVersions(volume, dirPath, markerPath, recursive, r.Context().Done())
@@ -576,7 +577,7 @@ func (s *storageRESTServer) WalkHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 	encoder := gob.NewEncoder(w)
 
 	fch, err := s.storage.Walk(volume, dirPath, markerPath, recursive, r.Context().Done())
@@ -659,7 +660,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
 
 	dErrsResp := &DeleteVersionsErrsResp{Errs: make([]error, totalVersions)}
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 	encoder := gob.NewEncoder(w)
 	done := keepHTTPResponseAlive(w)
 	errs := s.storage.DeleteVersions(volume, versions)
@@ -805,7 +806,7 @@ func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 	encoder := gob.NewEncoder(w)
 	done := keepHTTPResponseAlive(w)
 	err = s.storage.VerifyFile(volume, filePath, fi)