fix: infinite loop in cleanupStaleUploads of encrypted MPUs (#10845)

fixes #10588

.github/stale.yml

@@ -14,6 +14,7 @@ onlyLabels: []
 exemptLabels:
   - "security"
   - "pending discussion"
+  - "do not close"
 
 # Set to true to ignore issues in a project (defaults to false)
 exemptProjects: false

.github/workflows/codeql.yml

@@ -1,51 +0,0 @@
-name: "Code scanning - action"
-
-on:
-  push:
-  pull_request:
-  schedule:
-    - cron: '0 19 * * 0'
-
-jobs:
-  CodeQL-Build:
-
-    # CodeQL runs on ubuntu-latest and windows-latest
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-      
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-         languages: go, javascript
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/go.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
     - master
-    - release
 
 jobs:
   build:
@@ -12,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.13.x]
+        go-version: [1.14.x, 1.15.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2
@@ -38,7 +37,12 @@ jobs:
           GO111MODULE: on
           MINIO_CI_CD: 1
         run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           sudo apt-get install devscripts shellcheck
+          nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
+          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy
+          go list -m all | ./nancy sleuth
           make
           diff -au <(gofmt -s -d cmd) <(printf "")
           diff -au <(gofmt -s -d pkg) <(printf "")

.goreleaser.yml

@@ -114,11 +114,11 @@ checksum:
 
 signs:
   -
-    signature: "${artifact}.asc"
+    signature: "${artifact}.minisig"
     cmd: "sh"
     args:
       - '-c'
-      - 'gpg --quiet --detach-sign -a ${artifact}'
+      - 'minisign -s /media/${USER}/minio/minisign.key -qQSm ${artifact} < /media/${USER}/minio/minisign-passphrase'
     artifacts: all
 
 changelog:

CREDITS

@@ -10293,25 +10293,6 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
       the Mozilla Public License, v. 2.0.
 
 
-================================================================
-
-github.com/inconshreveable/go-update
-https://github.com/inconshreveable/go-update
-----------------------------------------------------------------
-Copyright 2015 Alan Shreve
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
 ================================================================
 
 github.com/jcmturner/gofork
@@ -11470,8 +11451,216 @@ https://github.com/minio/md5-simd
 
 ================================================================
 
-github.com/minio/minio-go/v6
-https://github.com/minio/minio-go/v6
+github.com/minio/minio-go/v7
+https://github.com/minio/minio-go/v7
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
+github.com/minio/selfupdate
+https://github.com/minio/selfupdate
 ----------------------------------------------------------------
 
                                  Apache License

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.15-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
@@ -13,11 +13,11 @@ RUN  \
 
 FROM alpine:3.12
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 EXPOSE 9000
 

Dockerfile.arm.release

@@ -5,20 +5,25 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio -o /usr/bin/minio && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-arm/minio.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
+     chmod +x /usr/bin/verify-minio.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS
 
 EXPOSE 9000
 

Dockerfile.arm64.release

@@ -5,20 +5,26 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio -o /usr/bin/minio && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-arm64/minio.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
+     chmod +x /usr/bin/verify-minio.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
+     /usr/bin/verify-minio.sh
 
 EXPOSE 9000
 

Dockerfile.ppc64le.release

@@ -5,20 +5,25 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
      curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio -o /usr/bin/minio && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-ppc64le/minio.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
+     chmod +x /usr/bin/verify-minio.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
+     /usr/bin/verify-minio.sh
 
 EXPOSE 9000
 

Dockerfile.release

@@ -2,20 +2,26 @@ FROM alpine:3.12
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/bin/minio && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-amd64/minio.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
+     chmod +x /usr/bin/verify-minio.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
+     /usr/bin/verify-minio.sh
 
 EXPOSE 9000
 

Dockerfile.s390x.release

@@ -5,20 +5,25 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
+     apk update && apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' minisign && \
      curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio -o /usr/bin/minio && \
-     curl -s -q https://raw.githubusercontent.com/minio/minio/release/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio.sha256sum -o /usr/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-s390x/minio.minisig -o /usr/bin/minio.minisig && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/verify-minio.sh -o /usr/bin/verify-minio.sh && \
+     curl -s -q https://raw.githubusercontent.com/minio/minio/master/dockerscripts/docker-entrypoint.sh -o /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     curl -s -q -O https://raw.githubusercontent.com/minio/minio/release/CREDITS
+     chmod +x /usr/bin/verify-minio.sh && \
+     curl -s -q -O https://raw.githubusercontent.com/minio/minio/master/CREDITS && \
+     /usr/bin/verify-minio.sh
 
 EXPOSE 9000
 

Makefile

@@ -17,11 +17,18 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
+	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...)
+	@which msgp 1>/dev/null || (echo "Installing msgp" && GO111MODULE=off go get github.com/tinylib/msgp)
+	@which stringer 1>/dev/null || (echo "Installing stringer" && GO111MODULE=off go get golang.org/x/tools/cmd/stringer)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps fmt lint
+verifiers: getdeps fmt lint ruleguard check-gen
+
+check-gen:
+	@go generate ./... >/dev/null
+	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
 fmt:
 	@echo "Running $@ check"
@@ -33,6 +40,10 @@ lint:
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
 
+ruleguard:
+	@echo "Running $@ check"
+	@${GOPATH}/bin/ruleguard -rules ruleguard.rules.go github.com/minio/minio/...
+
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build

README.md

@@ -88,7 +88,7 @@ service minio start
 ```
 
 ## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.13](https://golang.org/dl/#stable)
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio
@@ -177,7 +177,7 @@ mc admin update <minio alias, e.g., myminio>
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
 - `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
 - In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
-- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)x
+- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
 - If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
 - If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
@@ -193,4 +193,4 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 ## License
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fminio%2Fminio.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fminio%2Fminio?ref=badge_large)
+Use of MinIO is governed by the Apache 2.0 License found at [LICENSE](https://github.com/minio/minio/blob/master/LICENSE).

README_zh_CN.md

@@ -7,29 +7,33 @@ MinIO是一个非常轻量的服务,可以很简单的和其他应用的结合
 ## Docker 容器
 ### 稳定版
 ```
-docker pull minio/minio
-docker run -p 9000:9000 minio/minio server /data
+docker run -p 9000:9000 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  minio/minio server /data
 ```
 
 ### 尝鲜版
 ```
-docker pull minio/minio:edge
-docker run -p 9000:9000 minio/minio:edge server /data
+docker run -p 9000:9000 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  minio/minio:edge server /data
 ```
-更多Docker部署信息请访问 [这里](https://docs.min.io/docs/minio-docker-quickstart-guide)
+
+> 提示：除非你通过`-it`(TTY交互)参数启动容器，否则Docker将不会显示默认的密钥。一般情况下，并不推荐使用容器的默认密钥，更多Docker部署信息请访问 [这里](https://docs.min.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
-### Homebrew
+### Homebrew（推荐）
 使用 [Homebrew](http://brew.sh/)安装minio
 
 ```sh
 brew install minio/stable/minio
 minio server /data
 ```
-#### Note
-如果你之前使用 `brew install minio`安装过minio, 可以用 `minio/stable/minio` 官方镜像进行重装. 由于golang 1.8的bug,homebrew版本不太稳定。
 
-```
+> 提示：如果你之前使用 `brew install minio`安装过minio, 可以用 `minio/stable/minio` 官方镜像进行重装. 由于golang 1.8的bug,homebrew版本不太稳定。
+```sh
 brew uninstall minio
 brew install minio/stable/minio
 ```
@@ -49,6 +53,16 @@ chmod 755 minio
 | ---------- | --------     | ------                                                     |
 | GNU/Linux  | 64-bit Intel | https://dl.min.io/server/minio/release/linux-amd64/minio |
 ```sh
+wget https://dl.min.io/server/minio/release/linux-amd64/minio
+chmod +x minio
+./minio server /data
+```
+
+| 操作系统    | CPU架构      | 地址                                                        |
+| ---------- | --------     | ------                                                     |
+| GNU/Linux  | ppc64le      | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
+```sh
+wget https://dl.min.io/server/minio/release/linux-ppc64le/minio
 chmod +x minio
 ./minio server /data
 ```
@@ -64,7 +78,7 @@ minio.exe server D:\Photos
 
 ## FreeBSD
 ### Port
-使用 [pkg](https://github.com/freebsd/pkg)进行安装。
+使用 [pkg](https://github.com/freebsd/pkg)进行安装，, MinIO官方并没有提供FreeBSD二进制文件， 它由FreeBSD上游维护，点击 [这里](https://www.freshports.org/www/minio)查看。
 
 ```sh
 pkg install minio
@@ -75,14 +89,68 @@ service minio start
 
 ## 使用源码安装
 
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install).
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.14](https://golang.org/dl/#stable)
 
 ```sh
-go get -u github.com/minio/minio
+GO111MODULE=on go get github.com/minio/minio
+```
+
+## 为防火墙设置允许访问的端口
+
+默认情况下，MinIO 使用端口9000来侦听传入的连接。如果你的平台默认阻止了该端口，则需要启用对该端口的访问。
+
+### ufw
+
+对于启用了ufw的主机（基于Debian的发行版）, 你可以通过`ufw`命令允许指定端口上的所有流量连接. 通过如下命令允许访问端口9000
+
+```sh
+ufw allow 9000
+```
+
+如下命令允许端口9000-9010上的所有传入流量。
+
+```sh
+ufw allow 9000:9010/tcp
+```
+
+### firewall-cmd
+
+对于启用了firewall-cmd的主机（CentOS）, 你可以通过`firewall-cmd`命令允许指定端口上的所有流量连接。 通过如下命令允许访问端口9000
+
+```sh
+firewall-cmd --get-active-zones
+```
+
+这个命令获取当前正在使用的区域。 现在，就可以为以上返回的区域应用端口规则了。 假如返回的区域是 `public`, 使用如下命令
+
+```sh
+firewall-cmd --zone=public --add-port=9000/tcp --permanent
+```
+
+这里的`permanent`参数表示持久化存储规则，可用于防火墙启动、重启和重新加载。 最后，需要防火墙重新加载，让我们刚刚的修改生效。
+
+```sh
+firewall-cmd --reload
+```
+
+### iptables
+
+对于启用了iptables的主机（RHEL, CentOS, etc）, 你可以通过`iptables`命令允许指定端口上的所有流量连接。 通过如下命令允许访问端口9000
+
+```sh
+iptables -A INPUT -p tcp --dport 9000 -j ACCEPT
+service iptables restart
+```
+
+如下命令允许端口9000-9010上的所有传入流量。
+
+```sh
+iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
+service iptables restart
 ```
 
 ## 使用MinIO浏览器进行验证
-安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
+MinIO Server带有一个嵌入的Web对象浏览器，安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
 
 ![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.png?raw=true)
 
@@ -94,6 +162,25 @@ go get -u github.com/minio/minio
 
 上述描述对所有网关后端同样有效。
 
+## 升级 MinIO
+MinIO 服务端支持滚动升级, 也就是说你可以一次更新分布式集群中的一个MinIO实例。 这样可以在不停机的情况下进行升级。可以通过将二进制文件替换为最新版本并以滚动方式重新启动所有服务器来手动完成升级。但是, 我们建议所有用户从客户端使用 [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) 命令升级。 这将同时更新集群中的所有节点并重新启动它们, 如下命令所示:
+
+```
+mc admin update <minio alias, e.g., myminio>
+```
+
+> 注意: 有些发行版可能不允许滚动升级，这通常在发行说明中提到，所以建议在升级之前阅读发行说明。在这种情况下，建议使用`mc admin update`升级机制来一次升级所有服务器。
+
+### MinIO升级时要记住的重要事项
+
+- `mc admin update` 命令仅当运行MinIO的用户对二进制文件所在的父目录具有写权限时才工作, 比如当前二进制文件位于`/usr/local/bin/minio`, 你需要具备`/usr/local/bin`目录的写权限.
+- `mc admin update` 命令同时更新并重新启动所有服务器，应用程序将在升级后重试并继续各自的操作。
+- `mc admin update` 命令在 kubernetes/container 环境下是不能用的, 容器环境提供了它自己的更新机制来更新。
+- 对于联盟部署模式，应分别针对每个群集运行`mc admin update`。 在成功更新所有群集之前，不要将`mc`更新为任何新版本。
+- 如果将`kes`用作MinIO的KMS，只需替换二进制文件并重新启动`kes`，可以在 [这里](https://github.com/minio/kes/wiki) 找到有关`kes`的更多信息。
+- 如果将Vault作为MinIO的KMS，请确保已遵循如下Vault升级过程的概述：https://www.vaultproject.io/docs/upgrading/index.html
+- 如果将MindIO与etcd配合使用, 请确保已遵循如下etcd升级过程的概述: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
+
 ## 了解更多
 - [MinIO纠删码入门](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [`mc`快速入门](https://docs.min.io/docs/minio-client-quickstart-guide)
@@ -104,3 +191,6 @@ go get -u github.com/minio/minio
 
 ## 如何参与到MinIO项目
 请参考 [贡献者指南](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)。欢迎各位中国程序员加到MinIO项目中。
+
+## 授权许可
+MinIO的使用受 Apache 2.0 License 约束，你可以在 [LICENSE](./LICENSE) 查看许可。

browser/app/js/browser/BrowserDropdown.js

@@ -57,22 +57,6 @@ export class BrowserDropdown extends React.Component {
     const { fetchServerInfo } = this.props
     fetchServerInfo()
   }
-  fullScreen(e) {
-    e.preventDefault()
-    let el = document.documentElement
-    if (el.requestFullscreen) {
-      el.requestFullscreen()
-    }
-    if (el.mozRequestFullScreen) {
-      el.mozRequestFullScreen()
-    }
-    if (el.webkitRequestFullscreen) {
-      el.webkitRequestFullscreen()
-    }
-    if (el.msRequestFullscreen) {
-      el.msRequestFullscreen()
-    }
-  }
   logout(e) {
     e.preventDefault()
     web.Logout()
@@ -87,24 +71,30 @@ export class BrowserDropdown extends React.Component {
             <i className="fas fa-bars" />
           </Dropdown.Toggle>
           <Dropdown.Menu className="dropdown-menu-right">
+            <li>
+              <a href="" onClick={this.showChangePassword.bind(this)}>
+                Change Password <i className="fas fa-cog" />
+              </a>
+              {this.state.showChangePasswordModal && (
+                <ChangePasswordModal
+                  serverInfo={serverInfo}
+                  hideChangePassword={this.hideChangePassword.bind(this)}
+                />
+              )}
+            </li>
+            <li>
+              <a target="_blank" href="https://docs.min.io/?ref=ob">
+                Documentation <i className="fas fa-book" />
+              </a>
+            </li>
             <li>
               <a target="_blank" href="https://github.com/minio/minio">
                 GitHub <i className="fab fa-github" />
               </a>
             </li>
             <li>
-              <a href="" onClick={this.fullScreen}>
-                Fullscreen <i className="fas fa-expand" />
-              </a>
-            </li>
-            <li>
-              <a target="_blank" href="https://docs.min.io/">
-                Documentation <i className="fas fa-book" />
-              </a>
-            </li>
-            <li>
-              <a target="_blank" href="https://slack.min.io">
-                Ask for help <i className="fas fa-question-circle" />
+              <a target="_blank" href="https://min.io/pricing?ref=ob">
+                Get Support <i className="fas fa-question-circle" />
               </a>
             </li>
             <li>
@@ -118,20 +108,9 @@ export class BrowserDropdown extends React.Component {
                 />
               )}
             </li>
-            <li>
-              <a href="" onClick={this.showChangePassword.bind(this)}>
-                Change Password <i className="fas fa-cog" />
-              </a>
-              {this.state.showChangePasswordModal && (
-                <ChangePasswordModal
-                  serverInfo={serverInfo}
-                  hideChangePassword={this.hideChangePassword.bind(this)}
-                />
-              )}
-            </li>
             <li>
               <a href="" id="logout" onClick={this.logout}>
-                Sign Out <i className="fas fa-sign-out-alt" />
+                Logout <i className="fas fa-sign-out-alt" />
               </a>
             </li>
           </Dropdown.Menu>

browser/app/js/browser/Header.js

@@ -15,6 +15,7 @@
  */
 
 import React from "react"
+import ObjectsSearch from "../objects/ObjectsSearch"
 import Path from "../objects/Path"
 import StorageInfo from "./StorageInfo"
 import BrowserDropdown from "./BrowserDropdown"
@@ -27,6 +28,7 @@ export const Header = () => {
     <header className="fe-header">
       <Path />
       {loggedIn && <StorageInfo />}
+      {loggedIn && <ObjectsSearch />}
       <ul className="feh-actions">
         {loggedIn ? (
           <BrowserDropdown />

browser/app/js/buckets/selectors.js

@@ -22,7 +22,8 @@ const bucketsFilterSelector = state => state.buckets.filter
 export const getFilteredBuckets = createSelector(
   bucketsSelector,
   bucketsFilterSelector,
-  (buckets, filter) => buckets.filter(bucket => bucket.indexOf(filter) > -1)
+  (buckets, filter) => buckets.filter(
+    bucket => bucket.toLowerCase().indexOf(filter.toLowerCase()) > -1)
 )
 
 export const getCurrentBucket = state => state.buckets.currentBucket

browser/app/js/objects/ObjectsListContainer.js

@@ -18,6 +18,7 @@ import React from "react"
 import { connect } from "react-redux"
 import InfiniteScroll from "react-infinite-scroller"
 import ObjectsList from "./ObjectsList"
+import { getFilteredObjects } from "./selectors"
 
 export class ObjectsListContainer extends React.Component {
   constructor(props) {
@@ -39,22 +40,29 @@ export class ObjectsListContainer extends React.Component {
       })
     }
   }
+  componentDidUpdate(prevProps) {
+    if (this.props.filter !== prevProps.filter) {
+      this.setState({
+        page: 1
+      })
+    }
+  }
   loadNextPage() {
     this.setState(state => {
       return { page: state.page + 1 }
     })
   }
   render() {
-    const { objects, listLoading } = this.props
+    const { filteredObjects, listLoading } = this.props
 
-    const visibleObjects = objects.slice(0, this.state.page * 100)
+    const visibleObjects = filteredObjects.slice(0, this.state.page * 100)
 
     return (
       <div style={{ position: "relative" }}>
         <InfiniteScroll
           pageStart={0}
           loadMore={this.loadNextPage}
-          hasMore={objects.length > visibleObjects.length}
+          hasMore={filteredObjects.length > visibleObjects.length}
           useWindow={true}
           initialLoad={false}
         >
@@ -70,7 +78,8 @@ const mapStateToProps = state => {
   return {
     currentBucket: state.buckets.currentBucket,
     currentPrefix: state.objects.currentPrefix,
-    objects: state.objects.list,
+    filteredObjects: getFilteredObjects(state),
+    filter: state.objects.filter,
     sortBy: state.objects.sortBy,
     sortOrder: state.objects.sortOrder,
     listLoading: state.objects.listLoading

browser/app/js/objects/ObjectsSearch.js

@@ -0,0 +1,43 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { connect } from "react-redux"
+import * as actionsObjects from "./actions"
+
+export const ObjectsSearch = ({ onChange }) => (
+  <div
+    className="input-group ig-left ig-search-dark"
+    style={{ display: "block" }}
+  >
+    <input
+      className="ig-text"
+      type="input"
+      placeholder="Search Objects..."
+      onChange={e => onChange(e.target.value)}
+    />
+    <i className="ig-helpers" />
+  </div>
+)
+
+const mapDispatchToProps = dispatch => {
+  return {
+    onChange: filter =>
+      dispatch(actionsObjects.setFilter(filter))
+  }
+}
+
+export default connect(undefined, mapDispatchToProps)(ObjectsSearch)

browser/app/js/objects/PreviewObjectModal.js

@@ -46,7 +46,7 @@ class PreviewObjectModal extends React.Component {
         <ModalBody>
           <div className="input-group">
             {this.state.url && (
-              <object data={this.state.url}>
+              <object data={this.state.url} style={{ display: "block", width: "100%" }}>
                 <h3 style={{ textAlign: "center", display: "block", width: "100%" }}>
                   Do not have read permissions to preview "{this.props.object.name}"
                 </h3>

browser/app/js/objects/__tests__/ObjectsListContainer.test.js

@@ -20,13 +20,13 @@ import { ObjectsListContainer } from "../ObjectsListContainer"
 
 describe("ObjectsList", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsListContainer objects={[]} />)
+    shallow(<ObjectsListContainer filteredObjects={[]} />)
   })
 
   it("should render ObjectsList with objects", () => {
     const wrapper = shallow(
       <ObjectsListContainer
-        objects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
+        filteredObjects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
       />
     )
     expect(wrapper.find("ObjectsList").length).toBe(1)
@@ -40,7 +40,7 @@ describe("ObjectsList", () => {
     const wrapper = shallow(
       <ObjectsListContainer
         currentBucket="test1"
-        objects={[]}
+        filteredObjects={[]}
         listLoading={true}
       />
     )

browser/app/js/objects/__tests__/ObjectsSearch.test.js

@@ -0,0 +1,32 @@
+/*
+ * MinIO Cloud Storage (C) 2018 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { shallow } from "enzyme"
+import { ObjectsSearch } from "../ObjectsSearch"
+
+describe("ObjectsSearch", () => {
+  it("should render without crashing", () => {
+    shallow(<ObjectsSearch />)
+  })
+
+  it("should call onChange with search text", () => {
+    const onChange = jest.fn()
+    const wrapper = shallow(<ObjectsSearch onChange={onChange} />)
+    wrapper.find("input").simulate("change", { target: { value: "test" } })
+    expect(onChange).toHaveBeenCalledWith("test")
+  })
+})

browser/app/js/objects/__tests__/reducer.test.js

@@ -23,6 +23,7 @@ describe("objects reducer", () => {
     const initialState = reducer(undefined, {})
     expect(initialState).toEqual({
       list: [],
+      filter: "",
       listLoading: false,
       sortBy: "",
       sortOrder: SORT_ORDER_ASC,

browser/app/js/objects/actions.js

@@ -36,6 +36,7 @@ import { getServerInfo, hasServerPublicDomain } from '../browser/selectors'
 
 export const SET_LIST = "objects/SET_LIST"
 export const RESET_LIST = "objects/RESET_LIST"
+export const SET_FILTER = "objects/SET_FILTER"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -57,6 +58,13 @@ export const resetList = () => ({
   type: RESET_LIST,
 })
 
+export const setFilter = filter => {
+  return {
+    type: SET_FILTER,
+    filter
+  }
+}
+
 export const setListLoading = (listLoading) => ({
   type: SET_LIST_LOADING,
   listLoading,

browser/app/js/objects/reducer.js

@@ -28,6 +28,7 @@ const removeObject = (list, objectToRemove, lookup) => {
 export default (
   state = {
     list: [],
+    filter: "",
     listLoading: false,
     sortBy: "",
     sortOrder: SORT_ORDER_ASC,
@@ -53,6 +54,11 @@ export default (
         ...state,
         list: []
       }
+    case actionsObjects.SET_FILTER:
+      return {
+        ...state,
+        filter: action.filter
+      }
     case actionsObjects.SET_LIST_LOADING:
       return {
         ...state,

browser/app/js/objects/selectors.js

@@ -21,3 +21,13 @@ export const getCurrentPrefix = state => state.objects.currentPrefix
 export const getCheckedList = state => state.objects.checkedList
 
 export const getPrefixWritable = state => state.objects.prefixWritable
+
+const objectsSelector = state => state.objects.list
+const objectsFilterSelector = state => state.objects.filter
+
+export const getFilteredObjects = createSelector(
+  objectsSelector,
+  objectsFilterSelector,
+  (objects, filter) => objects.filter(
+    object => object.name.toLowerCase().startsWith(filter.toLowerCase()))
+)

browser/app/js/uploads/Dropzone.js

@@ -36,7 +36,7 @@ export class Dropzone extends React.Component {
     // Overwrite the default styling from react-dropzone; otherwise it
     // won't handle child elements correctly.
     const style = {
-      height: "100%",
+      flex: "1",
       borderWidth: "0",
       borderStyle: "dashed",
       borderColor: "#fff"

browser/app/less/inc/file-explorer.less

@@ -20,7 +20,8 @@
 	@media(max-width: @screen-sm-max) {
 		padding: 75px 0 80px;
 	}
-
+  display: flex;
+  flex-direction: column;
 	min-height:100vh;
 	overflow: auto;
 }

browser/app/less/inc/form.less

@@ -169,6 +169,24 @@ select.form-control {
     }
 }
 
+.ig-search-dark {
+    &:before {
+        font-family: @font-family-icon;
+        font-weight: 900;
+        content: '\f002';
+        font-size: 15px;
+        position: absolute;
+        left: 2px;
+        top: 8px;
+        color: rgba(0, 0, 0, 0.5);
+    }
+
+    .ig-text {
+        padding-left: 25px;
+        .placeholder(rgba(0, 0, 0, 0.5))
+    }
+}
+
 .ig-search {
     &:before {
         font-family: @font-family-icon;
@@ -270,4 +288,4 @@ select.form-control {
 .set-expire-decrease {
     bottom: -27px;
     .rotate(-180deg);
-}
+}

browser/app/less/inc/list.less

@@ -349,6 +349,7 @@ div.fesl-row {
         margin: 0;
         height: 100%;
         text-align: right;
+        white-space: nowrap;
     }
 
     .dropdown {
@@ -495,4 +496,4 @@ div.fesl-row {
         .opacity(1);
         right: 0;
     }
-}
+}

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
 }
 
 function _build() {

buildscripts/verify-build.sh

@@ -45,88 +45,63 @@ FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
 function start_minio_fs()
 {
     "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
-    minio_pid=$!
     sleep 10
-
-    echo "$minio_pid"
 }
 
 function start_minio_erasure()
 {
     "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
-    minio_pid=$!
     sleep 15
-
-    echo "$minio_pid"
 }
 
 function start_minio_erasure_sets()
 {
     "${MINIO[@]}" server "${WORK_DIR}/erasure-disk-sets{1...32}" >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
-    minio_pid=$!
     sleep 15
-
-    echo "$minio_pid"
 }
 
 function start_minio_zone_erasure_sets()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
 
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function start_minio_zone_erasure_sets_ipv6()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
 
-    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-
-    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
+    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9001.log" 2>&1 &
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function start_minio_dist_erasure()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
     "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
-    minio_pids[2]=$!
     "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
-    minio_pids[3]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function run_test_fs()
 {
-    minio_pid="$(start_minio_fs)"
+    start_minio_fs
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -138,12 +113,12 @@ function run_test_fs()
 }
 
 function run_test_erasure_sets() {
-    minio_pid="$(start_minio_erasure_sets)"
+    start_minio_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -156,14 +131,12 @@ function run_test_erasure_sets() {
 
 function run_test_zone_erasure_sets()
 {
-    minio_pids=( $(start_minio_zone_erasure_sets) )
+    start_minio_zone_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -182,16 +155,14 @@ function run_test_zone_erasure_sets()
 
 function run_test_zone_erasure_sets_ipv6()
 {
-    minio_pids=( $(start_minio_zone_erasure_sets_ipv6) )
+    start_minio_zone_erasure_sets_ipv6
 
     export SERVER_ENDPOINT="[::1]:9000"
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -210,12 +181,12 @@ function run_test_zone_erasure_sets_ipv6()
 
 function run_test_erasure()
 {
-    minio_pid="$(start_minio_erasure)"
+    start_minio_erasure
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -228,14 +199,12 @@ function run_test_erasure()
 
 function run_test_dist_erasure()
 {
-    minio_pids=( $(start_minio_dist_erasure) )
+    start_minio_dist_erasure
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then

cmd/admin-bucket-handlers.go

@@ -0,0 +1,296 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+const (
+	bucketQuotaConfigFile = "quota.json"
+	bucketTargetsFile     = "bucket-targets.json"
+)
+
+// PutBucketQuotaConfigHandler - PUT Bucket quota configuration.
+// ----------
+// Places a quota configuration on the specified bucket. The quota
+// specified in the quota configuration will be applied by default
+// to enforce total quota for the specified bucket.
+func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketQuotaConfig")
+
+	defer logger.AuditLog(w, r, "PutBucketQuotaConfig", mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Turn off quota commands if data usage info is unavailable.
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminBucketQuotaDisabled), r.URL)
+		return
+	}
+
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if _, err = parseBucketQuota(bucket, data); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, data); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketQuotaConfigHandler - gets bucket quota configuration
+func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketQuotaConfig")
+
+	defer logger.AuditLog(w, r, "GetBucketQuotaConfig", mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	config, err := globalBucketMetadataSys.GetQuotaConfig(bucket)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	configData, err := json.Marshal(config)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseJSON(w, configData)
+}
+
+// SetRemoteTargetHandler - sets a remote target for bucket
+func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetBucketTarget")
+
+	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	password := cred.SecretKey
+
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	var target madmin.BucketTarget
+	if err = json.Unmarshal(reqBytes, &target); err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	host, port, err := net.SplitHostPort(target.Endpoint)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	sameTarget, _ := isLocalHost(host, port, globalMinioPort)
+	if sameTarget && bucket == target.TargetBucket {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
+		return
+	}
+	target.SourceBucket = bucket
+	target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
+	if target.Arn == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+
+	}
+	tgtBytes, err := json.Marshal(&targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	data, err := json.Marshal(target.Arn)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}
+
+// ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target
+// for a particular ARN type
+func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListBucketTargets")
+
+	defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r))
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	arnType := vars["type"]
+
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketTargetAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if bucket != "" {
+		// Check if bucket exists.
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		if _, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+	targets := globalBucketTargetSys.ListTargets(ctx, bucket, arnType)
+	data, err := json.Marshal(targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}
+
+// RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN
+func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "RemoveBucketTarget")
+
+	defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r))
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	arn := vars["arn"]
+
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, arn); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	tgtBytes, err := json.Marshal(&targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessNoContent(w)
+}

cmd/admin-handlers-config-kv.go

@@ -42,7 +42,7 @@ import (
 
 func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return auth.Credentials{}, nil
@@ -136,7 +136,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -271,7 +271,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -383,7 +383,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = validateConfig(cfg); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}

cmd/admin-handlers-users.go

@@ -35,8 +35,8 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	var adminAPIErr APIErrorCode
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
 	}
@@ -386,8 +386,8 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -465,8 +465,8 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -520,8 +520,8 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -579,8 +579,8 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -722,7 +722,10 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	json.NewEncoder(w).Encode(policy)
+	if err = json.NewEncoder(w).Encode(policy); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 	w.(http.Flusher).Flush()
 }
 

cmd/admin-handlers.go

@@ -35,9 +35,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/notify"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -54,43 +52,22 @@ const (
 	maxEConfigJSONSize = 262272
 )
 
-// Type-safe query params.
-type mgmtQueryKey string
-
 // Only valid query params for mgmt admin APIs.
 const (
-	mgmtBucket      mgmtQueryKey = "bucket"
-	mgmtPrefix                   = "prefix"
-	mgmtClientToken              = "clientToken"
-	mgmtForceStart               = "forceStart"
-	mgmtForceStop                = "forceStop"
+	mgmtBucket      = "bucket"
+	mgmtPrefix      = "prefix"
+	mgmtClientToken = "clientToken"
+	mgmtForceStart  = "forceStart"
+	mgmtForceStop   = "forceStop"
 )
 
-func updateServer(updateURL, sha256Hex string, latestReleaseTime time.Time) (us madmin.ServerUpdateStatus, err error) {
-	minioMode := getMinioMode()
-	// No inputs provided we should try to update using the default URL.
-	if updateURL == "" && sha256Hex == "" && latestReleaseTime.IsZero() {
-		var updateMsg string
-		updateMsg, sha256Hex, _, latestReleaseTime, err = getUpdateInfo(updateTimeout, minioMode)
-		if err != nil {
-			return us, err
-		}
-		if updateMsg == "" {
-			us.CurrentVersion = Version
-			us.UpdatedVersion = Version
-			return us, nil
-		}
-		if runtime.GOOS == "windows" {
-			updateURL = minioReleaseURL + "minio.exe"
-		} else {
-			updateURL = minioReleaseURL + "minio"
-		}
-	}
-	if err = doUpdate(updateURL, sha256Hex, minioMode); err != nil {
+func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, mode string) (us madmin.ServerUpdateStatus, err error) {
+	if err = doUpdate(u, lrTime, sha256Sum, mode); err != nil {
 		return us, err
 	}
+
 	us.CurrentVersion = Version
-	us.UpdatedVersion = latestReleaseTime.Format(minioReleaseTagTimeLayout)
+	us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout)
 	return us, nil
 }
 
@@ -108,54 +85,77 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	if globalInplaceUpdateDisabled {
-		// if MINIO_UPDATE=off - inplace update is disabled, mostly
-		// in containers.
+		// if MINIO_UPDATE=off - inplace update is disabled, mostly in containers.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
 	}
 
 	vars := mux.Vars(r)
-	updateURL := vars[peerRESTUpdateURL]
+	updateURL := vars["updateURL"]
 	mode := getMinioMode()
-	var sha256Hex string
-	var latestReleaseTime time.Time
-	if updateURL != "" {
-		u, err := url.Parse(updateURL)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
+	if updateURL == "" {
+		updateURL = minioReleaseInfoURL
+		if runtime.GOOS == globalWindowsOSName {
+			updateURL = minioReleaseWindowsInfoURL
 		}
-
-		content, err := downloadReleaseURL(updateURL, updateTimeout, mode)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		sha256Hex, latestReleaseTime, err = parseReleaseData(content)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		if runtime.GOOS == "windows" {
-			u.Path = path.Dir(u.Path) + SlashSeparator + "minio.exe"
-		} else {
-			u.Path = path.Dir(u.Path) + SlashSeparator + "minio"
-		}
-
-		updateURL = u.String()
 	}
 
-	for _, nerr := range globalNotificationSys.ServerUpdate(updateURL, sha256Hex, latestReleaseTime) {
+	u, err := url.Parse(updateURL)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	content, err := downloadReleaseURL(u, updateTimeout, mode)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	sha256Sum, lrTime, err := parseReleaseData(content)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	u.Path = path.Dir(u.Path) + SlashSeparator + "minio.RELEASE." + lrTime.Format(minioReleaseTagTimeLayout)
+
+	crTime, err := GetCurrentReleaseTime()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if lrTime.Sub(crTime) < 0 {
+		updateStatus := madmin.ServerUpdateStatus{
+			CurrentVersion: Version,
+			UpdatedVersion: Version,
+		}
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(updateStatus)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		writeSuccessResponseJSON(w, jsonBytes)
+		return
+	}
+
+	for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
+			err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", nerr.Err)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
 
-	updateStatus, err := updateServer(updateURL, sha256Hex, latestReleaseTime)
+	updateStatus, err := updateServer(u, sha256Sum, lrTime, mode)
 	if err != nil {
+		err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -169,10 +169,15 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	writeSuccessResponseJSON(w, jsonBytes)
 
-	if updateStatus.CurrentVersion != updateStatus.UpdatedVersion {
-		// We did upgrade - restart all services.
-		globalServiceSignalCh <- serviceRestart
+	// Notify all other MinIO peers signal service.
+	for _, nerr := range globalNotificationSys.SignalService(serviceRestart) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
 	}
+
+	globalServiceSignalCh <- serviceRestart
 }
 
 // ServiceHandler - POST /minio/admin/v3/service?action={action}
@@ -287,6 +292,20 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	// ignores any errors here.
 	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
 
+	// Collect any disk healing.
+	healing, _ := getAggregatedBackgroundHealState(ctx)
+	healDisks := make(map[string]struct{}, len(healing.HealDisks))
+	for _, disk := range healing.HealDisks {
+		healDisks[disk] = struct{}{}
+	}
+
+	// find all disks which belong to each respective endpoints
+	for i, disk := range storageInfo.Disks {
+		if _, ok := healDisks[disk.Endpoint]; ok {
+			storageInfo.Disks[i].Healing = true
+		}
+	}
+
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
 	if err != nil {
@@ -334,7 +353,9 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEn
 		Resource:   resource,
 		ServerList: []string{server},
 		Source:     l.Source,
+		Owner:      l.Owner,
 		ID:         l.UID,
+		Quorum:     l.Quorum,
 	}
 	if l.Writer {
 		entry.Type = "WRITE"
@@ -344,7 +365,7 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEn
 	return entry
 }
 
-func topLockEntries(peerLocks []*PeerLocks, count int) madmin.LockEntries {
+func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 	entryMap := make(map[string]*madmin.LockEntry)
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
@@ -362,14 +383,17 @@ func topLockEntries(peerLocks []*PeerLocks, count int) madmin.LockEntries {
 			}
 		}
 	}
-	var lockEntries = make(madmin.LockEntries, 0, len(entryMap))
+	var lockEntries madmin.LockEntries
 	for _, v := range entryMap {
-		lockEntries = append(lockEntries, *v)
+		if stale {
+			lockEntries = append(lockEntries, *v)
+			continue
+		}
+		if len(v.ServerList) >= v.Quorum {
+			lockEntries = append(lockEntries, *v)
+		}
 	}
 	sort.Sort(lockEntries)
-	if len(lockEntries) > count {
-		lockEntries = lockEntries[:count]
-	}
 	return lockEntries
 }
 
@@ -399,23 +423,17 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 			return
 		}
 	}
+	stale := r.URL.Query().Get("stale") == "true" // list also stale locks
 
-	peerLocks := globalNotificationSys.GetLocks(ctx)
-	// Once we have received all the locks currently used from peers
-	// add the local peer locks list as well.
-	var getRespLocks GetLocksResp
-	for _, llocker := range globalLockServers {
-		getRespLocks = append(getRespLocks, llocker.DupLockMap())
+	peerLocks := globalNotificationSys.GetLocks(ctx, r)
+
+	topLocks := topLockEntries(peerLocks, stale)
+
+	// Marshal API response upto requested count.
+	if len(topLocks) > count && count > 0 {
+		topLocks = topLocks[:count]
 	}
 
-	peerLocks = append(peerLocks, &PeerLocks{
-		Addr:  getHostName(r),
-		Locks: getRespLocks,
-	})
-
-	topLocks := topLockEntries(peerLocks, count)
-
-	// Marshal API response
 	jsonBytes, err := json.Marshal(topLocks)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -561,8 +579,8 @@ type healInitParams struct {
 
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) {
-	hip.bucket = vars[string(mgmtBucket)]
-	hip.objPrefix = vars[string(mgmtPrefix)]
+	hip.bucket = vars[mgmtBucket]
+	hip.objPrefix = vars[mgmtPrefix]
 
 	if hip.bucket == "" {
 		if hip.objPrefix != "" {
@@ -581,13 +599,13 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 		return
 	}
 
-	if len(qParms[string(mgmtClientToken)]) > 0 {
-		hip.clientToken = qParms[string(mgmtClientToken)][0]
+	if len(qParms[mgmtClientToken]) > 0 {
+		hip.clientToken = qParms[mgmtClientToken][0]
 	}
-	if _, ok := qParms[string(mgmtForceStart)]; ok {
+	if _, ok := qParms[mgmtForceStart]; ok {
 		hip.forceStart = true
 	}
-	if _, ok := qParms[string(mgmtForceStop)]; ok {
+	if _, ok := qParms[mgmtForceStop]; ok {
 		hip.forceStop = true
 	}
 
@@ -684,7 +702,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					// Start writing response to client
 					started = true
 					setCommonHeaders(w)
-					w.Header().Set(xhttp.ContentType, "text/event-stream")
+					setEventStreamHeaders(w)
 					// Set 200 OK status
 					w.WriteHeader(200)
 				}
@@ -788,6 +806,59 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, r, respCh)
 }
 
+func getAggregatedBackgroundHealState(ctx context.Context) (madmin.BgHealState, error) {
+	var bgHealStates []madmin.BgHealState
+
+	localHealState, ok := getLocalBackgroundHealStatus()
+	if !ok {
+		return madmin.BgHealState{}, errServerNotInitialized
+	}
+
+	// Get local heal status first
+	bgHealStates = append(bgHealStates, localHealState)
+
+	if globalIsDistErasure {
+		// Get heal status from other peers
+		peersHealStates, nerrs := globalNotificationSys.BackgroundHealStatus()
+		var errCount int
+		for _, nerr := range nerrs {
+			if nerr.Err != nil {
+				logger.LogIf(ctx, nerr.Err)
+				errCount++
+			}
+		}
+		if errCount == len(nerrs) {
+			return madmin.BgHealState{}, fmt.Errorf("all remote servers failed to report heal status, cluster is unhealthy")
+		}
+		bgHealStates = append(bgHealStates, peersHealStates...)
+	}
+
+	// Aggregate healing result
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+		HealDisks:         bgHealStates[0].HealDisks,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
+	for _, state := range bgHealStates {
+		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		aggregatedHealStateResult.HealDisks = append(aggregatedHealStateResult.HealDisks, state.HealDisks...)
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
+		}
+	}
+
+	return aggregatedHealStateResult, nil
+}
+
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
@@ -804,39 +875,13 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	var bgHealStates []madmin.BgHealState
-
-	// Get local heal status first
-	bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
-
-	if globalIsDistErasure {
-		// Get heal status from other peers
-		peersHealStates := globalNotificationSys.BackgroundHealStatus()
-		bgHealStates = append(bgHealStates, peersHealStates...)
+	aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context())
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 
-	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{
-		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
-		LastHealActivity:  bgHealStates[0].LastHealActivity,
-		NextHealRound:     bgHealStates[0].NextHealRound,
-	}
-
-	bgHealStates = bgHealStates[1:]
-
-	for _, state := range bgHealStates {
-		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
-			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
-			// The node which has the last heal activity means its
-			// is the node that is orchestrating self healing operations,
-			// which also means it is the same node which decides when
-			// the next self healing operation will be done.
-			aggregatedHealStateResult.NextHealRound = state.NextHealRound
-		}
-	}
-
-	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
+	if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -848,7 +893,7 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	var cred auth.Credentials
 	var adminAPIErr APIErrorCode
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
@@ -984,13 +1029,13 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	setEventStreamHeaders(w)
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
@@ -1052,12 +1097,13 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	// Avoid reusing tcp connection if read timeout is hit
 	// This is needed to make r.Context().Done() work as
 	// expected in case of read timeout
-	w.Header().Add("Connection", "close")
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	w.Header().Set("Connection", "close")
+
+	setEventStreamHeaders(w)
 
 	logCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 
@@ -1202,7 +1248,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	vars := mux.Vars(r)
+	query := r.URL.Query()
 	obdInfo := madmin.OBDInfo{}
 	obdInfoCh := make(chan madmin.OBDInfo)
 
@@ -1212,7 +1258,9 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	setCommonHeaders(w)
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
+
+	setEventStreamHeaders(w)
+
 	w.WriteHeader(http.StatusOK)
 
 	errResp := func(err error) {
@@ -1234,11 +1282,10 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
-
 	defer cancel()
 
-	nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "obd-in-progress")
+	if err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
@@ -1247,7 +1294,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	go func() {
 		defer close(obdInfoCh)
 
-		if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
+		if cpu := query.Get("syscpu"); cpu == "true" {
 			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
 
 			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
@@ -1255,7 +1302,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
+		if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
 			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
@@ -1263,7 +1310,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
+		if osInfo := query.Get("sysosinfo"); osInfo == "true" {
 			osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
@@ -1271,7 +1318,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if mem, ok := vars["sysmem"]; ok && mem == "true" {
+		if mem := query.Get("sysmem"); mem == "true" {
 			memInfo := getLocalMemOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
@@ -1279,7 +1326,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if proc, ok := vars["sysprocess"]; ok && proc == "true" {
+		if proc := query.Get("sysprocess"); proc == "true" {
 			procInfo := getLocalProcOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
@@ -1287,14 +1334,14 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if config, ok := vars["minioconfig"]; ok && config == "true" {
+		if config := query.Get("minioconfig"); config == "true" {
 			cfg, err := readServerConfig(ctx, objectAPI)
 			logger.LogIf(ctx, err)
 			obdInfo.Minio.Config = cfg
 			partialWrite(obdInfo)
 		}
 
-		if drive, ok := vars["perfdrive"]; ok && drive == "true" {
+		if drive := query.Get("perfdrive"); drive == "true" {
 			// Get drive obd details from local server's drive(s)
 			driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
 			driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
@@ -1325,7 +1372,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure {
+		if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
 			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
 			partialWrite(obdInfo)
 
@@ -1339,6 +1386,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
 			partialWrite(obdInfo)
 		}
+
 	}()
 
 	ticker := time.NewTicker(30 * time.Second)
@@ -1365,6 +1413,31 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 
 }
 
+// BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
+// ----------
+// Get bandwidth consumption information
+func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "BandwidthMonitor")
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.BandwidthMonitorAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	setEventStreamHeaders(w)
+	bucketsRequestedString := r.URL.Query().Get("buckets")
+	bucketsRequested := strings.Split(bucketsRequestedString, ",")
+	consolidatedReport := globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
+	enc := json.NewEncoder(w)
+	err := enc.Encode(consolidatedReport)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+	}
+	w.(http.Flusher).Flush()
+}
+
 // ServerInfoHandler - GET /minio/admin/v3/info
 // ----------
 // Get server information
@@ -1378,12 +1451,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	cfg, err := readServerConfig(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	buckets := madmin.Buckets{}
 	objects := madmin.Objects{}
 	usage := madmin.Usage{}
@@ -1395,7 +1462,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
 	}
 
-	vault := fetchVaultStatus(cfg)
+	vault := fetchVaultStatus()
 
 	ldap := madmin.LDAP{}
 	if globalLDAPConfig.Enabled {
@@ -1411,10 +1478,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	log, audit := fetchLoggerInfo(cfg)
+	log, audit := fetchLoggerInfo()
 
 	// Get the notification target info
-	notifyTarget := fetchLambdaInfo(cfg)
+	notifyTarget := fetchLambdaInfo()
 
 	// Fetching the Storage information, ignore any errors.
 	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
@@ -1436,11 +1503,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	mode := "safemode"
-	if newObjectLayerFn() != nil {
-		mode = "online"
-	}
-
+	mode := "online"
 	server := getLocalServerProperty(globalEndpoints, r)
 	servers := globalNotificationSys.ServerInfo()
 	servers = append(servers, server)
@@ -1462,8 +1525,12 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 		}
 	}
+
 	// add all the disks local to this server.
 	for _, disk := range storageInfo.Disks {
+		if disk.DrivePath == "" && disk.Endpoint == "" {
+			continue
+		}
 		if disk.Endpoint == disk.DrivePath {
 			servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
 		}
@@ -1490,27 +1557,33 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	//Reply with storage information (across nodes in a
+	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
-
-	// Fetch the configured targets
-	tr := NewGatewayHTTPTransport()
-	defer tr.CloseIdleConnections()
-	targetList, err := notify.FetchRegisteredTargets(cfg, GlobalContext.Done(), tr, true, false)
-	if err != nil && err != notify.ErrTargetsOffline {
-		logger.LogIf(GlobalContext, err)
-		return nil
-	}
+func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
-	for targetID, target := range targetList.TargetMap() {
+	for _, tgt := range globalConfigTargetList.Targets() {
 		targetIDStatus := make(map[string]madmin.Status)
-		active, _ := target.IsActive()
+		active, _ := tgt.IsActive()
+		targetID := tgt.ID()
+		if active {
+			targetIDStatus[targetID.ID] = madmin.Status{Status: "Online"}
+		} else {
+			targetIDStatus[targetID.ID] = madmin.Status{Status: "Offline"}
+		}
+		list := lambdaMap[targetID.Name]
+		list = append(list, targetIDStatus)
+		lambdaMap[targetID.Name] = list
+	}
+
+	for _, tgt := range globalEnvTargetList.Targets() {
+		targetIDStatus := make(map[string]madmin.Status)
+		active, _ := tgt.IsActive()
+		targetID := tgt.ID()
 		if active {
 			targetIDStatus[targetID.ID] = madmin.Status{Status: "Online"}
 		} else {
@@ -1519,8 +1592,6 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 		list := lambdaMap[targetID.Name]
 		list = append(list, targetIDStatus)
 		lambdaMap[targetID.Name] = list
-		// Close any leaking connections
-		_ = target.Close()
 	}
 
 	notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
@@ -1535,7 +1606,7 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 }
 
 // fetchVaultStatus fetches Vault Info
-func fetchVaultStatus(cfg config.Config) madmin.Vault {
+func fetchVaultStatus() madmin.Vault {
 	vault := madmin.Vault{}
 	if GlobalKMS == nil {
 		vault.Status = "disabled"
@@ -1544,12 +1615,12 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 	keyID := GlobalKMS.DefaultKeyID()
 	kmsInfo := GlobalKMS.Info()
 
-	if kmsInfo.Endpoint == "" {
+	if len(kmsInfo.Endpoints) == 0 {
 		vault.Status = "KMS configured using master key"
 		return vault
 	}
 
-	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
+	if err := checkConnection(kmsInfo.Endpoints[0], 15*time.Second); err != nil {
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"
@@ -1578,62 +1649,72 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 }
 
 // fetchLoggerDetails return log info
-func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
-	loggerCfg, _ := logger.LookupConfig(cfg)
-
-	var logger []madmin.Logger
-	var auditlogger []madmin.Audit
-	for log, l := range loggerCfg.HTTP {
-		if l.Enabled {
-			err := checkConnection(l.Endpoint, 15*time.Second)
+func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
+	var loggerInfo []madmin.Logger
+	var auditloggerInfo []madmin.Audit
+	for _, target := range logger.Targets {
+		if target.Endpoint() != "" {
+			tgt := target.String()
+			err := checkConnection(target.Endpoint(), 15*time.Second)
 			if err == nil {
 				mapLog := make(map[string]madmin.Status)
-				mapLog[log] = madmin.Status{Status: "Online"}
-				logger = append(logger, mapLog)
+				mapLog[tgt] = madmin.Status{Status: "Online"}
+				loggerInfo = append(loggerInfo, mapLog)
 			} else {
 				mapLog := make(map[string]madmin.Status)
-				mapLog[log] = madmin.Status{Status: "offline"}
-				logger = append(logger, mapLog)
+				mapLog[tgt] = madmin.Status{Status: "offline"}
+				loggerInfo = append(loggerInfo, mapLog)
 			}
 		}
 	}
 
-	for audit, l := range loggerCfg.Audit {
-		if l.Enabled {
-			err := checkConnection(l.Endpoint, 15*time.Second)
+	for _, target := range logger.AuditTargets {
+		if target.Endpoint() != "" {
+			tgt := target.String()
+			err := checkConnection(target.Endpoint(), 15*time.Second)
 			if err == nil {
 				mapAudit := make(map[string]madmin.Status)
-				mapAudit[audit] = madmin.Status{Status: "Online"}
-				auditlogger = append(auditlogger, mapAudit)
+				mapAudit[tgt] = madmin.Status{Status: "Online"}
+				auditloggerInfo = append(auditloggerInfo, mapAudit)
 			} else {
 				mapAudit := make(map[string]madmin.Status)
-				mapAudit[audit] = madmin.Status{Status: "Offline"}
-				auditlogger = append(auditlogger, mapAudit)
+				mapAudit[tgt] = madmin.Status{Status: "Offline"}
+				auditloggerInfo = append(auditloggerInfo, mapAudit)
 			}
 		}
 	}
-	return logger, auditlogger
+
+	return loggerInfo, auditloggerInfo
 }
 
 // checkConnection - ping an endpoint , return err in case of no connection
 func checkConnection(endpointStr string, timeout time.Duration) error {
-	tr := newCustomHTTPTransport(&tls.Config{RootCAs: globalRootCAs}, timeout)()
-	defer tr.CloseIdleConnections()
-
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
 	defer cancel()
 
-	req, err := http.NewRequest(http.MethodHead, endpointStr, nil)
+	client := &http.Client{Transport: &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           xhttp.NewCustomDialContext(timeout),
+		ResponseHeaderTimeout: 5 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
+		ExpectContinueTimeout: 5 * time.Second,
+		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
+		// Go net/http automatically unzip if content-type is
+		// gzip disable this feature, as we are always interested
+		// in raw stream.
+		DisableCompression: true,
+	}}
+	defer client.CloseIdleConnections()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
 	if err != nil {
 		return err
 	}
 
-	client := &http.Client{Transport: tr}
-	resp, err := client.Do(req.WithContext(ctx))
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
 	defer xhttp.DrainBody(resp.Body)
-	resp.Body.Close()
 	return nil
 }

cmd/admin-handlers_test.go

@@ -105,8 +105,8 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer := &erasureZones{zones: make([]*erasureSets, 1)}
-	objLayer.zones[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
+	objLayer := &erasureServerSets{serverSets: make([]*erasureSets, 1)}
+	objLayer.serverSets[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -327,13 +327,13 @@ func TestExtractHealInitParams(t *testing.T) {
 	mkParams := func(clientToken string, forceStart, forceStop bool) url.Values {
 		v := url.Values{}
 		if clientToken != "" {
-			v.Add(string(mgmtClientToken), clientToken)
+			v.Add(mgmtClientToken, clientToken)
 		}
 		if forceStart {
-			v.Add(string(mgmtForceStart), "")
+			v.Add(mgmtForceStart, "")
 		}
 		if forceStop {
-			v.Add(string(mgmtForceStop), "")
+			v.Add(mgmtForceStop, "")
 		}
 		return v
 	}
@@ -351,11 +351,11 @@ func TestExtractHealInitParams(t *testing.T) {
 	}
 	varsArr := []map[string]string{
 		// Invalid cases
-		{string(mgmtPrefix): "objprefix"},
+		{mgmtPrefix: "objprefix"},
 		// Valid cases
 		{},
-		{string(mgmtBucket): "bucket"},
-		{string(mgmtBucket): "bucket", string(mgmtPrefix): "objprefix"},
+		{mgmtBucket: "bucket"},
+		{mgmtBucket: "bucket", mgmtPrefix: "objprefix"},
 	}
 
 	// Body is always valid - we do not test JSON decoding.

cmd/admin-heal-ops.go

@@ -63,7 +63,7 @@ var (
 	errHealStopSignalled = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
-		apiErr := toAPIError(ctx, err)
+		apiErr := toAdminAPIErr(ctx, err)
 		return fmt.Errorf("Heal internal error: %s: %s",
 			apiErr.Code, apiErr.Description)
 	}
@@ -85,16 +85,18 @@ type healSequenceStatus struct {
 
 // structure to hold state of all heal sequences in server memory
 type allHealState struct {
-	sync.Mutex
+	sync.RWMutex
 
 	// map of heal path to heal sequence
-	healSeqMap map[string]*healSequence
+	healSeqMap     map[string]*healSequence
+	healLocalDisks map[Endpoint]struct{}
 }
 
 // newHealState - initialize global heal state management
 func newHealState() *allHealState {
 	healState := &allHealState{
-		healSeqMap: make(map[string]*healSequence),
+		healSeqMap:     make(map[string]*healSequence),
+		healLocalDisks: map[Endpoint]struct{}{},
 	}
 
 	go healState.periodicHealSeqsClean(GlobalContext)
@@ -102,6 +104,42 @@ func newHealState() *allHealState {
 	return healState
 }
 
+func (ahs *allHealState) healDriveCount() int {
+	ahs.RLock()
+	defer ahs.RUnlock()
+
+	return len(ahs.healLocalDisks)
+}
+
+func (ahs *allHealState) getHealLocalDisks() Endpoints {
+	ahs.RLock()
+	defer ahs.RUnlock()
+
+	var endpoints Endpoints
+	for ep := range ahs.healLocalDisks {
+		endpoints = append(endpoints, ep)
+	}
+	return endpoints
+}
+
+func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	for _, ep := range healLocalDisks {
+		delete(ahs.healLocalDisks, ep)
+	}
+}
+
+func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	for _, ep := range healLocalDisks {
+		ahs.healLocalDisks[ep] = struct{}{}
+	}
+}
+
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
@@ -151,7 +189,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	he, exists := ahs.getHealSequence(path)
 	if !exists {
 		hsp = madmin.HealStopSuccess{
-			ClientToken: "invalid",
+			ClientToken: "unknown",
 			StartTime:   UTCNow(),
 		}
 	} else {
@@ -193,21 +231,14 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	respBytes []byte, apiErr APIError, errMsg string) {
 
-	existsAndLive := false
-	he, exists := ahs.getHealSequence(pathJoin(h.bucket, h.object))
-	if exists {
-		existsAndLive = !he.hasEnded()
-	}
-
-	if existsAndLive {
-		// A heal sequence exists on the given path.
-		if h.forceStarted {
-			// stop the running heal sequence - wait for it to finish.
-			he.stop()
-			for !he.hasEnded() {
-				time.Sleep(1 * time.Second)
-			}
-		} else {
+	if h.forceStarted {
+		_, apiErr = ahs.stopHealSequence(pathJoin(h.bucket, h.object))
+		if apiErr.Code != "" {
+			return respBytes, apiErr, ""
+		}
+	} else {
+		oh, exists := ahs.getHealSequence(pathJoin(h.bucket, h.object))
+		if exists && !oh.hasEnded() {
 			errMsg = "Heal is already running on the given path " +
 				"(use force-start option to stop and start afresh). " +
 				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
@@ -224,7 +255,6 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	hpath := pathJoin(h.bucket, h.object)
 	for k, hSeq := range ahs.healSeqMap {
 		if !hSeq.hasEnded() && (HasPrefix(k, hpath) || HasPrefix(hpath, k)) {
-
 			errMsg = "The provided heal sequence path overlaps with an existing " +
 				fmt.Sprintf("heal path: %s", k)
 			return nil, errorCodes.ToAPIErr(ErrHealOverlappingPaths), errMsg
@@ -249,7 +279,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	})
 	if err != nil {
 		logger.LogIf(h.ctx, err)
-		return nil, toAPIError(h.ctx, err), ""
+		return nil, toAdminAPIErr(h.ctx, err), ""
 	}
 	return b, noError, ""
 }
@@ -264,8 +294,11 @@ func (ahs *allHealState) PopHealStatusJSON(hpath string,
 	// fetch heal state for given path
 	h, exists := ahs.getHealSequence(hpath)
 	if !exists {
-		// If there is no such heal sequence, return error.
-		return nil, ErrHealNoSuchProcess
+		// heal sequence doesn't exist, must have finished.
+		jbytes, err := json.Marshal(healSequenceStatus{
+			Summary: healFinishedStatus,
+		})
+		return jbytes, toAdminAPIErrCode(GlobalContext, err)
 	}
 
 	// Check if client-token is valid
@@ -490,9 +523,12 @@ func (h *healSequence) isQuitting() bool {
 // check if the heal sequence has ended
 func (h *healSequence) hasEnded() bool {
 	h.mutex.RLock()
-	ended := len(h.currentStatus.Items) == 0 || h.currentStatus.Summary == healStoppedStatus || h.currentStatus.Summary == healFinishedStatus
-	h.mutex.RUnlock()
-	return ended
+	defer h.mutex.RUnlock()
+	// background heal never ends
+	if h.clientToken == bgHealingUUID {
+		return false
+	}
+	return !h.endTime.IsZero()
 }
 
 // stops the heal sequence - safe to call multiple times.
@@ -606,8 +642,7 @@ func (h *healSequence) healSequenceStart() {
 	case <-h.ctx.Done():
 		h.mutex.Lock()
 		h.endTime = UTCNow()
-		h.currentStatus.Summary = healStoppedStatus
-		h.currentStatus.FailureDetail = errHealStopSignalled.Error()
+		h.currentStatus.Summary = healFinishedStatus
 		h.mutex.Unlock()
 
 		// drain traverse channel so the traversal
@@ -633,6 +668,12 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	if source.opts != nil {
 		task.opts = *source.opts
 	}
+
+	h.mutex.Lock()
+	h.scannedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+	h.mutex.Unlock()
+
 	globalBackgroundHealRoutine.queueHealTask(task)
 
 	select {
@@ -640,9 +681,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		if !h.reportProgress {
 			// Object might have been deleted, by the time heal
 			// was attempted, we should ignore this object and
-			// return success.
+			// return the error and not calculate this object
+			// as part of the metrics.
 			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
-				return nil
+				// Return the error so that caller can handle it.
+				return res.err
 			}
 
 			h.mutex.Lock()
@@ -704,11 +747,15 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			}
 
 			if err := h.queueHealTask(source, itemType); err != nil {
-				logger.LogIf(h.ctx, err)
+				switch err.(type) {
+				case ObjectExistsAsDirectory:
+				case ObjectNotFound:
+				case VersionNotFound:
+				default:
+					logger.LogIf(h.ctx, fmt.Errorf("Heal attempt failed for %s: %w",
+						pathJoin(source.bucket, source.object), err))
+				}
 			}
-
-			h.scannedItemsMap[itemType]++
-			h.lastHealActivity = UTCNow()
 		case <-h.ctx.Done():
 			return nil
 		}
@@ -761,7 +808,7 @@ func (h *healSequence) traverseAndHeal() {
 func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 	return func() error {
 		// Get current object layer instance.
-		objectAPI := newObjectLayerWithoutSafeModeFn()
+		objectAPI := newObjectLayerFn()
 		if objectAPI == nil {
 			return errServerNotInitialized
 		}
@@ -774,17 +821,17 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 				return errHealStopSignalled
 			}
 
-			herr := h.queueHealTask(healSource{
+			err := h.queueHealTask(healSource{
 				bucket:    bucket,
 				object:    object,
 				versionID: versionID,
 			}, madmin.HealItemBucketMetadata)
 			// Object might have been deleted, by the time heal
 			// was attempted we ignore this object an move on.
-			if isErrObjectNotFound(herr) || isErrVersionNotFound(herr) {
+			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				return nil
 			}
-			return herr
+			return err
 		})
 	}
 }
@@ -797,7 +844,7 @@ func (h *healSequence) healDiskFormat() error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -817,7 +864,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -839,13 +886,15 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
-		return err
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
 	}
 
 	if bucketsOnly {
@@ -856,9 +905,12 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		if h.object != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
+			oi, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
-				if err = h.healObject(bucket, h.object, ""); err != nil {
+				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+						return nil
+					}
 					return err
 				}
 			}
@@ -868,7 +920,11 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	}
 
 	if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
-		return errFnHealFromAPIErr(h.ctx, err)
+		// Object might have been deleted, by the time heal
+		// was attempted we ignore this object an move on.
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return errFnHealFromAPIErr(h.ctx, err)
+		}
 	}
 	return nil
 }
@@ -876,7 +932,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object, versionID string) error {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -885,9 +941,10 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		return errHealStopSignalled
 	}
 
-	return h.queueHealTask(healSource{
+	err := h.queueHealTask(healSource{
 		bucket:    bucket,
 		object:    object,
 		versionID: versionID,
 	}, madmin.HealItemObject)
+	return err
 }

cmd/admin-quota-handlers.go

@@ -1,118 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/env"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
-)
-
-const (
-	bucketQuotaConfigFile = "quota.json"
-)
-
-// PutBucketQuotaConfigHandler - PUT Bucket quota configuration.
-// ----------
-// Places a quota configuration on the specified bucket. The quota
-// specified in the quota configuration will be applied by default
-// to enforce total quota for the specified bucket.
-func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "PutBucketQuotaConfig")
-
-	defer logger.AuditLog(w, r, "PutBucketQuotaConfig", mustGetClaimsFromToken(r))
-
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
-	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Turn off quota commands if data usage info is unavailable.
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminBucketQuotaDisabled), r.URL)
-		return
-	}
-
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	data, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-		return
-	}
-
-	if _, err = parseBucketQuota(bucket, data); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	if err = globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, data); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
-}
-
-// GetBucketQuotaConfigHandler - gets bucket quota configuration
-func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetBucketQuotaConfig")
-
-	defer logger.AuditLog(w, r, "GetBucketQuotaConfig", mustGetClaimsFromToken(r))
-
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
-	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	config, err := globalBucketMetadataSys.GetQuotaConfig(bucket)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	configData, err := json.Marshal(config)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Write success response.
-	writeSuccessResponseJSON(w, configData)
-}

cmd/admin-router.go

@@ -171,8 +171,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 		}
 
-		// Quota operations
 		if globalIsDistErasure || globalIsErasure {
+			// Quota operations
 			if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
 				// GetBucketQuotaConfig
 				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
@@ -180,9 +180,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 				// PutBucketQuotaConfig
 				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
 					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+
+				// Bucket replication operations
+				// GetBucketTargetHandler
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+					httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+				// SetRemoteTargetHandler
+				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+				// SetRemoteTargetHandler
+				adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 			}
 		}
-
 		// -- Top APIs --
 		// Top locks
 		if globalIsDistErasure {
@@ -202,22 +212,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		if !globalIsGateway {
 			// -- OBD API --
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").
-				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).
-				Queries("perfdrive", "{perfdrive:true|false}",
-					"perfnet", "{perfnet:true|false}",
-					"minioinfo", "{minioinfo:true|false}",
-					"minioconfig", "{minioconfig:true|false}",
-					"syscpu", "{syscpu:true|false}",
-					"sysdiskhw", "{sysdiskhw:true|false}",
-					"sysosinfo", "{sysosinfo:true|false}",
-					"sysmem", "{sysmem:true|false}",
-					"sysprocess", "{sysprocess:true|false}",
-				)
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
+				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
+				HandlerFunc(httpTraceHdrs(adminAPI.BandwidthMonitorHandler))
 		}
 	}
 
 	// If none of the routes match add default error handler routes
-	adminRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
-	adminRouter.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
+	adminRouter.NotFoundHandler = httpTraceAll(errorResponseHandler)
+	adminRouter.MethodNotAllowedHandler = httpTraceAll(methodNotAllowedHandler("Admin"))
 }

cmd/admin-server-info.go

@@ -24,13 +24,13 @@ import (
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerSets EndpointServerSets, r *http.Request) madmin.ServerProperties {
 	addr := r.Host
 	if globalIsDistErasure {
-		addr = GetLocalPeer(endpointZones)
+		addr = GetLocalPeer(endpointServerSets)
 	}
 	network := make(map[string]string)
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			nodeName := endpoint.Host
 			if nodeName == "" {

cmd/api-errors.go

@@ -27,13 +27,15 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"google.golang.org/api/googleapi"
 
-	minio "github.com/minio/minio-go/v6"
-	"github.com/minio/minio-go/v6/pkg/tags"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/replication"
+
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/bucket/versioning"
@@ -104,6 +106,21 @@ const (
 	ErrNoSuchCORSConfiguration
 	ErrNoSuchWebsiteConfiguration
 	ErrReplicationConfigurationNotFoundError
+	ErrRemoteDestinationNotFoundError
+	ErrReplicationDestinationMissingLock
+	ErrRemoteTargetNotFoundError
+	ErrReplicationRemoteConnectionError
+	ErrBucketRemoteIdenticalToSource
+	ErrBucketRemoteAlreadyExists
+	ErrBucketRemoteLabelInUse
+	ErrBucketRemoteArnTypeInvalid
+	ErrBucketRemoteArnInvalid
+	ErrBucketRemoteRemoveDisallowed
+	ErrRemoteTargetNotVersionedError
+	ErrReplicationSourceNotVersionedError
+	ErrReplicationNeedsVersioningError
+	ErrReplicationBucketNeedsVersioningError
+	ErrBucketReplicationDisabledError
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -216,6 +233,7 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
+	ErrClientDisconnected
 	ErrOperationMaxedOut
 	ErrInvalidRequest
 	// MinIO storage class error codes
@@ -648,20 +666,9 @@ var errorCodes = errorCodeMap{
 		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// right Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
-	// Need changes to make sure variable messages can be constructed.
 	ErrMalformedCredentialDate: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// right Description:    "Error parsing the X-Amz-Credential parameter; the region 'us-east-' is wrong; expecting 'us-east-1'".
-	// Need changes to make sure variable messages can be constructed.
-	ErrMalformedCredentialRegion: {
-		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; the region is wrong;",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format. This date in the credential must be in the format \"yyyyMMdd\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidRegion: {
@@ -669,9 +676,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Region does not match.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
-	// Need changes to make sure variable messages can be constructed.
 	ErrInvalidServiceS3: {
 		Code:           "AuthorizationParametersError",
 		Description:    "Error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
@@ -682,9 +686,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Error parsing the Credential parameter; incorrect service. This endpoint belongs to \"sts\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// Description:   "Error parsing the X-Amz-Credential parameter; incorrect terminal "aws4_reque". This endpoint uses "aws4_request".
-	// Need changes to make sure variable messages can be constructed.
 	ErrInvalidRequestVersion: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "Error parsing the X-Amz-Credential parameter; incorrect terminal. This endpoint uses \"aws4_request\".",
@@ -755,8 +756,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Your key is too long",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
-	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
 		Description:    "There were headers present in the request which were not signed",
@@ -812,6 +811,81 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication configuration was not found",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrRemoteDestinationNotFoundError: {
+		Code:           "RemoteDestinationNotFoundError",
+		Description:    "The remote destination bucket does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrReplicationDestinationMissingLock: {
+		Code:           "ReplicationDestinationMissingLockError",
+		Description:    "The replication destination bucket does not have object locking enabled",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrRemoteTargetNotFoundError: {
+		Code:           "XMinioAdminRemoteTargetNotFoundError",
+		Description:    "The remote target does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrReplicationRemoteConnectionError: {
+		Code:           "XMinioAdminReplicationRemoteConnectionError",
+		Description:    "Remote service endpoint or target bucket not available",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrBucketRemoteIdenticalToSource: {
+		Code:           "XMinioAdminRemoteIdenticalToSource",
+		Description:    "The remote target cannot be identical to source",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteAlreadyExists: {
+		Code:           "XMinioAdminBucketRemoteAlreadyExists",
+		Description:    "The remote target already exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteLabelInUse: {
+		Code:           "XMinioAdminBucketRemoteLabelInUse",
+		Description:    "The remote target with this label already exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteRemoveDisallowed: {
+		Code:           "XMinioAdminRemoteRemoveDisallowed",
+		Description:    "This ARN is in use by an existing configuration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteArnTypeInvalid: {
+		Code:           "XMinioAdminRemoteARNTypeInvalid",
+		Description:    "The bucket remote ARN type is not valid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketRemoteArnInvalid: {
+		Code:           "XMinioAdminRemoteArnInvalid",
+		Description:    "The bucket remote ARN does not have correct format",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrRemoteTargetNotVersionedError: {
+		Code:           "RemoteTargetNotVersionedError",
+		Description:    "The remote target does not have versioning enabled",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrReplicationSourceNotVersionedError: {
+		Code:           "ReplicationSourceNotVersionedError",
+		Description:    "The replication source does not have versioning enabled",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrReplicationNeedsVersioningError: {
+		Code:           "InvalidRequest",
+		Description:    "Versioning must be 'Enabled' on the bucket to apply a replication configuration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrReplicationBucketNeedsVersioningError: {
+		Code:           "InvalidRequest",
+		Description:    "Versioning must be 'Enabled' on the bucket to add a replication target",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketReplicationDisabledError: {
+		Code:           "XMinioAdminBucketReplicationDisabled",
+		Description:    "Replication specified but disk usage crawl is disabled on MinIO server",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
 		Description:    "The specified object does not have a ObjectLock configuration",
@@ -904,7 +978,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMetadataTooLarge: {
-		Code:           "InvalidArgument",
+		Code:           "MetadataTooLarge",
 		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
@@ -1143,6 +1217,11 @@ var errorCodes = errorCodeMap{
 		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrClientDisconnected: {
+		Code:           "ClientDisconnected",
+		Description:    "Client disconnected before response was ready",
+		HTTPStatusCode: 499, // No official code, use nginx value.
+	},
 	ErrOperationMaxedOut: {
 		Code:           "SlowDown",
 		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
@@ -1669,6 +1748,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Only return ErrClientDisconnected if the provided context is actually canceled.
+	// This way downstream context.Canceled will still report ErrOperationTimedOut
+	select {
+	case <-ctx.Done():
+		if ctx.Err() == context.Canceled {
+			return ErrClientDisconnected
+		}
+	default:
+	}
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -1837,6 +1926,30 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectLockConfigurationNotFound
 	case BucketQuotaConfigNotFound:
 		apiErr = ErrAdminNoSuchQuotaConfiguration
+	case BucketReplicationConfigNotFound:
+		apiErr = ErrReplicationConfigurationNotFoundError
+	case BucketRemoteDestinationNotFound:
+		apiErr = ErrRemoteDestinationNotFoundError
+	case BucketReplicationDestinationMissingLock:
+		apiErr = ErrReplicationDestinationMissingLock
+	case BucketRemoteTargetNotFound:
+		apiErr = ErrRemoteTargetNotFoundError
+	case BucketRemoteConnectionErr:
+		apiErr = ErrReplicationRemoteConnectionError
+	case BucketRemoteAlreadyExists:
+		apiErr = ErrBucketRemoteAlreadyExists
+	case BucketRemoteLabelInUse:
+		apiErr = ErrBucketRemoteLabelInUse
+	case BucketRemoteArnTypeInvalid:
+		apiErr = ErrBucketRemoteArnTypeInvalid
+	case BucketRemoteArnInvalid:
+		apiErr = ErrBucketRemoteArnInvalid
+	case BucketRemoteRemoveDisallowed:
+		apiErr = ErrBucketRemoteRemoveDisallowed
+	case BucketRemoteTargetNotVersioned:
+		apiErr = ErrRemoteTargetNotVersionedError
+	case BucketReplicationSourceNotVersioned:
+		apiErr = ErrReplicationSourceNotVersionedError
 	case BucketQuotaExceeded:
 		apiErr = ErrAdminBucketQuotaExceeded
 	case *event.ErrInvalidEventName:
@@ -1867,6 +1980,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrBackendDown
 	case ObjectNameTooLong:
 		apiErr = ErrKeyTooLongError
+	case dns.ErrInvalidBucketName:
+		apiErr = ErrInvalidBucketName
 	default:
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648
@@ -1903,6 +2018,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 	}
 
 	var apiErr = errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
+	e, ok := err.(dns.ErrInvalidBucketName)
+	if ok {
+		code := toAPIErrorCode(ctx, e)
+		apiErr = errorCodes.ToAPIErrWithErr(code, e)
+	}
+
 	if apiErr.Code == "InternalError" {
 		// If we see an internal error try to interpret
 		// any underlying errors if possible depending on
@@ -1941,6 +2062,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case replication.Error:
+			apiErr = APIError{
+				Code:           "MalformedXML",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case tags.Error:
 			apiErr = APIError{
 				Code:           e.Code(),

cmd/api-headers.go

@@ -38,6 +38,13 @@ func mustGetRequestID(t time.Time) string {
 	return fmt.Sprintf("%X", t.UnixNano())
 }
 
+// setEventStreamHeaders to allow proxies to avoid buffering proxy responses
+func setEventStreamHeaders(w http.ResponseWriter) {
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	w.Header().Set(xhttp.CacheControl, "no-cache") // nginx to turn off buffering
+	w.Header().Set("X-Accel-Buffering", "no")      // nginx to turn off buffering
+}
+
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
@@ -77,7 +84,7 @@ func setPartsCountHeaders(w http.ResponseWriter, objInfo ObjectInfo) {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec, opts ObjectOptions) (err error) {
 	// set common headers
 	setCommonHeaders(w)
 
@@ -108,10 +115,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set tag count if object has tags
-	tags, _ := url.ParseQuery(objInfo.UserTags)
-	tagCount := len(tags)
-	if tagCount > 0 {
-		w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
+	if len(objInfo.UserTags) > 0 {
+		tags, _ := url.ParseQuery(objInfo.UserTags)
+		if len(tags) > 0 {
+			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
+		}
 	}
 
 	// Set all other user defined metadata.
@@ -121,6 +129,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 			// values to client.
 			continue
 		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			continue
+		}
 		var isSet bool
 		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
 			if !strings.HasPrefix(k, userMetadataPrefix) {
@@ -135,15 +148,26 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		}
 	}
 
+	var start, rangeLen int64
 	totalObjectSize, err := objInfo.GetActualSize()
 	if err != nil {
 		return err
 	}
 
-	// for providing ranged content
-	start, rangeLen, err := rs.GetOffsetLength(totalObjectSize)
-	if err != nil {
-		return err
+	if opts.PartNumber > 0 {
+		var start, end int64
+		for i := 0; i < len(objInfo.Parts) && i < opts.PartNumber; i++ {
+			start = end
+			end = start + objInfo.Parts[i].ActualSize - 1
+		}
+		rs = &HTTPRangeSpec{Start: start, End: end}
+		rangeLen = end - start + 1
+	} else {
+		// for providing ranged content
+		start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Set content length.
@@ -157,7 +181,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if objInfo.VersionID != "" {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
-
+	if objInfo.ReplicationStatus.String() != "" {
+		w.Header()[xhttp.AmzBucketReplicationStatus] = []string{objInfo.ReplicationStatus.String()}
+	}
 	if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
 		ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
 			Name:         objInfo.Name,

cmd/api-response.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"net/http"
 	"net/url"
 	"path"
@@ -34,11 +35,11 @@ import (
 
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
-	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z"                     // Reply date format with nanosecond precision.
+	maxObjectList     = metacacheBlockSize - (metacacheBlockSize / 10) // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList     = 10000                                          // Limit number of objects deleted in a delete call.
+	maxUploadsList    = 10000                                          // Limit number of uploads in a listUploadsResponse.
+	maxPartsList      = 10000                                          // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -396,8 +397,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 	}
 	// If domain is set then we need to use bucket DNS style.
 	for _, domain := range domains {
-		if strings.Contains(r.Host, domain) {
-			u.Host = bucket + "." + r.Host
+		if strings.HasPrefix(r.Host, bucket+"."+domain) {
 			u.Path = path.Join(SlashSeparator, object)
 			break
 		}
@@ -408,7 +408,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 // generates ListBucketsResponse from array of BucketInfo which can be
 // serialized to match XML and JSON API spec output.
 func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
-	var listbuckets []Bucket
+	listbuckets := make([]Bucket, 0, len(buckets))
 	var data = ListBucketsResponse{}
 	var owner = Owner{}
 
@@ -428,8 +428,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
-	var versions []ObjectVersion
-	var prefixes []CommonPrefix
+	versions := make([]ObjectVersion, 0, len(resp.Objects))
 	var owner = Owner{}
 	var data = ListVersionsResponse{}
 
@@ -473,6 +472,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 	data.VersionIDMarker = versionIDMarker
 	data.IsTruncated = resp.IsTruncated
 
+	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
@@ -484,8 +484,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
-	var contents []Object
-	var prefixes []CommonPrefix
+	contents := make([]Object, 0, len(resp.Objects))
 	var owner = Owner{}
 	var data = ListObjectsResponse{}
 
@@ -517,9 +516,10 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 	data.Marker = s3EncodeName(marker, encodingType)
 	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.MaxKeys = maxKeys
-
 	data.NextMarker = s3EncodeName(resp.NextMarker, encodingType)
 	data.IsTruncated = resp.IsTruncated
+
+	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
@@ -531,8 +531,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
 func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
-	var contents []Object
-	var commonPrefixes []CommonPrefix
+	contents := make([]Object, 0, len(objects))
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
@@ -565,6 +564,10 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					// values to client.
 					continue
 				}
+				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+					continue
+				}
 				content.UserMetadata[k] = v
 			}
 		}
@@ -581,6 +584,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token))
 	data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken))
 	data.IsTruncated = isTruncated
+
+	commonPrefixes := make([]CommonPrefix, 0, len(prefixes))
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
@@ -698,10 +703,6 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, err
 }
 
 func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	if newObjectLayerFn() == nil {
-		// Server still in safe mode.
-		w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-	}
 	setCommonHeaders(w)
 	if mType != mimeNone {
 		w.Header().Set(xhttp.ContentType, string(mType))
@@ -760,14 +761,14 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "120")
+	case "InvalidRegion":
+		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalServerRegion)
+	case "AuthorizationHeaderMalformed":
+		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalBrowserEnabled {
-			if newObjectLayerFn() == nil {
-				// server still in safe mode.
-				w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-			}
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return
@@ -818,38 +819,3 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
-
-// writeCustomErrorResponseXML - similar to writeErrorResponse,
-// but accepts the error message directly (this allows messages to be
-// dynamically generated.)
-func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err APIError, errBody string, reqURL *url.URL, browser bool) {
-
-	switch err.Code {
-	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
-		// Set retry-after header to indicate user-agents to retry request after 120secs.
-		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set(xhttp.RetryAfter, "120")
-	case "AccessDenied":
-		// The request is from browser and also if browser
-		// is enabled we need to redirect.
-		if browser && globalBrowserEnabled {
-			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
-			w.WriteHeader(http.StatusTemporaryRedirect)
-			return
-		}
-	}
-
-	reqInfo := logger.GetReqInfo(ctx)
-	errorResponse := APIErrorResponse{
-		Code:       err.Code,
-		Message:    errBody,
-		Resource:   reqURL.Path,
-		BucketName: reqInfo.BucketName,
-		Key:        reqInfo.ObjectName,
-		RequestID:  w.Header().Get(xhttp.AmzRequestID),
-		HostID:     globalDeploymentID,
-	}
-
-	encodedErrorResponse := encodeResponse(errorResponse)
-	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
-}

cmd/api-response_test.go

@@ -77,7 +77,7 @@ func TestObjectLocation(t *testing.T) {
 		// Server with virtual domain name.
 		{
 			request: &http.Request{
-				Host:   "mys3.bucket.org",
+				Host:   "mybucket.mys3.bucket.org",
 				Header: map[string][]string{},
 			},
 			domains:          []string{"mys3.bucket.org"},
@@ -87,7 +87,7 @@ func TestObjectLocation(t *testing.T) {
 		},
 		{
 			request: &http.Request{
-				Host: "mys3.bucket.org",
+				Host: "mybucket.mys3.bucket.org",
 				Header: map[string][]string{
 					"X-Forwarded-Scheme": {httpsScheme},
 				},
@@ -98,11 +98,14 @@ func TestObjectLocation(t *testing.T) {
 			expectedLocation: "https://mybucket.mys3.bucket.org/test/1.txt",
 		},
 	}
-	for i, testCase := range testCases {
-		gotLocation := getObjectLocation(testCase.request, testCase.domains, testCase.bucket, testCase.object)
-		if testCase.expectedLocation != gotLocation {
-			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedLocation, gotLocation)
-		}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			gotLocation := getObjectLocation(testCase.request, testCase.domains, testCase.bucket, testCase.object)
+			if testCase.expectedLocation != gotLocation {
+				t.Errorf("expected %s, got %s", testCase.expectedLocation, gotLocation)
+			}
+		})
 	}
 }
 

cmd/api-router.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -31,60 +32,75 @@ func newHTTPServerFn() *xhttp.Server {
 	return globalHTTPServer
 }
 
-func newObjectLayerWithoutSafeModeFn() ObjectLayer {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
-	return globalObjectAPI
-}
-
 func newObjectLayerFn() ObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-	if globalSafeMode {
-		return nil
-	}
 	return globalObjectAPI
 }
 
 func newCachedObjectLayerFn() CacheObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-
-	if globalSafeMode {
-		return nil
-	}
 	return globalCacheObjectAPI
 }
 
+func setObjectLayer(o ObjectLayer) {
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = o
+	globalObjLayerMutex.Unlock()
+}
+
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
 	CacheAPI  func() CacheObjectLayer
-	// Returns true of handlers should interpret encryption.
-	EncryptionEnabled func() bool
-	// Returns true if handlers allow SSE-KMS encryption headers.
-	AllowSSEKMS func() bool
+}
+
+// getHost tries its best to return the request host.
+// According to section 14.23 of RFC 2616 the Host header
+// can include the port number if the default value of 80 is not used.
+func getHost(r *http.Request) string {
+	if r.URL.IsAbs() {
+		return r.URL.Host
+	}
+	return r.Host
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
+func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
 		CacheAPI:  newCachedObjectLayerFn,
-		EncryptionEnabled: func() bool {
-			return encryptionEnabled
-		},
-		AllowSSEKMS: func() bool {
-			return allowSSEKMS
-		},
 	}
 
 	// API Router
 	apiRouter := router.PathPrefix(SlashSeparator).Subrouter()
+
 	var routers []*mux.Router
 	for _, domainName := range globalDomainNames {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		if IsKubernetes() {
+			routers = append(routers, apiRouter.MatcherFunc(func(r *http.Request, match *mux.RouteMatch) bool {
+				host, _, err := net.SplitHostPort(getHost(r))
+				if err != nil {
+					host = r.Host
+				}
+				// Make sure to skip matching minio.<domain>` this is
+				// specifically meant for operator/k8s deployment
+				// The reason we need to skip this is for a special
+				// usecase where we need to make sure that
+				// minio.<namespace>.svc.<cluster_domain> is ignored
+				// by the bucketDNS style to ensure that path style
+				// is available and honored at this domain.
+				//
+				// All other `<bucket>.<namespace>.svc.<cluster_domain>`
+				// makes sure that buckets are routed through this matcher
+				// to match for `<bucket>`
+				return host != minioReservedBucket+"."+domainName
+			}).Host("{bucket:.+}."+domainName).Subrouter())
+		} else {
+			routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		}
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
@@ -94,7 +110,10 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler))))
 		// CopyObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).
+			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
@@ -138,7 +157,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler))))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
 		// PutObjectRetention
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
@@ -166,6 +186,21 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// GetBucketEncryption
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
+		// GetBucketObjectLockConfig
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+		// GetBucketReplicationConfig
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketreplicationconfiguration", httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
+
+		// GetBucketVersioning
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
+		// GetBucketNotification
+		bucket.Methods(http.MethodGet).HandlerFunc(
+			maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
+		// ListenNotification
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
@@ -192,9 +227,6 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// GetBucketLifecycleHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
-		// GetBucketReplicationHandler - this is a dummy call.
-		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler)))).Queries("replication", "")
 		// GetBucketTaggingHandler
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
@@ -205,17 +237,6 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodDelete).HandlerFunc(
 			maxClients(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
 
-		// GetBucketObjectLockConfig
-		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
-		// GetBucketVersioning
-		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
-		// GetBucketNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
-		// ListenBucketNotification
-		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listenbucketnotification", httpTraceAll(api.ListenBucketNotificationHandler))).Queries("events", "{events:.*}")
 		// ListMultipartUploads
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
@@ -234,6 +255,11 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// PutBucketLifecycle
 		bucket.Methods(http.MethodPut).HandlerFunc(
 			maxClients(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
+		// PutBucketReplicationConfig
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbucketreplicationconfiguration", httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
+		// GetObjectRetention
+
 		// PutBucketEncryption
 		bucket.Methods(http.MethodPut).HandlerFunc(
 			maxClients(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
@@ -269,6 +295,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// DeleteBucketPolicy
 		bucket.Methods(http.MethodDelete).HandlerFunc(
 			maxClients(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
+		// DeleteBucketReplication
+		bucket.Methods(http.MethodDelete).HandlerFunc(
+			maxClients(collectAPIStats("deletebucketreplicationconfiguration", httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
 		// DeleteBucketLifecycle
 		bucket.Methods(http.MethodDelete).HandlerFunc(
 			maxClients(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
@@ -282,6 +311,10 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 
 	/// Root operation
 
+	// ListenNotification
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
+		collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
+
 	// ListBuckets
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
 		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
@@ -292,8 +325,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
 
 	// If none of the routes match add default error handler routes
-	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(errorResponseHandler)))
-	apiRouter.MethodNotAllowedHandler = http.HandlerFunc(collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler)))
+	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
+	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
 
 }
 
@@ -309,6 +342,13 @@ func corsHandler(handler http.Handler) http.Handler {
 		xhttp.ContentEncoding,
 		xhttp.ContentLength,
 		xhttp.ContentType,
+		xhttp.ContentDisposition,
+		xhttp.LastModified,
+		xhttp.ContentLanguage,
+		xhttp.CacheControl,
+		xhttp.RetryAfter,
+		xhttp.AmzBucketRegion,
+		xhttp.Expires,
 		"X-Amz*",
 		"x-amz*",
 		"*",

cmd/auth-handler.go

@@ -213,15 +213,15 @@ func getClaimsFromToken(r *http.Request, token string) (map[string]interface{},
 
 	if globalPolicyOPA == nil {
 		// If OPA is not set and if ldap claim key is set, allow the claim.
-		if _, ok := claims.Lookup(ldapUser); ok {
+		if _, ok := claims.MapClaims[ldapUser]; ok {
 			return claims.Map(), nil
 		}
 
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.
-		_, pokOpenID := claims.Lookup(iamPolicyClaimNameOpenID())
-		_, pokSA := claims.Lookup(iamPolicyClaimNameSA())
+		_, pokOpenID := claims.MapClaims[iamPolicyClaimNameOpenID()]
+		_, pokSA := claims.MapClaims[iamPolicyClaimNameSA()]
 		if !pokOpenID && !pokSA {
 			return nil, errAuthentication
 		}
@@ -333,8 +333,12 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Populate payload again to handle it in HTTP handler.
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
 
-	if cred.AccessKey == "" {
+	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
+		// Anonymous checks are not meant for ListBuckets action
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Action:          action,
@@ -346,8 +350,26 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 			// Request is allowed return the appropriate access key.
 			return cred.AccessKey, owner, ErrNone
 		}
+
+		if action == policy.ListBucketVersionsAction {
+			// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
+			// verify as a fallback.
+			if globalPolicySys.IsAllowed(policy.Args{
+				AccountName:     cred.AccessKey,
+				Action:          policy.ListBucketAction,
+				BucketName:      bucketName,
+				ConditionValues: getConditionValues(r, locationConstraint, "", nil),
+				IsOwner:         false,
+				ObjectName:      objectName,
+			}) {
+				// Request is allowed return the appropriate access key.
+				return cred.AccessKey, owner, ErrNone
+			}
+		}
+
 		return cred.AccessKey, owner, ErrAccessDenied
 	}
+
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
 		Action:          iampolicy.Action(action),
@@ -360,6 +382,24 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Request is allowed return the appropriate access key.
 		return cred.AccessKey, owner, ErrNone
 	}
+
+	if action == policy.ListBucketVersionsAction {
+		// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
+		// verify as a fallback.
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          iampolicy.ListBucketAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ObjectName:      objectName,
+			IsOwner:         owner,
+			Claims:          claims,
+		}) {
+			// Request is allowed return the appropriate access key.
+			return cred.AccessKey, owner, ErrNone
+		}
+	}
+
 	return cred.AccessKey, owner, ErrAccessDenied
 }
 
@@ -519,7 +559,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		if retMode == objectlock.RetGovernance && byPassSet {
 			byPassSet = globalPolicySys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
-				Action:          policy.Action(policy.BypassGovernanceRetentionAction),
+				Action:          policy.BypassGovernanceRetentionAction,
 				BucketName:      bucketName,
 				ConditionValues: conditions,
 				IsOwner:         false,
@@ -528,7 +568,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		}
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.Action(policy.PutObjectRetentionAction),
+			Action:          policy.PutObjectRetentionAction,
 			BucketName:      bucketName,
 			ConditionValues: conditions,
 			IsOwner:         false,
@@ -551,7 +591,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	if retMode == objectlock.RetGovernance && byPassSet {
 		byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.BypassGovernanceRetentionAction,
+			Action:          iampolicy.BypassGovernanceRetentionAction,
 			BucketName:      bucketName,
 			ObjectName:      objectName,
 			ConditionValues: conditions,
@@ -561,7 +601,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	}
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
-		Action:          policy.PutObjectRetentionAction,
+		Action:          iampolicy.PutObjectRetentionAction,
 		BucketName:      bucketName,
 		ConditionValues: conditions,
 		ObjectName:      objectName,
@@ -579,7 +619,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
-func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
+func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	switch atype {
@@ -600,6 +640,10 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 		return s3Err
 	}
 
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
+
 	// Do not check for PutObjectRetentionAction permission,
 	// if mode and retain until date are not set.
 	// Can happen when bucket has default lock config set

cmd/auth-handler_test.go

@@ -52,7 +52,7 @@ func TestGetRequestAuthType(t *testing.T) {
 					"X-Amz-Content-Sha256": []string{streamingContentSHA256},
 					"Content-Encoding":     []string{streamingContentEncoding},
 				},
-				Method: "PUT",
+				Method: http.MethodPut,
 			},
 			authT: authTypeStreamingSigned,
 		},
@@ -111,7 +111,7 @@ func TestGetRequestAuthType(t *testing.T) {
 				Header: http.Header{
 					"Content-Type": []string{"multipart/form-data"},
 				},
-				Method: "POST",
+				Method: http.MethodPost,
 			},
 			authT: authTypePostPolicy,
 		},
@@ -212,7 +212,7 @@ func TestIsRequestPresignedSignatureV2(t *testing.T) {
 	for i, testCase := range testCases {
 		// creating an input HTTP request.
 		// Only the query parameters are relevant for this particular test.
-		inputReq, err := http.NewRequest("GET", "http://example.com", nil)
+		inputReq, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 		if err != nil {
 			t.Fatalf("Error initializing input HTTP request: %v", err)
 		}
@@ -246,7 +246,7 @@ func TestIsRequestPresignedSignatureV4(t *testing.T) {
 	for i, testCase := range testCases {
 		// creating an input HTTP request.
 		// Only the query parameters are relevant for this particular test.
-		inputReq, err := http.NewRequest("GET", "http://example.com", nil)
+		inputReq, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 		if err != nil {
 			t.Fatalf("Error initializing input HTTP request: %v", err)
 		}
@@ -369,15 +369,15 @@ func TestIsReqAuthenticated(t *testing.T) {
 		s3Error APIErrorCode
 	}{
 		// When request is unsigned, access denied is returned.
-		{mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
+		{mustNewRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
 		// Empty Content-Md5 header.
-		{mustNewSignedEmptyMD5Request("PUT", "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrInvalidDigest},
+		{mustNewSignedEmptyMD5Request(http.MethodPut, "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrInvalidDigest},
 		// Short Content-Md5 header.
-		{mustNewSignedShortMD5Request("PUT", "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrInvalidDigest},
+		{mustNewSignedShortMD5Request(http.MethodPut, "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrInvalidDigest},
 		// When request is properly signed, but has bad Content-MD5 header.
-		{mustNewSignedBadMD5Request("PUT", "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
+		{mustNewSignedBadMD5Request(http.MethodPut, "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
 		// When request is properly signed, error is none.
-		{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
+		{mustNewSignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrNone},
 	}
 
 	ctx := context.Background()
@@ -413,11 +413,11 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		Request *http.Request
 		ErrCode APIErrorCode
 	}{
-		{Request: mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
-		{Request: mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrNone},
-		{Request: mustNewSignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
-		{Request: mustNewPresignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
-		{Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+		{Request: mustNewRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+		{Request: mustNewSignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrNone},
+		{Request: mustNewSignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+		{Request: mustNewPresignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
+		{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
@@ -461,7 +461,7 @@ func TestValidateAdminSignature(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		req := mustNewRequest("GET", "http://localhost:9000/", 0, nil, t)
+		req := mustNewRequest(http.MethodGet, "http://localhost:9000/", 0, nil, t)
 		if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
 			t.Fatalf("Unable to inititalized new signed http request %s", err)
 		}

cmd/background-heal-ops.go

@@ -21,7 +21,6 @@ import (
 	"path"
 	"time"
 
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -55,15 +54,20 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
-func waitForLowHTTPReq(tolerance int32) {
+func waitForLowHTTPReq(tolerance int32, maxWait time.Duration) {
+	const wait = 10 * time.Millisecond
+	waitCount := maxWait / wait
+
+	// Bucket notification and http trace are not costly, it is okay to ignore them
+	// while counting the number of concurrent connections
+	tolerance += int32(globalHTTPListen.NumSubscribers() + globalHTTPTrace.NumSubscribers())
+
 	if httpServer := newHTTPServerFn(); httpServer != nil {
-		// Wait at max 10 minute for an inprogress request before proceeding to heal
-		waitCount := 600
 		// Any requests in progress, delay the heal.
 		for (httpServer.GetRequestCount() >= tolerance) &&
 			waitCount > 0 {
 			waitCount--
-			time.Sleep(1 * time.Second)
+			time.Sleep(wait)
 		}
 	}
 }
@@ -78,7 +82,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			}
 
 			// Wait and proceed if there are active requests
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()))
+			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
 
 			var res madmin.HealResultItem
 			var err error
@@ -96,6 +100,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 				ObjectPathUpdated(path.Join(task.bucket, task.object))
 			}
 			task.responseCh <- healResult{result: res, err: err}
+
 		case <-h.doneCh:
 			return
 		case <-ctx.Done():
@@ -112,24 +117,6 @@ func newHealRoutine() *healRoutine {
 
 }
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
-	go globalBackgroundHealRoutine.run(ctx, objAPI)
-
-	nh := newBgHealSequence()
-	// Heal any disk format and metadata early, if possible.
-	if err := nh.healDiskMeta(); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(nh.ctx, err)
-		}
-	}
-
-	globalBackgroundHealState.LaunchNewHealSequence(nh)
-}
-
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) {
@@ -141,24 +128,5 @@ func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpt
 		return madmin.HealResultItem{}, err
 	}
 
-	// Healing succeeded notify the peers to reload format and re-initialize disks.
-	// We will not notify peers if healing is not required.
-	if err == nil {
-		// Notify servers in background and retry if needed.
-		go func() {
-		retry:
-			for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
-				if nerr.Err != nil {
-					if nerr.Err.Error() == errServerNotInitialized.Error() {
-						time.Sleep(time.Second)
-						goto retry
-					}
-					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-					logger.LogIf(ctx, nerr.Err)
-				}
-			}
-		}()
-	}
-
 	return res, nil
 }

cmd/background-newdisks-heal-ops.go

@@ -18,26 +18,34 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
 )
 
-const defaultMonitorNewDiskInterval = time.Minute * 10
+const (
+	defaultMonitorNewDiskInterval = time.Second * 10
+	healingTrackerFilename        = ".healing.bin"
+)
 
-func initLocalDisksAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	go monitorLocalDisksAndHeal(ctx, objAPI)
+//go:generate msgp -file $GOFILE -unexported
+type healingTracker struct {
+	ID string
+
+	// future add more tracking capabilities
 }
 
-// monitorLocalDisksAndHeal - ensures that detected new disks are healed
-//  1. Only the concerned erasure set will be listed and healed
-//  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
-	z, ok := objAPI.(*erasureZones)
+func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
+	z, ok := objAPI.(*erasureServerSets)
 	if !ok {
 		return
 	}
 
+	initBackgroundHealing(ctx, objAPI) // start quick background healing
+
 	var bgSeq *healSequence
 	var found bool
 
@@ -49,75 +57,132 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 		time.Sleep(time.Second)
 	}
 
+	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
+
+	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+			drivesToHeal, defaultMonitorNewDiskInterval))
+
+		// Heal any disk format and metadata early, if possible.
+		if err := bgSeq.healDiskMeta(); err != nil {
+			if newObjectLayerFn() != nil {
+				// log only in situations, when object layer
+				// has fully initialized.
+				logger.LogIf(bgSeq.ctx, err)
+			}
+		}
+	}
+
+	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
+}
+
+func getLocalDisksToHeal() (disksToHeal Endpoints) {
+	for _, ep := range globalEndpoints {
+		for _, endpoint := range ep.Endpoints {
+			if !endpoint.IsLocal {
+				continue
+			}
+			// Try to connect to the current endpoint
+			// and reformat if the current disk is not formatted
+			disk, _, err := connectEndpoint(endpoint)
+			if errors.Is(err, errUnformattedDisk) {
+				disksToHeal = append(disksToHeal, endpoint)
+			} else if err == nil && disk != nil && disk.Healing() {
+				disksToHeal = append(disksToHeal, disk.Endpoint())
+			}
+		}
+	}
+	return disksToHeal
+
+}
+
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	// Run the background healer
+	globalBackgroundHealRoutine = newHealRoutine()
+	go globalBackgroundHealRoutine.run(ctx, objAPI)
+
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence())
+}
+
+// monitorLocalDisksAndHeal - ensures that detected new disks are healed
+//  1. Only the concerned erasure set will be listed and healed
+//  2. Only the node hosting the disk is responsible to perform the heal
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-time.After(defaultMonitorNewDiskInterval):
-			// Attempt a heal as the server starts-up first.
-			localDisksInZoneHeal := make([]Endpoints, len(z.zones))
-			var healNewDisks bool
-			for i, ep := range globalEndpoints {
-				localDisksToHeal := Endpoints{}
-				for _, endpoint := range ep.Endpoints {
-					if !endpoint.IsLocal {
-						continue
-					}
-					// Try to connect to the current endpoint
-					// and reformat if the current disk is not formatted
-					_, _, err := connectEndpoint(endpoint)
-					if err == errUnformattedDisk {
-						localDisksToHeal = append(localDisksToHeal, endpoint)
-					}
+			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
+
+			var erasureSetInZoneDisksToHeal []map[int][]StorageAPI
+
+			healDisks := globalBackgroundHealState.getHealLocalDisks()
+			if len(healDisks) > 0 {
+				// Reformat disks
+				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
+
+				// Ensure that reformatting disks is finished
+				bgSeq.sourceCh <- healSource{bucket: nopHeal}
+
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+					len(healDisks)))
+
+				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverSets))
+				for i := range z.serverSets {
+					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
 				}
-				if len(localDisksToHeal) == 0 {
+			}
+
+			// heal only if new disks found.
+			for _, endpoint := range healDisks {
+				disk, format, err := connectEndpoint(endpoint)
+				if err != nil {
+					printEndpointError(endpoint, err, true)
 					continue
 				}
-				localDisksInZoneHeal[i] = localDisksToHeal
-				healNewDisks = true
-			}
 
-			// Reformat disks only if needed.
-			if !healNewDisks {
-				continue
-			}
-
-			// Reformat disks
-			bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
-
-			// Ensure that reformatting disks is finished
-			bgSeq.sourceCh <- healSource{bucket: nopHeal}
-
-			var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
-			// Compute the list of erasure set to heal
-			for i, localDisksToHeal := range localDisksInZoneHeal {
-				var erasureSetToHeal []int
-				for _, endpoint := range localDisksToHeal {
-					// Load the new format of this passed endpoint
-					_, format, err := connectEndpoint(endpoint)
-					if err != nil {
-						logger.LogIf(ctx, err)
-						continue
-					}
-					// Calculate the set index where the current endpoint belongs
-					setIndex, _, err := findDiskIndex(z.zones[i].format, format)
-					if err != nil {
-						logger.LogIf(ctx, err)
-						continue
-					}
-
-					erasureSetToHeal = append(erasureSetToHeal, setIndex)
+				zoneIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
+				if zoneIdx < 0 {
+					continue
 				}
-				erasureSetInZoneToHeal[i] = erasureSetToHeal
+
+				// Calculate the set index where the current endpoint belongs
+				z.serverSets[zoneIdx].erasureDisksMu.RLock()
+				// Protect reading reference format.
+				setIndex, _, err := findDiskIndex(z.serverSets[zoneIdx].format, format)
+				z.serverSets[zoneIdx].erasureDisksMu.RUnlock()
+				if err != nil {
+					printEndpointError(endpoint, err, false)
+					continue
+				}
+
+				erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk)
 			}
 
-			// Heal all erasure sets that need
-			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
-				for _, setIndex := range erasureSetToHeal {
-					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].drivesPerSet)
-					if err != nil {
-						logger.LogIf(ctx, err)
+			buckets, _ := z.ListBucketsHeal(ctx)
+			for i, setMap := range erasureSetInZoneDisksToHeal {
+				for setIndex, disks := range setMap {
+					for _, disk := range disks {
+						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+
+						lbDisks := z.serverSets[i].sets[setIndex].getOnlineDisks()
+						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
+							logger.LogIf(ctx, err)
+							continue
+						}
+
+						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
+
+						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
+							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
+							logger.LogIf(ctx, err)
+							continue
+						}
+
+						// Only upon success pop the healed disk.
+						globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())
 					}
 				}
 			}

cmd/background-newdisks-heal-ops_gen.go

@@ -0,0 +1,110 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ID":
+			z.ID, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "ID")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 1
+	// write "ID"
+	err = en.Append(0x81, 0xa2, 0x49, 0x44)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.ID)
+	if err != nil {
+		err = msgp.WrapError(err, "ID")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "ID"
+	o = append(o, 0x81, 0xa2, 0x49, 0x44)
+	o = msgp.AppendString(o, z.ID)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ID":
+			z.ID, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ID")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z healingTracker) Msgsize() (s int) {
+	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID)
+	return
+}

cmd/background-newdisks-heal-ops_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalhealingTracker(t *testing.T) {
+	v := healingTracker{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsghealingTracker(b *testing.B) {
+	v := healingTracker{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsghealingTracker(b *testing.B) {
+	v := healingTracker{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalhealingTracker(b *testing.B) {
+	v := healingTracker{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodehealingTracker(t *testing.T) {
+	v := healingTracker{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodehealingTracker Msgsize() is inaccurate")
+	}
+
+	vn := healingTracker{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodehealingTracker(b *testing.B) {
+	v := healingTracker{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodehealingTracker(b *testing.B) {
+	v := healingTracker{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bitrot-streaming.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -80,7 +81,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
 			totalFileSize = bitrotSumsTotalSize + length
 		}
-		err := disk.CreateFile(volume, filePath, totalFileSize, r)
+		err := disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r)
 		r.CloseWithError(err)
 		close(bw.canClose)
 	}()
@@ -118,7 +119,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
-		b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
 		if err != nil {
 			return 0, err
 		}
@@ -139,8 +140,8 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err := &errHashMismatch{fmt.Sprintf("hashes do not match expected %s, got %s",
-			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
+		err := &errHashMismatch{fmt.Sprintf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
 		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}

cmd/bitrot-whole.go

@@ -17,6 +17,8 @@
 package cmd
 
 import (
+	"context"
+	"fmt"
 	"hash"
 	"io"
 
@@ -33,14 +35,14 @@ type wholeBitrotWriter struct {
 }
 
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
-	err := b.disk.AppendFile(b.volume, b.filePath, p)
+	err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	return len(p), nil
@@ -68,15 +70,13 @@ type wholeBitrotReader struct {
 func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) {
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
-		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			ctx := GlobalContext
-			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
-			logger.LogIf(ctx, err)
+		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, errLessData)
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"io"
 	"io/ioutil"
 	"log"
@@ -34,12 +35,12 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	volume := "testvol"
 	filePath := "testfile"
 
-	disk, err := newXLStorage(tmpDir, "")
+	disk, err := newLocalXLStorage(tmpDir)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	disk.MakeVol(volume)
+	disk.MakeVol(context.Background(), volume)
 
 	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
 

cmd/bootstrap-peer-server.go

@@ -18,9 +18,7 @@ package cmd
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -29,7 +27,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
@@ -54,7 +52,7 @@ type bootstrapRESTServer struct{}
 type ServerSystemConfig struct {
 	MinioPlatform  string
 	MinioRuntime   string
-	MinioEndpoints EndpointZones
+	MinioEndpoints EndpointServerSets
 }
 
 // Diff - returns error on first difference found in two configs.
@@ -131,7 +129,7 @@ func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method s
 		values = make(url.Values)
 	}
 
-	respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
+	respBody, err = client.restClient.Call(ctx, method, values, body, length)
 	if err == nil {
 		return respBody, nil
 	}
@@ -161,9 +159,9 @@ func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSyst
 	return srcCfg.Diff(recvCfg)
 }
 
-func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones) error {
+func verifyServerSystemConfig(ctx context.Context, endpointServerSets EndpointServerSets) error {
 	srcCfg := getServerSystemCfg()
-	clnts := newBootstrapRESTClients(endpointZones)
+	clnts := newBootstrapRESTClients(endpointServerSets)
 	var onlineServers int
 	var offlineEndpoints []string
 	var retries int
@@ -178,25 +176,30 @@ func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones)
 			}
 			onlineServers++
 		}
-		// Sleep for a while - so that we don't go into
-		// 100% CPU when half the endpoints are offline.
-		time.Sleep(500 * time.Millisecond)
-		retries++
-		// after 5 retries start logging that servers are not reachable yet
-		if retries >= 5 {
-			logger.Info(fmt.Sprintf("Waiting for atleast %d servers to be online for bootstrap check", len(clnts)/2))
-			logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
-			retries = 0 // reset to log again after 5 retries.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			// Sleep for a while - so that we don't go into
+			// 100% CPU when half the endpoints are offline.
+			time.Sleep(100 * time.Millisecond)
+			retries++
+			// after 5 retries start logging that servers are not reachable yet
+			if retries >= 5 {
+				logger.Info(fmt.Sprintf("Waiting for atleast %d remote servers to be online for bootstrap check", len(clnts)/2))
+				logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
+				retries = 0 // reset to log again after 5 retries.
+			}
+			offlineEndpoints = nil
 		}
-		offlineEndpoints = nil
 	}
 	return nil
 }
 
-func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient {
+func newBootstrapRESTClients(endpointServerSets EndpointServerSets) []*bootstrapRESTClient {
 	seenHosts := set.NewStringSet()
 	var clnts []*bootstrapRESTClient
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if seenHosts.Contains(endpoint.Host) {
 				continue
@@ -220,26 +223,8 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		Path:   bootstrapRESTPath,
 	}
 
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: endpoint.Hostname(),
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
-	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
-	restClient.HealthCheckFn = func() bool {
-		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
-		// Instantiate a new rest client for healthcheck
-		// to avoid recursive healthCheckFn()
-		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
-		xhttp.DrainBody(respBody)
-		cancel()
-		var ne *rest.NetworkError
-		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
-	}
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
+	restClient.HealthCheckFn = nil
 
 	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}
 }

cmd/bucket-encryption.go

@@ -34,7 +34,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 // Get - gets bucket encryption config for the given bucket.
 func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}

cmd/bucket-handlers.go

@@ -30,14 +30,17 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"github.com/minio/minio-go/v6/pkg/set"
-	"github.com/minio/minio-go/v6/pkg/tags"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/replication"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
@@ -46,8 +49,9 @@ import (
 )
 
 const (
-	objectLockConfig    = "object-lock.xml"
-	bucketTaggingConfig = "tagging.xml"
+	objectLockConfig        = "object-lock.xml"
+	bucketTaggingConfig     = "tagging.xml"
+	bucketReplicationConfig = "replication.xml"
 )
 
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -68,7 +72,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
-	if err != nil && err != dns.ErrNoEntriesFound {
+	if err != nil && err != dns.ErrNoEntriesFound && err != dns.ErrNotImplemented {
 		logger.LogIf(GlobalContext, err)
 		return
 	}
@@ -76,33 +80,35 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	bucketsSet := set.NewStringSet()
 	bucketsToBeUpdated := set.NewStringSet()
 	bucketsInConflict := set.NewStringSet()
-	for _, bucket := range buckets {
-		bucketsSet.Add(bucket.Name)
-		r, ok := dnsBuckets[bucket.Name]
-		if !ok {
-			bucketsToBeUpdated.Add(bucket.Name)
-			continue
-		}
-		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-			if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				// No difference in terms of domainIPs and nothing
-				// has changed so we don't change anything on the etcd.
+	if dnsBuckets != nil {
+		for _, bucket := range buckets {
+			bucketsSet.Add(bucket.Name)
+			r, ok := dnsBuckets[bucket.Name]
+			if !ok {
+				bucketsToBeUpdated.Add(bucket.Name)
 				continue
 			}
-			// if domain IPs intersect then it won't be an empty set.
-			// such an intersection means that bucket exists on etcd.
-			// but if we do see a difference with local domain IPs with
-			// hostSlice from etcd then we should update with newer
-			// domainIPs, we proceed to do that here.
-			bucketsToBeUpdated.Add(bucket.Name)
-			continue
+			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+					// No difference in terms of domainIPs and nothing
+					// has changed so we don't change anything on the etcd.
+					continue
+				}
+				// if domain IPs intersect then it won't be an empty set.
+				// such an intersection means that bucket exists on etcd.
+				// but if we do see a difference with local domain IPs with
+				// hostSlice from etcd then we should update with newer
+				// domainIPs, we proceed to do that here.
+				bucketsToBeUpdated.Add(bucket.Name)
+				continue
+			}
+			// No IPs seem to intersect, this means that bucket exists but has
+			// different IP addresses perhaps from a different deployment.
+			// bucket names are globally unique in federation at a given
+			// path prefix, name collision is not allowed. We simply log
+			// an error and continue.
+			bucketsInConflict.Add(bucket.Name)
 		}
-		// No IPs seem to intersect, this means that bucket exists but has
-		// different IP addresses perhaps from a different deployment.
-		// bucket names are globally unique in federation at a given
-		// path prefix, name collision is not allowed. We simply log
-		// an error and continue.
-		bucketsInConflict.Add(bucket.Name)
 	}
 
 	// Add/update buckets that are not registered with the DNS
@@ -377,6 +383,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
+	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
+	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
+
 	// Before proceeding validate if bucket exists.
 	_, err := objectAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
@@ -445,14 +455,18 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
-		Versioned: globalBucketVersioningSys.Enabled(bucket),
+		Versioned:        globalBucketVersioningSys.Enabled(bucket),
+		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
-		dindex := objectsToDelete[deleteList[i]]
+		dindex := objectsToDelete[ObjectToDelete{
+			ObjectName: dObjects[i].ObjectName,
+			VersionID:  dObjects[i].VersionID,
+		}]
 		apiErr := toAPIError(ctx, errs[i])
-		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" {
+		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" {
 			deletedObjects[dindex] = dObjects[i]
 			continue
 		}
@@ -480,19 +494,21 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
+		eventName := event.ObjectRemovedDelete
+
 		objInfo := ObjectInfo{
 			Name:      dobj.ObjectName,
 			VersionID: dobj.VersionID,
 		}
+
 		if dobj.DeleteMarker {
-			objInfo = ObjectInfo{
-				Name:         dobj.ObjectName,
-				DeleteMarker: dobj.DeleteMarker,
-				VersionID:    dobj.DeleteMarkerVersionID,
-			}
+			objInfo.DeleteMarker = dobj.DeleteMarker
+			objInfo.VersionID = dobj.DeleteMarkerVersionID
+			eventName = event.ObjectRemovedDeleteMarkerCreated
 		}
+
 		sendEvent(eventArgs{
-			EventName:    event.ObjectRemovedDelete,
+			EventName:    eventName,
 			BucketName:   bucket,
 			Object:       objInfo,
 			ReqParams:    extractReqParams(r),
@@ -557,7 +573,9 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	if globalDNSConfig != nil {
 		sr, err := globalDNSConfig.Get(bucket)
 		if err != nil {
-			if err == dns.ErrNoEntriesFound {
+			// ErrNotImplemented indicates a DNS backend that doesn't need to check if bucket already
+			// exists elsewhere
+			if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -578,6 +596,16 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 					getObjectLocation(r, globalDomainNames, bucket, ""))
 
 				writeSuccessResponseHeadersOnly(w)
+
+				sendEvent(eventArgs{
+					EventName:    event.BucketCreated,
+					BucketName:   bucket,
+					ReqParams:    extractReqParams(r),
+					RespElements: extractRespElements(w),
+					UserAgent:    r.UserAgent(),
+					Host:         handlers.GetSourceIP(r),
+				})
+
 				return
 			}
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -610,6 +638,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	w.Header().Set(xhttp.Location, path.Clean(r.URL.Path)) // Clean any trailing slashes.
 
 	writeSuccessResponseHeadersOnly(w)
+
+	sendEvent(eventArgs{
+		EventName:    event.BucketCreated,
+		BucketName:   bucket,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
 }
 
 // PostPolicyBucketHandler - POST policy
@@ -631,16 +668,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucket := mux.Vars(r)["bucket"]
 
-	// To detect if the client has disconnected.
-	r.Body = &contextReader{r.Body, r.Context()}
-
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
@@ -739,7 +774,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 		// Make sure formValues adhere to policy restrictions.
 		if err = checkPostPolicy(formValues, postPolicyForm); err != nil {
-			writeCustomErrorResponseXML(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), err.Error(), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrAccessDenied, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -781,7 +816,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
 		// This request header needs to be set prior to setting ObjectOptions
 		if !crypto.SSEC.IsRequested(r.Header) {
-			r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+			r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 		}
 	}
 
@@ -976,7 +1011,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -984,6 +1019,15 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	// Write success response.
 	writeSuccessNoContent(w)
+
+	sendEvent(eventArgs{
+		EventName:    event.BucketRemoved,
+		BucketName:   bucket,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
 }
 
 // PutBucketObjectLockConfigHandler - PUT Bucket object lock configuration.
@@ -1004,7 +1048,10 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -1191,3 +1238,147 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
+
+// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
+// ----------
+// Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
+func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketReplicationConfig")
+	defer logger.AuditLog(w, r, "PutBucketReplicationConfig", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+	// Turn off replication if disk crawl is unavailable.
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketReplicationDisabledError), r.URL)
+		return
+	}
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	replicationConfig, err := replication.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr.Description = err.Error()
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
+		return
+	}
+	sameTarget, err := validateReplicationDestination(ctx, bucket, replicationConfig)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	// Validate the received bucket replication config
+	if err = replicationConfig.Validate(bucket, sameTarget); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	configData, err := xml.Marshal(replicationConfig)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketReplicationConfigHandler - GET Bucket replication configuration.
+// ----------
+// Gets the replication configuration for a bucket.
+func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketReplicationConfig")
+
+	defer logger.AuditLog(w, r, "GetBucketReplicationConfig", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	config, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	configData, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseXML(w, configData)
+}
+
+// DeleteBucketReplicationConfigHandler - DELETE Bucket replication config.
+// ----------
+func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketReplicationConfig")
+	defer logger.AuditLog(w, r, "DeleteBucketReplicationConfig", mustGetClaimsFromToken(r))
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if err := globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, nil); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}

cmd/bucket-handlers_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -45,7 +44,7 @@ func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, a
 	// initialize httptest Recorder, this records any mutations to response writer inside the handler.
 	rec := httptest.NewRecorder()
 	// construct HTTP request for DELETE bucket.
-	req, err := newTestSignedRequestV4("DELETE", getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
+	req, err := newTestSignedRequestV4(http.MethodDelete, getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 	if err != nil {
 		t.Fatalf("Test %s: Failed to create HTTP request for RemoveBucketHandler: <ERROR> %v", instanceType, err)
 	}
@@ -61,7 +60,7 @@ func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, a
 	// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 	recV2 := httptest.NewRecorder()
 	// construct HTTP request for DELETE bucket.
-	reqV2, err := newTestSignedRequestV2("DELETE", getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
+	reqV2, err := newTestSignedRequestV2(http.MethodDelete, getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 	if err != nil {
 		t.Fatalf("Test %s: Failed to create HTTP request for RemoveBucketHandler: <ERROR> %v", instanceType, err)
 	}
@@ -129,7 +128,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get bucket location.
-		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		req, err := newTestSignedRequestV4(http.MethodGet, getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -161,7 +160,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		reqV2, err := newTestSignedRequestV2(http.MethodGet, getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -192,7 +191,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	// Test for Anonymous/unsigned http request.
 	// ListBucketsHandler doesn't support bucket policies, setting the policies shouldn't make any difference.
-	anonReq, err := newTestRequest("GET", getBucketLocationURL("", bucketName), 0, nil)
+	anonReq, err := newTestRequest(http.MethodGet, getBucketLocationURL("", bucketName), 0, nil)
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request.", instanceType)
 	}
@@ -208,7 +207,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
 
 	nilBucket := "dummy-bucket"
-	nilReq, err := newTestRequest("GET", getBucketLocationURL("", nilBucket), 0, nil)
+	nilReq, err := newTestRequest(http.MethodGet, getBucketLocationURL("", nilBucket), 0, nil)
 
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -265,7 +264,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for HEAD bucket.
-		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		req, err := newTestSignedRequestV4(http.MethodHead, getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -280,7 +279,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		reqV2, err := newTestSignedRequestV2(http.MethodHead, getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -295,7 +294,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 	}
 
 	// Test for Anonymous/unsigned http request.
-	anonReq, err := newTestRequest("HEAD", getHEADBucketURL("", bucketName), 0, nil)
+	anonReq, err := newTestRequest(http.MethodHead, getHEADBucketURL("", bucketName), 0, nil)
 
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
@@ -313,7 +312,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
 
 	nilBucket := "dummy-bucket"
-	nilReq, err := newTestRequest("HEAD", getHEADBucketURL("", nilBucket), 0, nil)
+	nilReq, err := newTestRequest(http.MethodHead, getHEADBucketURL("", nilBucket), 0, nil)
 
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -482,7 +481,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// construct HTTP request for List multipart uploads endpoint.
 		u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
-		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		req, gerr := newTestSignedRequestV4(http.MethodGet, u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if gerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
 		}
@@ -499,7 +498,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		reqV2, err := newTestSignedRequestV2(http.MethodGet, u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -516,7 +515,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 	// construct HTTP request for List multipart uploads endpoint.
 	u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
-	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "", nil) // Generate an anonymous request.
+	req, err := newTestSignedRequestV4(http.MethodGet, u, 0, nil, "", "", nil) // Generate an anonymous request.
 	if err != nil {
 		t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
 	}
@@ -530,7 +529,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 	url := getListMultipartUploadsURLWithParams("", testCases[6].bucket, testCases[6].prefix, testCases[6].keyMarker,
 		testCases[6].uploadIDMarker, testCases[6].delimiter, testCases[6].maxUploads)
 	// Test for Anonymous/unsigned http request.
-	anonReq, err := newTestRequest("GET", url, 0, nil)
+	anonReq, err := newTestRequest(http.MethodGet, url, 0, nil)
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
 			instanceType, bucketName, err)
@@ -550,7 +549,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 	url = getListMultipartUploadsURLWithParams("", nilBucket, "dummy-prefix", testCases[6].keyMarker,
 		testCases[6].uploadIDMarker, testCases[6].delimiter, testCases[6].maxUploads)
 
-	nilReq, err := newTestRequest("GET", url, 0, nil)
+	nilReq, err := newTestRequest(http.MethodGet, url, 0, nil)
 
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -596,7 +595,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		req, lerr := newTestSignedRequestV4(http.MethodGet, getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if lerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
 		}
@@ -613,7 +612,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
+		reqV2, err := newTestSignedRequestV2(http.MethodGet, getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -628,7 +627,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 
 	// Test for Anonymous/unsigned http request.
 	// ListBucketsHandler doesn't support bucket policies, setting the policies shouldn't make a difference.
-	anonReq, err := newTestRequest("GET", getListBucketURL(""), 0, nil)
+	anonReq, err := newTestRequest(http.MethodGet, getListBucketURL(""), 0, nil)
 
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request.", instanceType)
@@ -644,7 +643,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
 	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
 
-	nilReq, err := newTestRequest("GET", getListBucketURL(""), 0, nil)
+	nilReq, err := newTestRequest(http.MethodGet, getListBucketURL(""), 0, nil)
 
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -808,10 +807,10 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Generate a signed or anonymous request based on the testCase
 		if testCase.accessKey != "" {
-			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
+			req, err = newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", bucketName),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil)
 		} else {
-			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
+			req, err = newTestRequest(http.MethodPost, getDeleteMultipleObjectsURL("", bucketName),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
 		}
 
@@ -837,7 +836,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Verify whether the bucket obtained object is same as the one created.
 		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
-			fmt.Println(string(testCase.expectedContent), string(actualContent))
+			t.Log(string(testCase.expectedContent), string(actualContent))
 			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
 		}
 	}
@@ -850,7 +849,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	nilBucket := "dummy-bucket"
 	nilObject := ""
 
-	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "", nil)
+	nilReq, err := newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "", nil)
 	if err != nil {
 		t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}

cmd/bucket-lifecycle-handlers_test.go

@@ -50,7 +50,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 	}{
 		// GET empty credentials
 		{
-			method: "GET", bucketName: bucketName,
+			method: http.MethodGet, bucketName: bucketName,
 			accessKey:          "",
 			secretKey:          "",
 			expectedRespStatus: http.StatusForbidden,
@@ -64,7 +64,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 		},
 		// GET wrong credentials
 		{
-			method: "GET", bucketName: bucketName,
+			method: http.MethodGet, bucketName: bucketName,
 			accessKey:          "abcd",
 			secretKey:          "abcd",
 			expectedRespStatus: http.StatusForbidden,
@@ -78,7 +78,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 		},
 		// PUT empty credentials
 		{
-			method:             "PUT",
+			method:             http.MethodPut,
 			bucketName:         bucketName,
 			accessKey:          "",
 			secretKey:          "",
@@ -93,7 +93,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 		},
 		// PUT wrong credentials
 		{
-			method:             "PUT",
+			method:             http.MethodPut,
 			bucketName:         bucketName,
 			accessKey:          "abcd",
 			secretKey:          "abcd",
@@ -108,7 +108,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 		},
 		// DELETE empty credentials
 		{
-			method:             "DELETE",
+			method:             http.MethodDelete,
 			bucketName:         bucketName,
 			accessKey:          "",
 			secretKey:          "",
@@ -123,7 +123,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 		},
 		// DELETE wrong credentials
 		{
-			method:             "DELETE",
+			method:             http.MethodDelete,
 			bucketName:         bucketName,
 			accessKey:          "abcd",
 			secretKey:          "abcd",
@@ -168,7 +168,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 		// Test case - 1.
 		// Filter contains more than (Prefix,Tag,And) rule
 		{
-			method:             "PUT",
+			method:             http.MethodPut,
 			bucketName:         bucketName,
 			accessKey:          creds.AccessKey,
 			secretKey:          creds.SecretKey,
@@ -185,7 +185,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 		},
 		// Date contains wrong format
 		{
-			method:             "PUT",
+			method:             http.MethodPut,
 			bucketName:         bucketName,
 			accessKey:          creds.AccessKey,
 			secretKey:          creds.SecretKey,
@@ -201,7 +201,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			shouldPass: false,
 		},
 		{
-			method:             "PUT",
+			method:             http.MethodPut,
 			bucketName:         bucketName,
 			accessKey:          creds.AccessKey,
 			secretKey:          creds.SecretKey,
@@ -212,7 +212,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			shouldPass:         true,
 		},
 		{
-			method:             "GET",
+			method:             http.MethodGet,
 			accessKey:          creds.AccessKey,
 			secretKey:          creds.SecretKey,
 			bucketName:         bucketName,
@@ -223,7 +223,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			shouldPass:         true,
 		},
 		{
-			method:             "DELETE",
+			method:             http.MethodDelete,
 			accessKey:          creds.AccessKey,
 			secretKey:          creds.SecretKey,
 			bucketName:         bucketName,
@@ -234,7 +234,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			shouldPass:         true,
 		},
 		{
-			method:             "GET",
+			method:             http.MethodGet,
 			accessKey:          creds.AccessKey,
 			secretKey:          creds.SecretKey,
 			bucketName:         bucketName,

cmd/bucket-lifecycle.go

@@ -31,7 +31,7 @@ type LifecycleSys struct{}
 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}

cmd/bucket-listobjects-handlers.go

@@ -18,18 +18,42 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
+	inParallel := func(objects []ObjectInfo) {
+		g := errgroup.WithNErrs(len(objects))
+		for index := range objects {
+			index := index
+			g.Go(func() error {
+				objects[index].ETag = objects[index].GetActualETag(nil)
+				objects[index].Size, _ = objects[index].GetActualSize()
+				return nil
+			}, index)
+		}
+		g.Wait()
+	}
+	const maxConcurrent = 500
+	for {
+		if len(objects) < maxConcurrent {
+			inParallel(objects)
+			return
+		}
+		inParallel(objects[:maxConcurrent])
+		objects = objects[maxConcurrent:]
+	}
+}
+
 // Validate all the ListObjects query arguments, returns an APIErrorCode
 // if one of the args do not meet the required conditions.
 // Special conditions required by MinIO server are as below
@@ -69,7 +93,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -100,16 +124,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	for i := range listObjectVersionsInfo.Objects {
-		if crypto.IsEncrypted(listObjectVersionsInfo.Objects[i].UserDefined) {
-			listObjectVersionsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectVersionsInfo.Objects[i], false)
-		}
-		listObjectVersionsInfo.Objects[i].Size, err = listObjectVersionsInfo.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects)
 
 	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 
@@ -120,7 +135,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
-// of the objects in a bucket. You can use the request parame<ters as selection
+// of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
 // NOTE: It is recommended that this API to be used for application development.
@@ -160,13 +175,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -178,22 +186,10 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-		}
-		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
@@ -246,13 +242,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -264,24 +253,9 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-		}
-		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
-	// The next continuation token has id@node_index format to optimize paginated listing
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
@@ -289,18 +263,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func getLocalNodeIndex() int {
-	if len(globalProxyEndpoints) == 0 {
-		return -1
-	}
-	for i, ep := range globalProxyEndpoints {
-		if ep.IsLocal {
-			return i
-		}
-	}
-	return -1
-}
-
 func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1
@@ -339,8 +301,8 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 	return proxyRequest(ctx, w, r, ep)
 }
 
-func proxyRequestByBucket(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket string) (success bool) {
-	return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(bucket, len(globalProxyEndpoints)))
+func proxyRequestByStringHash(ctx context.Context, w http.ResponseWriter, r *http.Request, str string) (success bool) {
+	return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(str, len(globalProxyEndpoints)))
 }
 
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
@@ -381,7 +343,12 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if proxyRequestByBucket(ctx, w, r, bucket) {
+	// Forward the request using Source IP or bucket
+	forwardStr := handlers.GetSourceIPFromHeaders(r)
+	if forwardStr == "" {
+		forwardStr = bucket
+	}
+	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
 		return
 	}
 
@@ -396,16 +363,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	for i := range listObjectsInfo.Objects {
-		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
-			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-		}
-		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsInfo.Objects)
 
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 

cmd/bucket-metadata-sys.go

@@ -23,11 +23,13 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/minio/minio-go/v6/pkg/tags"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
@@ -47,6 +49,7 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
 	}
 	sys.Lock()
 	delete(sys.metadataMap, bucket)
+	globalBucketMonitor.DeleteBucket(bucket)
 	sys.Unlock()
 }
 
@@ -70,7 +73,7 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
 func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -79,7 +82,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		// This code is needed only for gateway implementations.
 		switch configFile {
 		case bucketSSEConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -88,7 +91,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketLifecycleConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -97,7 +100,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketTaggingConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -106,7 +109,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketNotificationConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -147,12 +150,28 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		meta.EncryptionConfigXML = configData
 	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
-	case objectLockConfig:
-		meta.ObjectLockConfigXML = configData
-	case bucketVersioningConfig:
-		meta.VersioningConfigXML = configData
 	case bucketQuotaConfigFile:
 		meta.QuotaConfigJSON = configData
+	case objectLockConfig:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
+		meta.ObjectLockConfigXML = configData
+	case bucketVersioningConfig:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
+		meta.VersioningConfigXML = configData
+	case bucketReplicationConfig:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
+		meta.ReplicationConfigXML = configData
+	case bucketTargetsFile:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
+		meta.BucketTargetsConfigJSON = configData
 	default:
 		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
@@ -171,6 +190,13 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 // If no metadata exists errConfigNotFound is returned and a new metadata is returned.
 // Only a shallow copy is returned, so referenced data should not be modified,
 // but can be replaced atomically.
+//
+// This function should only be used with
+// - GetBucketInfo
+// - ListBuckets
+// - ListBucketsHeal (only in case of erasure coding mode)
+// For all other bucket specific metadata, use the relevant
+// calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 	if globalIsGateway || bucket == minioMetaBucket {
 		return newBucketMetadata(bucket), errConfigNotFound
@@ -248,9 +274,9 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
-	if globalIsGateway && globalGatewayName == "nas" {
+	if globalIsGateway && globalGatewayName == NASBackendGateway {
 		// Only needed in case of NAS gateway.
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -288,7 +314,7 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -318,10 +344,54 @@ func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota
 	return meta.quotaConfig, nil
 }
 
-// GetConfig returns the current bucket metadata
+// GetReplicationConfig returns configured bucket replication config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketReplicationConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+
+	if meta.replicationConfig == nil {
+		return nil, BucketReplicationConfigNotFound{Bucket: bucket}
+	}
+	return meta.replicationConfig, nil
+}
+
+// GetBucketTargetsConfig returns configured bucket targets for this bucket
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		return nil, err
+	}
+	if meta.bucketTargetConfig == nil {
+		return nil, BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	return meta.bucketTargetConfig, nil
+}
+
+// GetBucketTarget returns the target for the bucket and arn.
+func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) {
+	targets, err := sys.GetBucketTargetsConfig(bucket)
+	if err != nil {
+		return madmin.BucketTarget{}, err
+	}
+	for _, t := range targets.Targets {
+		if t.Arn == arn {
+			return t, nil
+		}
+	}
+	return madmin.BucketTarget{}, errConfigNotFound
+}
+
+// GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}
@@ -362,12 +432,13 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return nil
 	}
 
-	// Load PolicySys once during boot.
-	return sys.load(ctx, buckets, objAPI)
+	// Load bucket metadata sys in background
+	go sys.load(ctx, buckets, objAPI)
+	return nil
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
 	g := errgroup.WithNErrs(len(buckets))
 	for index := range buckets {
 		index := index
@@ -384,22 +455,20 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	}
 	for _, err := range g.Wait() {
 		if err != nil {
-			return err
+			logger.LogIf(ctx, err)
 		}
 	}
-	return nil
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
 	count := 100 // load 100 bucket metadata at a time.
 	for {
 		if len(buckets) < count {
-			return sys.concurrentLoad(ctx, buckets, objAPI)
-		}
-		if err := sys.concurrentLoad(ctx, buckets[:count], objAPI); err != nil {
-			return err
+			sys.concurrentLoad(ctx, buckets, objAPI)
+			return
 		}
+		sys.concurrentLoad(ctx, buckets[:count], objAPI)
 		buckets = buckets[count:]
 	}
 }

cmd/bucket-metadata.go

@@ -20,18 +20,20 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
 	"path"
 	"time"
 
-	"github.com/minio/minio-go/v6/pkg/tags"
+	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/bucket/versioning"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
@@ -59,17 +61,19 @@ var (
 // bucketMetadataFormat refers to the format.
 // bucketMetadataVersion can be used to track a rolling upgrade of a field.
 type BucketMetadata struct {
-	Name                  string
-	Created               time.Time
-	LockEnabled           bool // legacy not used anymore.
-	PolicyConfigJSON      []byte
-	NotificationConfigXML []byte
-	LifecycleConfigXML    []byte
-	ObjectLockConfigXML   []byte
-	VersioningConfigXML   []byte
-	EncryptionConfigXML   []byte
-	TaggingConfigXML      []byte
-	QuotaConfigJSON       []byte
+	Name                    string
+	Created                 time.Time
+	LockEnabled             bool // legacy not used anymore.
+	PolicyConfigJSON        []byte
+	NotificationConfigXML   []byte
+	LifecycleConfigXML      []byte
+	ObjectLockConfigXML     []byte
+	VersioningConfigXML     []byte
+	EncryptionConfigXML     []byte
+	TaggingConfigXML        []byte
+	QuotaConfigJSON         []byte
+	ReplicationConfigXML    []byte
+	BucketTargetsConfigJSON []byte
 
 	// Unexported fields. Must be updated atomically.
 	policyConfig       *policy.Policy
@@ -80,6 +84,8 @@ type BucketMetadata struct {
 	sseConfig          *bucketsse.BucketSSEConfig
 	taggingConfig      *tags.Tags
 	quotaConfig        *madmin.BucketQuota
+	replicationConfig  *replication.Config
+	bucketTargetConfig *madmin.BucketTargets
 }
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
@@ -94,12 +100,17 @@ func newBucketMetadata(name string) BucketMetadata {
 		versioningConfig: &versioning.Versioning{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
+		bucketTargetConfig: &madmin.BucketTargets{},
 	}
 }
 
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+	if name == "" {
+		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
+		return errors.New("bucket name cannot be empty")
+	}
 	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)
 	if err != nil {
@@ -119,21 +130,21 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 	default:
 		return fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
 	}
-
 	// OK, parse data.
 	_, err = b.UnmarshalMsg(data[4:])
+	b.Name = name // in-case parsing failed for some reason, make sure bucket name is not empty.
 	return err
 }
 
 // loadBucketMetadata loads and migrates to bucket metadata.
 func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
-	err := b.Load(ctx, objectAPI, bucket)
+	err := b.Load(ctx, objectAPI, b.Name)
 	if err == nil {
 		return b, b.convertLegacyConfigs(ctx, objectAPI)
 	}
 
-	if err != errConfigNotFound {
+	if !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
 
@@ -213,6 +224,22 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 		}
 	}
 
+	if len(b.ReplicationConfigXML) != 0 {
+		b.replicationConfig, err = replication.ParseConfig(bytes.NewReader(b.ReplicationConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.replicationConfig = nil
+	}
+
+	if len(b.BucketTargetsConfigJSON) != 0 {
+		if err = json.Unmarshal(b.BucketTargetsConfigJSON, b.bucketTargetConfig); err != nil {
+			return err
+		}
+	} else {
+		b.bucketTargetConfig = &madmin.BucketTargets{}
+	}
 	return nil
 }
 
@@ -225,6 +252,8 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		bucketQuotaConfigFile,
 		bucketSSEConfig,
 		bucketTaggingConfig,
+		bucketReplicationConfig,
+		bucketTargetsFile,
 		objectLockConfig,
 	}
 
@@ -242,6 +271,12 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 
 		configData, err := readConfig(ctx, objectAPI, configFile)
 		if err != nil {
+			switch err.(type) {
+			case ObjectExistsAsDirectory:
+				// in FS mode it possible that we have actual
+				// files in this folder with `.minio.sys/buckets/bucket/configFile`
+				continue
+			}
 			if errors.Is(err, errConfigNotFound) {
 				// legacy file config not found, proceed to look for new metadata.
 				continue
@@ -281,6 +316,10 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.VersioningConfigXML = enabledBucketVersioningConfig
 		case bucketQuotaConfigFile:
 			b.QuotaConfigJSON = configData
+		case bucketReplicationConfig:
+			b.ReplicationConfigXML = configData
+		case bucketTargetsFile:
+			b.BucketTargetsConfigJSON = configData
 		}
 	}
 
@@ -315,14 +354,13 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 	if err != nil {
 		return err
 	}
-
 	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
 	return saveConfig(ctx, api, configFile, data)
 }
 
 // deleteBucketMetadata deletes bucket metadata
 // If config does not exist no error is returned.
-func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
+func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string) error {
 	metadataFiles := []string{
 		dataUsageCacheName,
 		bucketMetadataFile,

cmd/bucket-metadata_gen.go

@@ -90,6 +90,18 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "QuotaConfigJSON")
 				return
 			}
+		case "ReplicationConfigXML":
+			z.ReplicationConfigXML, err = dc.ReadBytes(z.ReplicationConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicationConfigXML")
+				return
+			}
+		case "BucketTargetsConfigJSON":
+			z.BucketTargetsConfigJSON, err = dc.ReadBytes(z.BucketTargetsConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -103,9 +115,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 11
+	// map header, size 13
 	// write "Name"
-	err = en.Append(0x8b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -214,15 +226,35 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "QuotaConfigJSON")
 		return
 	}
+	// write "ReplicationConfigXML"
+	err = en.Append(0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.ReplicationConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicationConfigXML")
+		return
+	}
+	// write "BucketTargetsConfigJSON"
+	err = en.Append(0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.BucketTargetsConfigJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigJSON")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 11
+	// map header, size 13
 	// string "Name"
-	o = append(o, 0x8b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -254,6 +286,12 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "QuotaConfigJSON"
 	o = append(o, 0xaf, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
 	o = msgp.AppendBytes(o, z.QuotaConfigJSON)
+	// string "ReplicationConfigXML"
+	o = append(o, 0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.ReplicationConfigXML)
+	// string "BucketTargetsConfigJSON"
+	o = append(o, 0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.BucketTargetsConfigJSON)
 	return
 }
 
@@ -341,6 +379,18 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "QuotaConfigJSON")
 				return
 			}
+		case "ReplicationConfigXML":
+			z.ReplicationConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.ReplicationConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicationConfigXML")
+				return
+			}
+		case "BucketTargetsConfigJSON":
+			z.BucketTargetsConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigJSON")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -355,6 +405,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON)
 	return
 }

cmd/bucket-notification-handlers.go

@@ -17,15 +17,12 @@
 package cmd
 
 import (
-	"encoding/json"
 	"encoding/xml"
 	"io"
 	"net/http"
 	"reflect"
-	"time"
 
 	"github.com/gorilla/mux"
-	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
@@ -173,137 +170,3 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	writeSuccessResponseHeadersOnly(w)
 }
-
-func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListenBucketNotification")
-
-	defer logger.AuditLog(w, r, "ListenBucketNotification", mustGetClaimsFromToken(r))
-
-	// Validate if bucket exists.
-	objAPI := api.ObjectAPI()
-	if objAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	if !objAPI.IsListenBucketSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	vars := mux.Vars(r)
-	bucketName := vars["bucket"]
-
-	values := r.URL.Query()
-	values.Set(peerRESTListenBucket, bucketName)
-
-	var prefix string
-	if len(values[peerRESTListenPrefix]) > 1 {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNamePrefix), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	if len(values[peerRESTListenPrefix]) == 1 {
-		if err := event.ValidateFilterRuleValue(values[peerRESTListenPrefix][0]); err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-
-		prefix = values[peerRESTListenPrefix][0]
-	}
-
-	var suffix string
-	if len(values[peerRESTListenSuffix]) > 1 {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNameSuffix), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	if len(values[peerRESTListenSuffix]) == 1 {
-		if err := event.ValidateFilterRuleValue(values[peerRESTListenSuffix][0]); err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-
-		suffix = values[peerRESTListenSuffix][0]
-	}
-
-	pattern := event.NewPattern(prefix, suffix)
-
-	var eventNames []event.Name
-	for _, s := range values[peerRESTListenEvents] {
-		eventName, err := event.ParseName(s)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-
-		eventNames = append(eventNames, eventName)
-	}
-
-	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()})
-
-	w.Header().Set(xhttp.ContentType, "text/event-stream")
-
-	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
-	// Use buffered channel to take care of burst sends or slow w.Write()
-	listenCh := make(chan interface{}, 4000)
-
-	peers := newPeerRestClients(globalEndpoints)
-
-	globalHTTPListen.Subscribe(listenCh, ctx.Done(), func(evI interface{}) bool {
-		ev, ok := evI.(event.Event)
-		if !ok {
-			return false
-		}
-		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
-			return false
-		}
-		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
-	})
-
-	for _, peer := range peers {
-		if peer == nil {
-			continue
-		}
-		peer.Listen(listenCh, ctx.Done(), values)
-	}
-
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
-	defer keepAliveTicker.Stop()
-
-	enc := json.NewEncoder(w)
-	for {
-		select {
-		case evI := <-listenCh:
-			ev := evI.(event.Event)
-			if len(string(ev.EventName)) > 0 {
-				if err := enc.Encode(struct{ Records []event.Event }{[]event.Event{ev}}); err != nil {
-					return
-				}
-			} else {
-				if _, err := w.Write([]byte(" ")); err != nil {
-					return
-				}
-			}
-			w.(http.Flusher).Flush()
-		case <-keepAliveTicker.C:
-			if _, err := w.Write([]byte(" ")); err != nil {
-				return
-			}
-			w.(http.Flusher).Flush()
-		case <-ctx.Done():
-			return
-		}
-	}
-
-}

cmd/bucket-object-lock.go

@@ -33,7 +33,7 @@ type BucketObjectLockSys struct{}
 // Get - Get retention configuration.
 func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return r, errServerNotInitialized
 		}

cmd/bucket-policy-handlers_test.go

@@ -252,7 +252,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testCase.bucketName),
+		reqV4, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", testCase.bucketName),
 			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -266,7 +266,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testCase.bucketName),
+		reqV2, err := newTestSignedRequestV2(http.MethodPut, getPutPolicyURL("", testCase.bucketName),
 			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -283,7 +283,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
 	bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)
 	// create unsigned HTTP request for PutBucketPolicyHandler.
-	anonReq, err := newTestRequest("PUT", getPutPolicyURL("", bucketName),
+	anonReq, err := newTestRequest(http.MethodPut, getPutPolicyURL("", bucketName),
 		int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)))
 
 	if err != nil {
@@ -302,7 +302,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
 	nilBucket := "dummy-bucket"
 
-	nilReq, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", nilBucket),
+	nilReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", nilBucket),
 		0, nil, "", "", nil)
 
 	if err != nil {
@@ -344,7 +344,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
+		reqV4, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName),
 			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -358,7 +358,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
+		reqV2, err := newTestSignedRequestV2(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName),
 			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -415,7 +415,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV4, err := newTestSignedRequestV4("GET", getGetPolicyURL("", testCase.bucketName),
+		reqV4, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", testCase.bucketName),
 			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
@@ -454,7 +454,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("GET", getGetPolicyURL("", testCase.bucketName),
+		reqV2, err := newTestSignedRequestV2(http.MethodGet, getGetPolicyURL("", testCase.bucketName),
 			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -491,7 +491,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// Test for Anonymous/unsigned http request.
 	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
 	// create unsigned HTTP request for PutBucketPolicyHandler.
-	anonReq, err := newTestRequest("GET", getPutPolicyURL("", bucketName), 0, nil)
+	anonReq, err := newTestRequest(http.MethodGet, getPutPolicyURL("", bucketName), 0, nil)
 
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
@@ -509,7 +509,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
 	nilBucket := "dummy-bucket"
 
-	nilReq, err := newTestSignedRequestV4("GET", getGetPolicyURL("", nilBucket),
+	nilReq, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", nilBucket),
 		0, nil, "", "", nil)
 
 	if err != nil {
@@ -589,7 +589,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
+		reqV4, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName),
 			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -639,7 +639,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
-		reqV4, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", testCase.bucketName),
+		reqV4, err := newTestSignedRequestV4(http.MethodDelete, getDeletePolicyURL("", testCase.bucketName),
 			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -661,7 +661,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
+		reqV2, err := newTestSignedRequestV2(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName),
 			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -678,7 +678,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("DELETE", getDeletePolicyURL("", testCase.bucketName),
+		reqV2, err := newTestSignedRequestV2(http.MethodDelete, getDeletePolicyURL("", testCase.bucketName),
 			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -694,7 +694,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// Test for Anonymous/unsigned http request.
 	// Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference.
 	// create unsigned HTTP request for PutBucketPolicyHandler.
-	anonReq, err := newTestRequest("DELETE", getPutPolicyURL("", bucketName), 0, nil)
+	anonReq, err := newTestRequest(http.MethodDelete, getPutPolicyURL("", bucketName), 0, nil)
 
 	if err != nil {
 		t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": <ERROR> %v",
@@ -712,7 +712,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
 	nilBucket := "dummy-bucket"
 
-	nilReq, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", nilBucket),
+	nilReq, err := newTestSignedRequestV4(http.MethodDelete, getDeletePolicyURL("", nilBucket),
 		0, nil, "", "", nil)
 
 	if err != nil {

cmd/bucket-policy.go

@@ -25,7 +25,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
-	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/policy"
@@ -68,6 +68,12 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	principalType := "Anonymous"
 	if username != "" {
 		principalType = "User"
+		if len(claims) > 0 {
+			principalType = "AssumedRole"
+		}
+		if username == globalActiveCred.AccessKey {
+			principalType = "Account"
+		}
 	}
 
 	vid := r.URL.Query().Get("versionId")
@@ -75,9 +81,6 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		if u, err := url.Parse(r.Header.Get(xhttp.AmzCopySource)); err == nil {
 			vid = u.Query().Get("versionId")
 		}
-		if vid == "" {
-			vid = r.Header.Get(xhttp.AmzCopySourceVersionID)
-		}
 	}
 
 	args := map[string][]string{
@@ -146,7 +149,12 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	for k, v := range claims {
 		vStr, ok := v.(string)
 		if ok {
-			args[k] = []string{vStr}
+			// Special case for AD/LDAP STS users
+			if k == ldapUser {
+				args["user"] = []string{vStr}
+			} else {
+				args[k] = []string{vStr}
+			}
 		}
 	}
 

cmd/bucket-quota.go

@@ -22,9 +22,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -65,48 +63,43 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 }
 
 func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
 
-	q, err := sys.Get(bucket)
-	if err != nil {
-		return nil
-	}
-
-	if q.Type == madmin.FIFOQuota {
-		return nil
-	}
-
-	if q.Quota == 0 {
-		// No quota set return quickly.
-		return nil
-	}
-
 	sys.bucketStorageCache.Once.Do(func() {
-		sys.bucketStorageCache.TTL = 10 * time.Second
+		sys.bucketStorageCache.TTL = 1 * time.Second
 		sys.bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
+			defer done()
 			return loadDataUsageFromBackend(ctx, objAPI)
 		}
 	})
 
-	v, err := sys.bucketStorageCache.Get()
+	q, err := sys.Get(bucket)
 	if err != nil {
 		return err
 	}
 
-	dui := v.(DataUsageInfo)
+	if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
+		v, err := sys.bucketStorageCache.Get()
+		if err != nil {
+			return err
+		}
 
-	bui, ok := dui.BucketsUsage[bucket]
-	if !ok {
-		// bucket not found, cannot enforce quota
-		// call will fail anyways later.
-		return nil
-	}
+		dui := v.(DataUsageInfo)
 
-	if (bui.Size + uint64(size)) > q.Quota {
-		return BucketQuotaExceeded{Bucket: bucket}
+		bui, ok := dui.BucketsUsage[bucket]
+		if !ok {
+			// bucket not found, cannot enforce quota
+			// call will fail anyways later.
+			return nil
+		}
+
+		if (bui.Size + uint64(size)) >= q.Quota {
+			return BucketQuotaExceeded{Bucket: bucket}
+		}
 	}
 
 	return nil
@@ -120,149 +113,112 @@ func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
 	return globalBucketQuotaSys.check(ctx, bucket, size)
 }
 
-const (
-	bgQuotaInterval = 1 * time.Hour
-)
-
-// initQuotaEnforcement starts the routine that deletes objects in bucket
-// that exceeds the FIFO quota
-func initQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
-		go startBucketQuotaEnforcement(ctx, objAPI)
-	}
-}
-
-func startBucketQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.NewTimer(bgQuotaInterval).C:
-			enforceFIFOQuota(ctx, objAPI)
-		}
-
-	}
-}
-
 // enforceFIFOQuota deletes objects in FIFO order until sufficient objects
-// have been deleted so as to bring bucket usage within quota
-func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) {
-	// Turn off quota enforcement if data usage info is unavailable.
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
+// have been deleted so as to bring bucket usage within quota.
+func enforceFIFOQuotaBucket(ctx context.Context, objectAPI ObjectLayer, bucket string, bui BucketUsageInfo) {
+	// Check if the current bucket has quota restrictions, if not skip it
+	cfg, err := globalBucketQuotaSys.Get(bucket)
+	if err != nil {
 		return
 	}
 
-	buckets, err := objectAPI.ListBuckets(ctx)
+	if cfg.Type != madmin.FIFOQuota {
+		return
+	}
+
+	var toFree uint64
+	if bui.Size > cfg.Quota && cfg.Quota > 0 {
+		toFree = bui.Size - cfg.Quota
+	}
+
+	if toFree <= 0 {
+		return
+	}
+
+	// Allocate new results channel to receive ObjectInfo.
+	objInfoCh := make(chan ObjectInfo)
+
+	versioned := globalBucketVersioningSys.Enabled(bucket)
+
+	// Walk through all objects
+	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+
+	// reuse the fileScorer used by disk cache to score entries by
+	// ModTime to find the oldest objects in bucket to delete. In
+	// the context of bucket quota enforcement - number of hits are
+	// irrelevant.
+	scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
 
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
-
-	for _, binfo := range buckets {
-		bucket := binfo.Name
-
-		bui, ok := dataUsageInfo.BucketsUsage[bucket]
-		if !ok {
-			// bucket doesn't exist anymore, or we
-			// do not have any information to proceed.
-			continue
-		}
-
-		// Check if the current bucket has quota restrictions, if not skip it
-		cfg, err := globalBucketQuotaSys.Get(bucket)
-		if err != nil {
-			continue
-		}
-
-		if cfg.Type != madmin.FIFOQuota {
-			continue
-		}
-
-		var toFree uint64
-		if bui.Size > cfg.Quota && cfg.Quota > 0 {
-			toFree = bui.Size - cfg.Quota
-		}
-
-		if toFree == 0 {
-			continue
-		}
-
-		// Allocate new results channel to receive ObjectInfo.
-		objInfoCh := make(chan ObjectInfo)
-
-		versioned := globalBucketVersioningSys.Enabled(bucket)
-
-		// Walk through all objects
-		if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkVersions: versioned}); err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-
-		// reuse the fileScorer used by disk cache to score entries by
-		// ModTime to find the oldest objects in bucket to delete. In
-		// the context of bucket quota enforcement - number of hits are
-		// irrelevant.
-		scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-
-		rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-		for obj := range objInfoCh {
-			if obj.DeleteMarker {
-				// Delete markers are automatically added for FIFO purge.
-				scorer.addFileWithObjInfo(obj, 1)
-				continue
-			}
-			// skip objects currently under retention
-			if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
-				continue
-			}
+	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
+	for obj := range objInfoCh {
+		if obj.DeleteMarker {
+			// Delete markers are automatically added for FIFO purge.
 			scorer.addFileWithObjInfo(obj, 1)
+			continue
+		}
+		// skip objects currently under retention
+		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
+			continue
+		}
+		scorer.addFileWithObjInfo(obj, 1)
+	}
+
+	// If we saw less than quota we are good.
+	if scorer.seenBytes <= cfg.Quota {
+		return
+	}
+	// Calculate how much we want to delete now.
+	toFreeNow := scorer.seenBytes - cfg.Quota
+	// We were less over quota than we thought. Adjust so we delete less.
+	// If we are more over, leave it for the next run to pick up.
+	if toFreeNow < toFree {
+		if !scorer.adjustSaveBytes(int64(toFreeNow) - int64(toFree)) {
+			// We got below or at quota.
+			return
+		}
+	}
+
+	var objects []ObjectToDelete
+	numKeys := len(scorer.fileObjInfos())
+	for i, obj := range scorer.fileObjInfos() {
+		objects = append(objects, ObjectToDelete{
+			ObjectName: obj.Name,
+			VersionID:  obj.VersionID,
+		})
+		if len(objects) < maxDeleteList && (i < numKeys-1) {
+			// skip deletion until maxDeleteList or end of slice
+			continue
 		}
 
-		var objects []ObjectToDelete
-		numKeys := len(scorer.fileObjInfos())
-		for i, obj := range scorer.fileObjInfos() {
-			objects = append(objects, ObjectToDelete{
-				ObjectName: obj.Name,
-				VersionID:  obj.VersionID,
-			})
-			if len(objects) < maxDeleteList && (i < numKeys-1) {
-				// skip deletion until maxDeleteList or end of slice
+		if len(objects) == 0 {
+			break
+		}
+
+		// Deletes a list of objects.
+		_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
+			Versioned: versioned,
+		})
+		for i := range deleteErrs {
+			if deleteErrs[i] != nil {
+				logger.LogIf(ctx, deleteErrs[i])
 				continue
 			}
 
-			if len(objects) == 0 {
-				break
-			}
-
-			// Deletes a list of objects.
-			_, deleteErrs := objectAPI.DeleteObjects(ctx, bucket, objects, ObjectOptions{
-				Versioned: versioned,
+			// Notify object deleted event.
+			sendEvent(eventArgs{
+				EventName:  event.ObjectRemovedDelete,
+				BucketName: bucket,
+				Object:     obj,
+				Host:       "Internal: [FIFO-QUOTA-EXPIRY]",
 			})
-			for i := range deleteErrs {
-				if deleteErrs[i] != nil {
-					logger.LogIf(ctx, deleteErrs[i])
-					continue
-				}
-
-				// Notify object deleted event.
-				sendEvent(eventArgs{
-					EventName:  event.ObjectRemovedDelete,
-					BucketName: bucket,
-					Object:     obj,
-					Host:       "Internal: [FIFO-QUOTA-EXPIRY]",
-				})
-			}
-			objects = nil
 		}
+		objects = nil
 	}
 }

cmd/bucket-replication.go

@@ -0,0 +1,367 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"runtime"
+	"strings"
+	"time"
+
+	miniogo "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/encrypt"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/bandwidth"
+	"github.com/minio/minio/pkg/bucket/replication"
+	"github.com/minio/minio/pkg/event"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
+)
+
+// gets replication config associated to a given bucket name.
+func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+
+		return nil, BucketReplicationConfigNotFound{Bucket: bucketName}
+	}
+
+	return globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
+}
+
+// validateReplicationDestination returns error if replication destination bucket missing or not configured
+// It also returns true if replication destination is same as this server.
+func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
+	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rCfg.RoleArn)
+	if clnt == nil {
+		return false, BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	if found, _ := clnt.BucketExists(ctx, rCfg.GetDestination().Bucket); !found {
+		return false, BucketRemoteDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
+	}
+	if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil {
+		if ret.LockEnabled {
+			lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, rCfg.GetDestination().Bucket)
+			if err != nil || lock != "Enabled" {
+				return false, BucketReplicationDestinationMissingLock{Bucket: rCfg.GetDestination().Bucket}
+			}
+		}
+	}
+	// validate replication ARN against target endpoint
+	c, ok := globalBucketTargetSys.arnRemotesMap[rCfg.RoleArn]
+	if ok {
+		if c.EndpointURL().String() == clnt.EndpointURL().String() {
+			sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+			return sameTarget, nil
+		}
+	}
+	return false, BucketRemoteTargetNotFound{Bucket: bucket}
+}
+
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) bool {
+	if permErr != ErrNone {
+		return false
+	}
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+}
+
+// mustReplicate returns true if object meets replication criteria.
+func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+		return false
+	}
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+}
+
+// mustReplicater returns true if object meets replication criteria.
+func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+	if globalIsGateway {
+		return false
+	}
+	if rs, ok := meta[xhttp.AmzBucketReplicationStatus]; ok {
+		replStatus = rs
+	}
+	if replication.StatusType(replStatus) == replication.Replica {
+		return false
+	}
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return false
+	}
+	opts := replication.ObjectOpts{
+		Name: object,
+		SSEC: crypto.SSEC.IsEncrypted(meta),
+	}
+	tagStr, ok := meta[xhttp.AmzObjectTagging]
+	if ok {
+		opts.UserTags = tagStr
+	}
+	return cfg.Replicate(opts)
+}
+
+func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+	meta := make(map[string]string)
+	for k, v := range objInfo.UserDefined {
+		if k == xhttp.AmzBucketReplicationStatus {
+			continue
+		}
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			continue
+		}
+		meta[k] = v
+	}
+	tag, err := tags.ParseObjectTags(objInfo.UserTags)
+	if err != nil {
+		return
+	}
+	sc := dest.StorageClass
+	if sc == "" {
+		sc = objInfo.StorageClass
+	}
+	putOpts = miniogo.PutObjectOptions{
+		UserMetadata:    meta,
+		UserTags:        tag.ToMap(),
+		ContentType:     objInfo.ContentType,
+		ContentEncoding: objInfo.ContentEncoding,
+		StorageClass:    sc,
+		Internal: miniogo.AdvancedPutOptions{
+			SourceVersionID:   objInfo.VersionID,
+			ReplicationStatus: miniogo.ReplicationStatusReplica,
+			SourceMTime:       objInfo.ModTime,
+			SourceETag:        objInfo.ETag,
+		},
+	}
+	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
+		rmode := miniogo.RetentionMode(mode)
+		putOpts.Mode = rmode
+	}
+	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
+		rdate, err := time.Parse(time.RFC3339, retainDateStr)
+		if err != nil {
+			return
+		}
+		putOpts.RetainUntilDate = rdate
+	}
+	if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
+		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
+	}
+	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
+		putOpts.ServerSideEncryption = encrypt.NewSSE()
+	}
+
+	return
+}
+
+// replicateObject replicates the specified version of the object to destination bucket
+// The source object is then updated to reflect the replication status.
+func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLayer) {
+	bucket := objInfo.Bucket
+	object := objInfo.Name
+
+	cfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
+	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, cfg.RoleArn))
+		return
+	}
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	})
+	if err != nil {
+		return
+	}
+	objInfo = gr.ObjInfo
+	size, err := objInfo.GetActualSize()
+	if err != nil {
+		logger.LogIf(ctx, err)
+		gr.Close()
+		return
+	}
+
+	dest := cfg.GetDestination()
+	if dest.Bucket == "" {
+		gr.Close()
+		return
+	}
+
+	// if heal encounters a pending replication status, either replication
+	// has failed due to server shutdown or crawler and PutObject replication are in contention.
+	healPending := objInfo.ReplicationStatus == replication.Pending
+
+	// In the rare event that replication is in pending state either due to
+	// server shut down/crash before replication completed or healing and PutObject
+	// race - do an additional stat to see if the version ID exists
+	if healPending {
+		_, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
+		if err == nil {
+			gr.Close()
+			// object with same VersionID already exists, replication kicked off by
+			// PutObject might have completed.
+			return
+		}
+	}
+
+	target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
+		return
+	}
+	putOpts := putReplicationOpts(ctx, dest, objInfo)
+	replicationStatus := replication.Complete
+
+	// Setup bandwidth throttling
+	peers, _ := globalEndpoints.peers()
+	totalNodesCount := len(peers)
+	b := target.BandwidthLimit / int64(totalNodesCount)
+	var headerSize int
+	for k, v := range putOpts.Header() {
+		headerSize += len(k) + len(v)
+	}
+	r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+
+	_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
+	r.Close()
+	if err != nil {
+		replicationStatus = replication.Failed
+	}
+	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
+	if objInfo.UserTags != "" {
+		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
+	}
+
+	// FIXME: add support for missing replication events
+	// - event.ObjectReplicationNotTracked
+	// - event.ObjectReplicationMissedThreshold
+	// - event.ObjectReplicationReplicatedAfterThreshold
+	if replicationStatus == replication.Failed {
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationFailed,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
+	}
+
+	objInfo.metadataOnly = true // Perform only metadata updates.
+	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	}, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	}); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
+	}
+}
+
+// filterReplicationStatusMetadata filters replication status metadata for COPY
+func filterReplicationStatusMetadata(metadata map[string]string) map[string]string {
+	// Copy on write
+	dst := metadata
+	var copied bool
+	delKey := func(key string) {
+		if _, ok := metadata[key]; !ok {
+			return
+		}
+		if !copied {
+			dst = make(map[string]string, len(metadata))
+			for k, v := range metadata {
+				dst[k] = v
+			}
+			copied = true
+		}
+		delete(dst, key)
+	}
+
+	delKey(xhttp.AmzBucketReplicationStatus)
+	return dst
+}
+
+type replicationState struct {
+	// add future metrics here
+	replicaCh chan ObjectInfo
+}
+
+func (r *replicationState) queueReplicaTask(oi ObjectInfo) {
+	select {
+	case r.replicaCh <- oi:
+	default:
+	}
+}
+
+var (
+	globalReplicationState *replicationState
+	// TODO: currently keeping it conservative
+	// but eventually can be tuned in future,
+	// take only half the CPUs for replication
+	// conservatively.
+	globalReplicationConcurrent = runtime.GOMAXPROCS(0) / 2
+)
+
+func newReplicationState() *replicationState {
+
+	// fix minimum concurrent replication to 1 for single CPU setup
+	if globalReplicationConcurrent == 0 {
+		globalReplicationConcurrent = 1
+	}
+	rs := &replicationState{
+		replicaCh: make(chan ObjectInfo, 10000),
+	}
+	go func() {
+		<-GlobalContext.Done()
+		close(rs.replicaCh)
+	}()
+	return rs
+}
+
+// addWorker creates a new worker to process tasks
+func (r *replicationState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
+	// Add a new worker.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case oi, ok := <-r.replicaCh:
+				if !ok {
+					return
+				}
+				replicateObject(ctx, oi, objectAPI)
+			}
+		}
+	}()
+}
+
+func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
+	if globalReplicationState == nil {
+		return
+	}
+
+	// Start with globalReplicationConcurrent.
+	for i := 0; i < globalReplicationConcurrent; i++ {
+		globalReplicationState.addWorker(ctx, objectAPI)
+	}
+}

cmd/bucket-targets.go

@@ -0,0 +1,317 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"time"
+
+	minio "github.com/minio/minio-go/v7"
+	miniogo "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio/pkg/bucket/versioning"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+// BucketTargetSys represents bucket targets subsystem
+type BucketTargetSys struct {
+	sync.RWMutex
+	arnRemotesMap map[string]*miniogo.Core
+	targetsMap    map[string][]madmin.BucketTarget
+	clientsCache  map[string]*miniogo.Core
+}
+
+// ListTargets lists bucket targets across tenant or for individual bucket, and returns
+// results filtered by arnType
+func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) {
+	if bucket != "" {
+		if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil {
+			for _, t := range ts.Targets {
+				if string(t.Type) == arnType || arnType == "" {
+					targets = append(targets, t.Clone())
+				}
+			}
+		}
+		return targets
+	}
+	sys.RLock()
+	defer sys.RUnlock()
+	for _, tgts := range sys.targetsMap {
+		for _, t := range tgts {
+			if string(t.Type) == arnType || arnType == "" {
+				targets = append(targets, t.Clone())
+			}
+		}
+	}
+	return
+}
+
+// ListBucketTargets - gets list of bucket targets for this bucket.
+func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	tgts, ok := sys.targetsMap[bucket]
+	if ok {
+		return &madmin.BucketTargets{Targets: tgts}, nil
+	}
+	return nil, BucketRemoteTargetNotFound{Bucket: bucket}
+}
+
+// SetTarget - sets a new minio-go client target for this bucket.
+func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget) error {
+	if globalIsGateway {
+		return nil
+	}
+	if !tgt.Type.IsValid() {
+		return BucketRemoteArnTypeInvalid{Bucket: bucket}
+	}
+	clnt, err := sys.getRemoteTargetClient(tgt)
+	if err != nil {
+		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+	}
+	if tgt.Type == madmin.ReplicationService {
+		if !globalBucketVersioningSys.Enabled(bucket) {
+			return BucketReplicationSourceNotVersioned{Bucket: bucket}
+		}
+		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
+		if err != nil {
+			if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
+				return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+			}
+			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
+		}
+		if vcfg.Status != string(versioning.Enabled) {
+			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
+		}
+	}
+	sys.Lock()
+	defer sys.Unlock()
+
+	tgts := sys.targetsMap[bucket]
+	newtgts := make([]madmin.BucketTarget, len(tgts))
+	found := false
+	for idx, t := range tgts {
+		if t.Type == tgt.Type {
+			if t.Arn == tgt.Arn {
+				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
+			}
+			if t.Label == tgt.Label {
+				return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
+			}
+			newtgts[idx] = *tgt
+			found = true
+			continue
+		}
+		newtgts[idx] = t
+	}
+	if !found {
+		newtgts = append(newtgts, *tgt)
+	}
+
+	sys.targetsMap[bucket] = newtgts
+	sys.arnRemotesMap[tgt.Arn] = clnt
+	if _, ok := sys.clientsCache[clnt.EndpointURL().String()]; !ok {
+		sys.clientsCache[clnt.EndpointURL().String()] = clnt
+	}
+	return nil
+}
+
+// RemoveTarget - removes a remote bucket target for this source bucket.
+func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error {
+	if globalIsGateway {
+		return nil
+	}
+	if arnStr == "" {
+		return BucketRemoteArnInvalid{Bucket: bucket}
+	}
+	arn, err := madmin.ParseARN(arnStr)
+	if err != nil {
+		return BucketRemoteArnInvalid{Bucket: bucket}
+	}
+	if arn.Type == madmin.ReplicationService {
+		// reject removal of remote target if replication configuration is present
+		rcfg, err := getReplicationConfig(ctx, bucket)
+		if err == nil && rcfg.RoleArn == arnStr {
+			if _, ok := sys.arnRemotesMap[arnStr]; ok {
+				return BucketRemoteRemoveDisallowed{Bucket: bucket}
+			}
+		}
+	}
+	// delete ARN type from list of matching targets
+	sys.Lock()
+	defer sys.Unlock()
+	targets := make([]madmin.BucketTarget, 0)
+	found := false
+	tgts := sys.targetsMap[bucket]
+	for _, tgt := range tgts {
+		if tgt.Arn != arnStr {
+			targets = append(targets, tgt)
+			continue
+		}
+		found = true
+	}
+	if !found {
+		return BucketRemoteTargetNotFound{Bucket: bucket}
+	}
+	sys.targetsMap[bucket] = targets
+	delete(sys.arnRemotesMap, arnStr)
+	return nil
+}
+
+// GetRemoteTargetClient returns minio-go client for replication target instance
+func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
+	sys.RLock()
+	defer sys.RUnlock()
+	return sys.arnRemotesMap[arn]
+}
+
+// NewBucketTargetSys - creates new replication system.
+func NewBucketTargetSys() *BucketTargetSys {
+	return &BucketTargetSys{
+		arnRemotesMap: make(map[string]*miniogo.Core),
+		targetsMap:    make(map[string][]madmin.BucketTarget),
+		clientsCache:  make(map[string]*miniogo.Core),
+	}
+}
+
+// Init initializes the bucket targets subsystem for buckets which have targets configured.
+func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// In gateway mode, bucket targets is not supported.
+	if globalIsGateway {
+		return nil
+	}
+
+	// Load bucket targets once during boot in background.
+	go sys.load(ctx, buckets, objAPI)
+	return nil
+}
+
+// UpdateAllTargets updates target to reflect metadata updates
+func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
+	if sys == nil {
+		return
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	if tgts == nil || tgts.Empty() {
+		// remove target and arn association
+		if tgts, ok := sys.targetsMap[bucket]; ok {
+			for _, t := range tgts {
+				delete(sys.arnRemotesMap, t.Arn)
+			}
+		}
+		delete(sys.targetsMap, bucket)
+		return
+	}
+
+	if len(tgts.Targets) > 0 {
+		sys.targetsMap[bucket] = tgts.Targets
+	}
+	for _, tgt := range tgts.Targets {
+		tgtClient, err := sys.getRemoteTargetClient(&tgt)
+		if err != nil {
+			continue
+		}
+		sys.arnRemotesMap[tgt.Arn] = tgtClient
+		if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
+			sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
+		}
+	}
+	sys.targetsMap[bucket] = tgts.Targets
+}
+
+// create minio-go clients for buckets having remote targets
+func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+	for _, bucket := range buckets {
+		cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket.Name)
+		if err != nil {
+			continue
+		}
+		if cfg == nil || cfg.Empty() {
+			continue
+		}
+		if len(cfg.Targets) > 0 {
+			sys.targetsMap[bucket.Name] = cfg.Targets
+		}
+		for _, tgt := range cfg.Targets {
+			tgtClient, err := sys.getRemoteTargetClient(&tgt)
+			if err != nil {
+				continue
+			}
+			sys.arnRemotesMap[tgt.Arn] = tgtClient
+			if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
+				sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
+			}
+		}
+		sys.targetsMap[bucket.Name] = cfg.Targets
+	}
+}
+
+// getRemoteTargetInstanceTransport contains a singleton roundtripper.
+var getRemoteTargetInstanceTransport http.RoundTripper
+var getRemoteTargetInstanceTransportOnce sync.Once
+
+// Returns a minio-go Client configured to access remote host described in replication target config.
+func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
+	if clnt, ok := sys.clientsCache[tcfg.Endpoint]; ok {
+		return clnt, nil
+	}
+	config := tcfg.Credentials
+	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
+
+	getRemoteTargetInstanceTransportOnce.Do(func() {
+		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour)
+	})
+
+	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{
+		Creds:     creds,
+		Secure:    tcfg.Secure,
+		Transport: getRemoteTargetInstanceTransport,
+	})
+	return core, err
+}
+
+// getRemoteARN gets existing ARN for an endpoint or generates a new one.
+func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) string {
+	if target == nil {
+		return ""
+	}
+	tgts := sys.targetsMap[bucket]
+	for _, tgt := range tgts {
+		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL() == tgt.URL() {
+			return tgt.Arn
+		}
+	}
+	if !madmin.ServiceType(target.Type).IsValid() {
+		return ""
+	}
+	arn := madmin.ARN{
+		Type:   target.Type,
+		ID:     mustGetUUID(),
+		Region: target.Region,
+		Bucket: target.TargetBucket,
+	}
+	return arn.String()
+}

cmd/bucket-versioning-handler.go

@@ -70,6 +70,14 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		}, r.URL, guessIsBrowserReq(r))
 		return
 	}
+	if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "InvalidBucketState",
+			Description:    "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
+			HTTPStatusCode: http.StatusConflict,
+		}, r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	configData, err := xml.Marshal(v)
 	if err != nil {

cmd/common-main.go

@@ -20,16 +20,21 @@ import (
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
+	"fmt"
+	"math/rand"
 	"net"
+	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
@@ -40,15 +45,14 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	// Initialize globalConsoleSys system
-	globalConsoleSys = NewConsoleLogger(GlobalContext)
-	logger.AddTarget(globalConsoleSys)
+	rand.Seed(time.Now().UTC().UnixNano())
+	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
 
 	gob.Register(StorageErr(""))
 }
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
-	if (globalAutoEncryption || GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
+	if (GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
 		logger.Fatal(errInvalidArgument,
 			"Encryption support is requested but '%s' does not support encryption", name)
 	}
@@ -68,17 +72,40 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 
 // Check for updates and print a notification message
 func checkUpdate(mode string) {
-	// Its OK to ignore any errors during doUpdate() here.
-	if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil {
-		if updateMsg == "" {
-			return
-		}
-		if globalInplaceUpdateDisabled {
-			logStartupMessage(updateMsg)
-		} else {
-			logStartupMessage(prepareUpdateMessage("Run `mc admin update`", latestReleaseTime.Sub(currentReleaseTime)))
-		}
+	updateURL := minioReleaseInfoURL
+	if runtime.GOOS == globalWindowsOSName {
+		updateURL = minioReleaseWindowsInfoURL
 	}
+
+	u, err := url.Parse(updateURL)
+	if err != nil {
+		return
+	}
+
+	// Its OK to ignore any errors during doUpdate() here.
+	crTime, err := GetCurrentReleaseTime()
+	if err != nil {
+		return
+	}
+
+	_, lrTime, err := getLatestReleaseTime(u, 2*time.Second, mode)
+	if err != nil {
+		return
+	}
+
+	var older time.Duration
+	var downloadURL string
+	if lrTime.After(crTime) {
+		older = lrTime.Sub(crTime)
+		downloadURL = getDownloadURL(releaseTimeToReleaseTag(lrTime))
+	}
+
+	updateMsg := prepareUpdateMessage(downloadURL, older)
+	if updateMsg == "" {
+		return
+	}
+
+	logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -223,7 +250,11 @@ func handleCommonEnvVars() {
 	} else {
 		// Add found interfaces IP address to global domain IPS,
 		// loopback addresses will be naturally dropped.
-		updateDomainIPs(mustGetLocalIP4())
+		domainIPs := mustGetLocalIP4()
+		for _, host := range globalEndpoints.Hostnames() {
+			domainIPs.Add(host)
+		}
+		updateDomainIPs(domainIPs)
 	}
 
 	// In place update is true by default if the MINIO_UPDATE is not set
@@ -260,7 +291,7 @@ func logStartupMessage(msg string) {
 	logger.StartupMessage(msg)
 }
 
-func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
+func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, false, nil
 	}
@@ -269,11 +300,71 @@ func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn b
 		return nil, nil, false, err
 	}
 
-	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
+	manager, err = certs.NewManager(GlobalContext, getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
 	if err != nil {
 		return nil, nil, false, err
 	}
 
+	// MinIO has support for multiple certificates. It expects the following structure:
+	//  certs/
+	//   │
+	//   ├─ public.crt
+	//   ├─ private.key
+	//   │
+	//   ├─ example.com/
+	//   │   │
+	//   │   ├─ public.crt
+	//   │   └─ private.key
+	//   └─ foobar.org/
+	//      │
+	//      ├─ public.crt
+	//      └─ private.key
+	//   ...
+	//
+	// Therefore, we read all filenames in the cert directory and check
+	// for each directory whether it contains a public.crt and private.key.
+	// If so, we try to add it to certificate manager.
+	root, err := os.Open(globalCertsDir.Get())
+	if err != nil {
+		return nil, nil, false, err
+	}
+	defer root.Close()
+
+	files, err := root.Readdir(-1)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	for _, file := range files {
+		// Ignore all
+		// - regular files
+		// - "CAs" directory
+		// - any directory which starts with ".."
+		if file.Mode().IsRegular() || file.Name() == "CAs" || strings.HasPrefix(file.Name(), "..") {
+			continue
+		}
+		if file.Mode()&os.ModeSymlink == os.ModeSymlink {
+			file, err = os.Stat(filepath.Join(root.Name(), file.Name()))
+			if err != nil {
+				// not accessible ignore
+				continue
+			}
+			if !file.IsDir() {
+				continue
+			}
+		}
+
+		var (
+			certFile = filepath.Join(root.Name(), file.Name(), publicCertFile)
+			keyFile  = filepath.Join(root.Name(), file.Name(), privateKeyFile)
+		)
+		if !isFile(certFile) || !isFile(keyFile) {
+			continue
+		}
+		if err = manager.AddCertificate(certFile, keyFile); err != nil {
+			err = fmt.Errorf("Unable to load TLS certificate '%s,%s': %w", certFile, keyFile, err)
+			logger.LogIf(GlobalContext, err, logger.Minio)
+		}
+	}
 	secureConn = true
-	return x509Certs, c, secureConn, nil
+	return x509Certs, manager, secureConn, nil
 }

cmd/config-common.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 )
 
@@ -36,8 +35,6 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 			return nil, errConfigNotFound
 		}
 
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
 		return nil, err
 	}
 
@@ -49,7 +46,11 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 	return buffer.Bytes(), nil
 }
 
-func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+type objectDeleter interface {
+	DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
+}
+
+func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error {
 	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound
@@ -74,8 +75,6 @@ func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) err
 			return errConfigNotFound
 		}
 
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
 		return err
 	}
 	return nil

cmd/config-current.go

@@ -25,8 +25,9 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/heal"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
@@ -55,6 +56,7 @@ func initHelp() {
 		config.KmsKesSubSys:         crypto.DefaultKesKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
+		config.HealSubSys:           heal.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -92,7 +94,7 @@ func initHelp() {
 		},
 		config.HelpKV{
 			Key:         config.PolicyOPASubSys,
-			Description: "enable external OPA for policy enforcement",
+			Description: "[DEPRECATED] enable external OPA for policy enforcement",
 		},
 		config.HelpKV{
 			Key:         config.KmsVaultSubSys,
@@ -106,6 +108,10 @@ func initHelp() {
 			Key:         config.APISubSys,
 			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
 		},
+		config.HelpKV{
+			Key:         config.HealSubSys,
+			Description: "manage object healing frequency and bitrot verification checks",
+		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
 			Description:     "send server logs to webhook endpoints",
@@ -185,6 +191,7 @@ func initHelp() {
 		config.EtcdSubSys:           etcd.Help,
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
+		config.HealSubSys:           heal.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -213,7 +220,7 @@ var (
 	globalServerConfigMu sync.RWMutex
 )
 
-func validateConfig(s config.Config) error {
+func validateConfig(s config.Config, setDriveCount int) error {
 	// Disable merging env values with config for validation.
 	env.SetEnvOff()
 
@@ -233,8 +240,7 @@ func validateConfig(s config.Config) error {
 	}
 
 	if globalIsErasure {
-		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
-			globalErasureSetDriveCount); err != nil {
+		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
 			return err
 		}
 	}
@@ -247,6 +253,10 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
+	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
+		return err
+	}
+
 	{
 		etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 		if err != nil {
@@ -307,11 +317,10 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
-	return notify.TestNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(),
-		globalNotificationSys.ConfiguredTargetIDs())
+	return notify.TestNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs())
 }
 
-func lookupConfigs(s config.Config) {
+func lookupConfigs(s config.Config, setDriveCount int) {
 	ctx := GlobalContext
 
 	var err error
@@ -323,6 +332,19 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
+	if dnsURL, dnsUser, dnsPass, ok := env.LookupEnv(config.EnvDNSWebhook); ok {
+		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
+			dns.Authentication(dnsUser, dnsPass),
+			dns.RootCAs(globalRootCAs))
+		if err != nil {
+			if globalIsGateway {
+				logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+			}
+		}
+	}
+
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		if globalIsGateway {
@@ -344,19 +366,25 @@ func lookupConfigs(s config.Config) {
 			}
 		}
 
-		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil && globalDNSConfig == nil {
-			globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
-				dns.DomainNames(globalDomainNames),
-				dns.DomainIPs(globalDomainIPs),
-				dns.DomainPort(globalMinioPort),
-				dns.CoreDNSPath(etcdCfg.CoreDNSPath),
-			)
-			if err != nil {
-				if globalIsGateway {
-					logger.FatalIf(err, "Unable to initialize DNS config")
-				} else {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-						globalDomainNames, err))
+		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+			if globalDNSConfig != nil {
+				// if global DNS is already configured, indicate with a warning, incase
+				// users are confused.
+				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
+			} else {
+				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
+					dns.DomainNames(globalDomainNames),
+					dns.DomainIPs(globalDomainIPs),
+					dns.DomainPort(globalMinioPort),
+					dns.CoreDNSPath(etcdCfg.CoreDNSPath),
+				)
+				if err != nil {
+					if globalIsGateway {
+						logger.FatalIf(err, "Unable to initialize DNS config")
+					} else {
+						logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+							globalDomainNames, err))
+					}
 				}
 			}
 		}
@@ -379,11 +407,15 @@ func lookupConfigs(s config.Config) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	globalAPIConfig.init(apiConfig)
+	globalAPIConfig.init(apiConfig, setDriveCount)
+
+	// Initialize remote instance transport once.
+	getRemoteInstanceTransportOnce.Do(func() {
+		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
+	})
 
 	if globalIsErasure {
-		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
-			globalErasureSetDriveCount)
+		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
 		}
@@ -406,6 +438,10 @@ func lookupConfigs(s config.Config) {
 			}
 		}
 	}
+	globalHealConfig, err = heal.LookupConfig(s[config.HealSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to read heal config: %w", err))
+	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 	if err != nil {
@@ -419,6 +455,9 @@ func lookupConfigs(s config.Config) {
 
 	// Enable auto-encryption if enabled
 	globalAutoEncryption = kmsCfg.AutoEncryption
+	if globalAutoEncryption && !globalIsGateway {
+		logger.LogIf(ctx, fmt.Errorf("%s env is deprecated please migrate to using `mc encrypt` at bucket level", crypto.EnvKMSAutoEncryption))
+	}
 
 	globalCompressConfig, err = compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 	if err != nil {
@@ -454,40 +493,48 @@ func lookupConfigs(s config.Config) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
 	}
 
-	for _, l := range loggerCfg.HTTP {
+	for k, l := range loggerCfg.HTTP {
 		if l.Enabled {
 			// Enable http logging
-			logger.AddTarget(
-				http.New(http.WithEndpoint(l.Endpoint),
+			if err = logger.AddTarget(
+				http.New(
+					http.WithTargetName(k),
+					http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
 					http.WithTransport(NewGatewayHTTPTransport()),
 				),
-			)
+			); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
+			}
 		}
 	}
 
-	for _, l := range loggerCfg.Audit {
+	for k, l := range loggerCfg.Audit {
 		if l.Enabled {
 			// Enable http audit logging
-			logger.AddAuditTarget(
-				http.New(http.WithEndpoint(l.Endpoint),
+			if err = logger.AddAuditTarget(
+				http.New(
+					http.WithTargetName(k),
+					http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
 					http.WithTransport(NewGatewayHTTPTransport()),
 				),
-			)
+			); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
+			}
 		}
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(), false)
+	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport(), true)
+	globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), NewGatewayHTTPTransport(), true)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
@@ -601,7 +648,7 @@ func loadConfig(objAPI ObjectLayer) error {
 	}
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg)
+	lookupConfigs(srvCfg, objAPI.SetDriveCount())
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/config-encrypted.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"time"
 	"unicode/utf8"
@@ -194,7 +193,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 				// Config is already encrypted with right keys
 				continue
 			}
-			return errors.New("config data not in plain-text form or encrypted")
+			return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 		}
 
 		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -274,7 +273,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 					// Config is already encrypted with right keys
 					continue
 				}
-				return errors.New("config data not in plain-text form or encrypted")
+				return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 			}
 
 			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)

cmd/config-migrate.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/cache"
@@ -2506,7 +2507,7 @@ func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (
 		return false, nil, err
 	}
 
-	if globalConfigEncrypted {
+	if globalConfigEncrypted && !utf8.Valid(data) {
 		data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 		if err != nil {
 			if err == madmin.ErrMaliciousData {

cmd/config.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"sort"
 	"strings"
+	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
@@ -63,7 +64,7 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 				if err != nil {
 					return nil, err
 				}
-				if globalConfigEncrypted {
+				if globalConfigEncrypted && !utf8.Valid(data) {
 					data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 					if err != nil {
 						return nil, err
@@ -102,7 +103,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
 		return nil, err
 	}
 
-	if globalConfigEncrypted {
+	if globalConfigEncrypted && !utf8.Valid(data) {
 		data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 	}
 
@@ -155,7 +156,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 		return nil, err
 	}
 
-	if globalConfigEncrypted {
+	if globalConfigEncrypted && !utf8.Valid(configData) {
 		configData, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(configData))
 		if err != nil {
 			if err == madmin.ErrMaliciousData {

cmd/config/api/api.go

@@ -29,15 +29,28 @@ import (
 
 // API sub-system constants
 const (
-	apiRequestsMax      = "requests_max"
-	apiRequestsDeadline = "requests_deadline"
-	apiReadyDeadline    = "ready_deadline"
-	apiCorsAllowOrigin  = "cors_allow_origin"
+	apiRequestsMax             = "requests_max"
+	apiRequestsDeadline        = "requests_deadline"
+	apiClusterDeadline         = "cluster_deadline"
+	apiCorsAllowOrigin         = "cors_allow_origin"
+	apiRemoteTransportDeadline = "remote_transport_deadline"
+	apiListQuorum              = "list_quorum"
+	apiExtendListCacheLife     = "extend_list_cache_life"
 
-	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
-	EnvAPICorsAllowOrigin  = "MINIO_API_CORS_ALLOW_ORIGIN"
+	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
+	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
+	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
+	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
+	EnvAPIExtendListCacheLife     = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
+	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS"
+)
+
+// Deprecated key and ENVs
+const (
+	apiReadyDeadline    = "ready_deadline"
+	EnvAPIReadyDeadline = "MINIO_API_READY_DEADLINE"
 )
 
 // DefaultKVS - default storage class config
@@ -52,22 +65,37 @@ var (
 			Value: "10s",
 		},
 		config.KV{
-			Key:   apiReadyDeadline,
+			Key:   apiClusterDeadline,
 			Value: "10s",
 		},
 		config.KV{
 			Key:   apiCorsAllowOrigin,
 			Value: "*",
 		},
+		config.KV{
+			Key:   apiRemoteTransportDeadline,
+			Value: "2h",
+		},
+		config.KV{
+			Key:   apiListQuorum,
+			Value: "optimal",
+		},
+		config.KV{
+			Key:   apiExtendListCacheLife,
+			Value: "0s",
+		},
 	}
 )
 
 // Config storage class configuration
 type Config struct {
-	APIRequestsMax      int           `json:"requests_max"`
-	APIRequestsDeadline time.Duration `json:"requests_deadline"`
-	APIReadyDeadline    time.Duration `json:"ready_deadline"`
-	APICorsAllowOrigin  []string      `json:"cors_allow_origin"`
+	RequestsMax             int           `json:"requests_max"`
+	RequestsDeadline        time.Duration `json:"requests_deadline"`
+	ClusterDeadline         time.Duration `json:"cluster_deadline"`
+	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
+	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
+	ListQuorum              string        `json:"list_strict_quorum"`
+	ExtendListLife          time.Duration `json:"extend_list_cache_life"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -81,8 +109,29 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 	return json.Unmarshal(data, &aux)
 }
 
+// GetListQuorum interprets list quorum values and returns appropriate
+// acceptable quorum expected for list operations
+func (sCfg Config) GetListQuorum() int {
+	switch sCfg.ListQuorum {
+	case "optimal":
+		return 3
+	case "reduced":
+		return 2
+	case "disk":
+		// smallest possible value, generally meant for testing.
+		return 1
+	case "strict":
+		return -1
+	}
+	// Defaults to 3 drives per set.
+	return 3
+}
+
 // LookupConfig - lookup api config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	// remove this since we have removed this already.
+	kvs.Delete(apiReadyDeadline)
+
 	if err = config.CheckValidKeys(config.APISubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
@@ -102,16 +151,37 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
-	readyDeadline, err := time.ParseDuration(env.Get(EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	clusterDeadline, err := time.ParseDuration(env.Get(EnvAPIClusterDeadline, kvs.Get(apiClusterDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
 	corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
+
+	remoteTransportDeadline, err := time.ParseDuration(env.Get(EnvAPIRemoteTransportDeadline, kvs.Get(apiRemoteTransportDeadline)))
+	if err != nil {
+		return cfg, err
+	}
+
+	listQuorum := env.Get(EnvAPIListQuorum, kvs.Get(apiListQuorum))
+	switch listQuorum {
+	case "strict", "optimal", "reduced", "disk":
+	default:
+		return cfg, errors.New("invalid value for list strict quorum")
+	}
+
+	listLife, err := time.ParseDuration(env.Get(EnvAPIExtendListCacheLife, kvs.Get(apiExtendListCacheLife)))
+	if err != nil {
+		return cfg, err
+	}
+
 	return Config{
-		APIRequestsMax:      requestsMax,
-		APIRequestsDeadline: requestsDeadline,
-		APIReadyDeadline:    readyDeadline,
-		APICorsAllowOrigin:  corsAllowOrigin,
+		RequestsMax:             requestsMax,
+		RequestsDeadline:        requestsDeadline,
+		ClusterDeadline:         clusterDeadline,
+		CorsAllowOrigin:         corsAllowOrigin,
+		RemoteTransportDeadline: remoteTransportDeadline,
+		ListQuorum:              listQuorum,
+		ExtendListLife:          listLife,
 	}, nil
 }

cmd/config/api/help.go

@@ -39,5 +39,11 @@ var (
 			Optional:    true,
 			Type:        "csv",
 		},
+		config.HelpKV{
+			Key:         apiRemoteTransportDeadline,
+			Description: `set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"`,
+			Optional:    true,
+			Type:        "duration",
+		},
 	}
 )

cmd/config/cache/config.go

@@ -28,16 +28,17 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled       bool     `json:"-"`
-	Drives        []string `json:"drives"`
-	Expiry        int      `json:"expiry"`
-	MaxUse        int      `json:"maxuse"`
-	Quota         int      `json:"quota"`
-	Exclude       []string `json:"exclude"`
-	After         int      `json:"after"`
-	WatermarkLow  int      `json:"watermark_low"`
-	WatermarkHigh int      `json:"watermark_high"`
-	Range         bool     `json:"range"`
+	Enabled         bool     `json:"-"`
+	Drives          []string `json:"drives"`
+	Expiry          int      `json:"expiry"`
+	MaxUse          int      `json:"maxuse"`
+	Quota           int      `json:"quota"`
+	Exclude         []string `json:"exclude"`
+	After           int      `json:"after"`
+	WatermarkLow    int      `json:"watermark_low"`
+	WatermarkHigh   int      `json:"watermark_high"`
+	Range           bool     `json:"range"`
+	CommitWriteback bool     `json:"-"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -152,3 +153,13 @@ func parseCacheExcludes(excludes string) ([]string, error) {
 
 	return excludesSlice, nil
 }
+
+func parseCacheCommitMode(commitStr string) (bool, error) {
+	switch strings.ToLower(commitStr) {
+	case "writeback":
+		return true, nil
+	case "writethrough":
+		return false, nil
+	}
+	return false, config.ErrInvalidCacheCommitValue(nil).Msg("cache commit value must be `writeback` or `writethrough`")
+}

cmd/config/cache/help.go

@@ -74,5 +74,11 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         Commit,
+			Description: `set to control cache commit behavior, defaults to "writethrough"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -35,6 +35,7 @@ const (
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
 	Range         = "range"
+	Commit        = "commit"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -45,6 +46,7 @@ const (
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 	EnvCacheRange         = "MINIO_CACHE_RANGE"
+	EnvCacheCommit        = "MINIO_CACHE_COMMIT"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -53,6 +55,7 @@ const (
 	DefaultAfter         = "0"
 	DefaultWaterMarkLow  = "70"
 	DefaultWaterMarkHigh = "80"
+	DefaultCacheCommit   = "writethrough"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -90,6 +93,10 @@ var (
 			Key:   Range,
 			Value: config.EnableOn,
 		},
+		config.KV{
+			Key:   Commit,
+			Value: DefaultCacheCommit,
+		},
 	}
 )
 
@@ -210,6 +217,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		}
 		cfg.Range = rng
 	}
+	if commit := env.Get(EnvCacheCommit, kvs.Get(Commit)); commit != "" {
+		cfg.CommitWriteback, err = parseCacheCommitMode(commit)
+		if err != nil {
+			return cfg, err
+		}
+	}
 
 	return cfg, nil
 }

cmd/config/certs.go

@@ -24,8 +24,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"io/ioutil"
-	"os"
-	"path"
 
 	"github.com/minio/minio/pkg/env"
 )
@@ -69,38 +67,6 @@ func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 	return x509Certs, nil
 }
 
-// GetRootCAs - returns all the root CAs into certPool
-// at the input certsCADir
-func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
-	rootCAs, _ := x509.SystemCertPool()
-	if rootCAs == nil {
-		// In some systems (like Windows) system cert pool is
-		// not supported or no certificates are present on the
-		// system - so we create a new cert pool.
-		rootCAs = x509.NewCertPool()
-	}
-
-	fis, err := ioutil.ReadDir(certsCAsDir)
-	if err != nil {
-		if os.IsNotExist(err) || os.IsPermission(err) {
-			// Return success if CA's directory is missing or permission denied.
-			err = nil
-		}
-		return rootCAs, err
-	}
-
-	// Load all custom CA files.
-	for _, fi := range fis {
-		caCert, err := ioutil.ReadFile(path.Join(certsCAsDir, fi.Name()))
-		if err != nil {
-			// ignore files which are not readable.
-			continue
-		}
-		rootCAs.AppendCertsFromPEM(caCert)
-	}
-	return rootCAs, nil
-}
-
 // LoadX509KeyPair - load an X509 key pair (private key , certificate)
 // from the provided paths. The private key may be encrypted and is
 // decrypted using the ENV_VAR: MINIO_CERT_PASSWD.

cmd/config/certs_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"runtime"
 	"testing"
 )
@@ -194,60 +193,6 @@ M9ofSEt/bdRD
 	}
 }
 
-func TestGetRootCAs(t *testing.T) {
-	emptydir, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", emptydir)
-	}
-	defer os.RemoveAll(emptydir)
-
-	dir1, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir1)
-	}
-	defer os.RemoveAll(dir1)
-	if err = os.Mkdir(filepath.Join(dir1, "empty-dir"), 0755); err != nil {
-		t.Fatalf("Unable create empty dir. %v", err)
-	}
-
-	dir2, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir2)
-	}
-	defer os.RemoveAll(dir2)
-	if err = ioutil.WriteFile(filepath.Join(dir2, "empty-file"), []byte{}, 0644); err != nil {
-		t.Fatalf("Unable create test file. %v", err)
-	}
-
-	testCases := []struct {
-		certCAsDir  string
-		expectedErr error
-	}{
-		// ignores non-existent directories.
-		{"nonexistent-dir", nil},
-		// Ignores directories.
-		{dir1, nil},
-		// Ignore empty directory.
-		{emptydir, nil},
-		// Loads the cert properly.
-		{dir2, nil},
-	}
-
-	for _, testCase := range testCases {
-		_, err := GetRootCAs(testCase.certCAsDir)
-
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-	}
-}
-
 func TestLoadX509KeyPair(t *testing.T) {
 	for i, testCase := range loadX509KeyPairTests {
 		privateKey, err := createTempFile("private.key", testCase.privateKey)

cmd/config/config.go

@@ -24,7 +24,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
@@ -76,6 +76,7 @@ const (
 	KmsKesSubSys         = "kms_kes"
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
+	HealSubSys           = "heal"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -112,6 +113,7 @@ var SubSystems = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
 	NotifyKafkaSubSys,
@@ -138,6 +140,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	HealSubSys,
 }...)
 
 // Constant separators
@@ -264,6 +267,16 @@ func (kvs KVS) Get(key string) string {
 	return ""
 }
 
+// Delete - deletes the key if present from the KV list.
+func (kvs *KVS) Delete(key string) {
+	for i, kv := range *kvs {
+		if kv.Key == key {
+			*kvs = append((*kvs)[:i], (*kvs)[i+1:]...)
+			return
+		}
+	}
+}
+
 // Lookup - lookup a key in a list of KVS
 func (kvs KVS) Lookup(key string) (string, bool) {
 	for _, kv := range kvs {
@@ -448,6 +461,11 @@ func (c Config) Merge() Config {
 					ckvs.Set(kv.Key, kv.Value)
 				}
 			}
+			if _, ok := cp[subSys]; !ok {
+				// A config subsystem was removed or server was downgraded.
+				Logger.Info("config: ignoring unknown subsystem config %q\n", subSys)
+				continue
+			}
 			cp[subSys][tgt] = ckvs
 		}
 	}

cmd/config/constants.go

@@ -31,11 +31,13 @@ const (
 	EnvDomain       = "MINIO_DOMAIN"
 	EnvRegionName   = "MINIO_REGION_NAME"
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
-	EnvEndpoints    = "MINIO_ENDPOINTS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
+	EnvArgs         = "MINIO_ARGS"
+	EnvDNSWebhook   = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
 	EnvUpdate = "MINIO_UPDATE"
 
-	EnvWorm   = "MINIO_WORM"   // legacy
-	EnvRegion = "MINIO_REGION" // legacy
+	EnvEndpoints = "MINIO_ENDPOINTS" // legacy
+	EnvWorm      = "MINIO_WORM"      // legacy
+	EnvRegion    = "MINIO_REGION"    // legacy
 )

cmd/config/dns/etcd_dns.go

@@ -26,9 +26,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-go/v6/pkg/set"
-
 	"github.com/coredns/coredns/plugin/etcd/msg"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"go.etcd.io/etcd/v3/clientv3"
 )
 
@@ -48,8 +47,9 @@ func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, err
 }
 
 // Close closes the internal etcd client and cannot be used further
-func (c *CoreDNS) Close() {
+func (c *CoreDNS) Close() error {
 	c.etcdClient.Close()
+	return nil
 }
 
 // List - Retrieves list of DNS entries for the domain.
@@ -213,6 +213,11 @@ func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
 	return nil
 }
 
+// String stringer name for this implementation of dns.Store
+func (c *CoreDNS) String() string {
+	return "etcdDNS"
+}
+
 // CoreDNS - represents dns config for coredns server.
 type CoreDNS struct {
 	domainNames []string
@@ -222,13 +227,13 @@ type CoreDNS struct {
 	etcdClient  *clientv3.Client
 }
 
-// Option - functional options pattern style
-type Option func(*CoreDNS)
+// EtcdOption - functional options pattern style
+type EtcdOption func(*CoreDNS)
 
 // DomainNames set a list of domain names used by this CoreDNS
 // client setting, note this will fail if set to empty when
 // constructor initializes.
-func DomainNames(domainNames []string) Option {
+func DomainNames(domainNames []string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainNames = domainNames
 	}
@@ -236,14 +241,14 @@ func DomainNames(domainNames []string) Option {
 
 // DomainIPs set a list of custom domain IPs, note this will
 // fail if set to empty when constructor initializes.
-func DomainIPs(domainIPs set.StringSet) Option {
+func DomainIPs(domainIPs set.StringSet) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainIPs = domainIPs
 	}
 }
 
 // DomainPort - is a string version of server port
-func DomainPort(domainPort string) Option {
+func DomainPort(domainPort string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainPort = domainPort
 	}
@@ -252,14 +257,14 @@ func DomainPort(domainPort string) Option {
 // CoreDNSPath - custom prefix on etcd to populate DNS
 // service records, optional and can be empty.
 // if empty then c.prefixPath is used i.e "/skydns"
-func CoreDNSPath(prefix string) Option {
+func CoreDNSPath(prefix string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.prefixPath = prefix
 	}
 }
 
 // NewCoreDNS - initialize a new coreDNS set/unset values.
-func NewCoreDNS(cfg clientv3.Config, setters ...Option) (*CoreDNS, error) {
+func NewCoreDNS(cfg clientv3.Config, setters ...EtcdOption) (Store, error) {
 	etcdClient, err := clientv3.New(cfg)
 	if err != nil {
 		return nil, err

cmd/config/dns/operator_dns.go

@@ -0,0 +1,234 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dns
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dgrijalva/jwt-go"
+	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
+)
+
+var (
+	defaultOperatorContextTimeout = 10 * time.Second
+	// ErrNotImplemented - Indicates the functionality which is not implemented
+	ErrNotImplemented = errors.New("The method is not implemented")
+)
+
+func (c *OperatorDNS) addAuthHeader(r *http.Request) error {
+	if c.username == "" || c.password == "" {
+		return nil
+	}
+
+	claims := &jwt.StandardClaims{
+		ExpiresAt: int64(15 * time.Minute),
+		Issuer:    c.username,
+		Subject:   config.EnvDNSWebhook,
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
+	ss, err := token.SignedString([]byte(c.password))
+	if err != nil {
+		return err
+	}
+
+	r.Header.Set("Authorization", "Bearer "+ss)
+	return nil
+}
+
+func (c *OperatorDNS) endpoint(bucket string, delete bool) (string, error) {
+	u, err := url.Parse(c.Endpoint)
+	if err != nil {
+		return "", err
+	}
+	q := u.Query()
+	q.Add("bucket", bucket)
+	q.Add("delete", strconv.FormatBool(delete))
+	u.RawQuery = q.Encode()
+	return u.String(), nil
+}
+
+// Put - Adds DNS entries into operator webhook server
+func (c *OperatorDNS) Put(bucket string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
+	defer cancel()
+	e, err := c.endpoint(bucket, false)
+	if err != nil {
+		return newError(bucket, err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
+	if err != nil {
+		return newError(bucket, err)
+	}
+	if err = c.addAuthHeader(req); err != nil {
+		return newError(bucket, err)
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		if derr := c.Delete(bucket); derr != nil {
+			return newError(bucket, derr)
+		}
+	}
+	var errorStringBuilder strings.Builder
+	io.Copy(&errorStringBuilder, io.LimitReader(resp.Body, resp.ContentLength))
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		errorString := errorStringBuilder.String()
+		return newError(bucket, fmt.Errorf("service create for bucket %s, failed with status %s, error %s", bucket, resp.Status, errorString))
+	}
+	return nil
+}
+
+func newError(bucket string, err error) error {
+	e := Error{bucket, err}
+	if strings.Contains(err.Error(), "invalid bucket name") {
+		return ErrInvalidBucketName(e)
+	}
+	return e
+}
+
+// Delete - Removes DNS entries added in Put().
+func (c *OperatorDNS) Delete(bucket string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
+	defer cancel()
+	e, err := c.endpoint(bucket, true)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
+	if err != nil {
+		return err
+	}
+	if err = c.addAuthHeader(req); err != nil {
+		return err
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request to delete the service for bucket %s, failed with status %s", bucket, resp.Status)
+	}
+	return nil
+}
+
+// DeleteRecord - Removes a specific DNS entry
+// No Op for Operator because operator deals on with bucket entries
+func (c *OperatorDNS) DeleteRecord(record SrvRecord) error {
+	return ErrNotImplemented
+}
+
+// Close closes the internal http client
+func (c *OperatorDNS) Close() error {
+	c.httpClient.CloseIdleConnections()
+	return nil
+}
+
+// List - Retrieves list of DNS entries for the domain.
+// This is a No Op for Operator because, there is no intent to enforce global
+// namespace at MinIO level with this DNS entry. The global namespace in
+// enforced by the Kubernetes Operator
+func (c *OperatorDNS) List() (srvRecords map[string][]SrvRecord, err error) {
+	return nil, ErrNotImplemented
+}
+
+// Get - Retrieves DNS records for a bucket.
+// This is a No Op for Operator because, there is no intent to enforce global
+// namespace at MinIO level with this DNS entry. The global namespace in
+// enforced by the Kubernetes Operator
+func (c *OperatorDNS) Get(bucket string) (srvRecords []SrvRecord, err error) {
+	return nil, ErrNotImplemented
+}
+
+// String stringer name for this implementation of dns.Store
+func (c *OperatorDNS) String() string {
+	return "webhookDNS"
+}
+
+// OperatorDNS - represents dns config for MinIO k8s operator.
+type OperatorDNS struct {
+	httpClient *http.Client
+	Endpoint   string
+	rootCAs    *x509.CertPool
+	username   string
+	password   string
+}
+
+// OperatorOption - functional options pattern style for OperatorDNS
+type OperatorOption func(*OperatorDNS)
+
+// Authentication - custom username and password for authenticating at the endpoint
+func Authentication(username, password string) OperatorOption {
+	return func(args *OperatorDNS) {
+		args.username = username
+		args.password = password
+	}
+}
+
+// RootCAs - add custom trust certs pool
+func RootCAs(CAs *x509.CertPool) OperatorOption {
+	return func(args *OperatorDNS) {
+		args.rootCAs = CAs
+	}
+}
+
+// NewOperatorDNS - initialize a new K8S Operator DNS set/unset values.
+func NewOperatorDNS(endpoint string, setters ...OperatorOption) (Store, error) {
+	if endpoint == "" {
+		return nil, errors.New("invalid argument")
+	}
+
+	args := &OperatorDNS{
+		Endpoint: endpoint,
+	}
+	for _, setter := range setters {
+		setter(args)
+	}
+	args.httpClient = &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   3 * time.Second,
+				KeepAlive: 5 * time.Second,
+			}).DialContext,
+			ResponseHeaderTimeout: 3 * time.Second,
+			TLSHandshakeTimeout:   3 * time.Second,
+			ExpectContinueTimeout: 3 * time.Second,
+			TLSClientConfig: &tls.Config{
+				RootCAs: args.rootCAs,
+			},
+			// Go net/http automatically unzip if content-type is
+			// gzip disable this feature, as we are always interested
+			// in raw stream.
+			DisableCompression: true,
+		},
+	}
+	return args, nil
+}

cmd/config/dns/store.go

@@ -0,0 +1,44 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dns
+
+// Error - DNS related errors error.
+type Error struct {
+	Bucket string
+	Err    error
+}
+
+// ErrInvalidBucketName for buckets with invalid name
+type ErrInvalidBucketName Error
+
+func (e ErrInvalidBucketName) Error() string {
+	return "invalid bucket name error: " + e.Err.Error()
+}
+func (e Error) Error() string {
+	return "dns related error: " + e.Err.Error()
+}
+
+// Store dns record store
+type Store interface {
+	Put(bucket string) error
+	Get(bucket string) ([]SrvRecord, error)
+	Delete(bucket string) error
+	List() (map[string][]SrvRecord, error)
+	DeleteRecord(record SrvRecord) error
+	Close() error
+	String() string
+}

cmd/config/errors-utils.go

@@ -19,7 +19,6 @@ package config
 import (
 	"errors"
 	"fmt"
-	"io"
 	"net"
 	"syscall"
 
@@ -111,18 +110,16 @@ func ErrorToErr(err error) Err {
 		case *net.OpError:
 			return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
 		}
-		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
-	} else if errors.Is(err, io.ErrUnexpectedEOF) {
-		return ErrUnexpectedDataContent(err)
-	} else {
-		// Failed to identify what type of error this, return a simple UI error
-		return Err{msg: err.Error()}
 	}
+
+	// Failed to identify what type of error this, return a simple UI error
+	return Err{msg: err.Error()}
 }
 
 // FmtError converts a fatal error message to a more clear error
 // using some colors
 func FmtError(introMsg string, err error, jsonFlag bool) string {
+
 	renderedTxt := ""
 	uiErr := ErrorToErr(err)
 	// JSON print

cmd/config/errors.go

@@ -102,6 +102,12 @@ var (
 		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
 	)
 
+	ErrInvalidCacheCommitValue = newErrFn(
+		"Invalid cache commit value",
+		"Please check the passed value",
+		"MINIO_CACHE_COMMIT: Valid expected value is `writeback` or `writethrough`",
+	)
+
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",
@@ -187,12 +193,6 @@ Example 1:
 		"",
 	)
 
-	ErrCorruptedBackend = newErrFn(
-		"Unable to use the specified backend, pre-existing content detected",
-		"Please ensure your disk mount does not have any pre-existing content",
-		"",
-	)
-
 	ErrUnableToWriteInBackend = newErrFn(
 		"Unable to write to the backend",
 		"Please ensure MinIO binary has write permissions for the backend",
@@ -211,12 +211,6 @@ Example 1:
 		`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
 	)
 
-	ErrNoPermissionsToAccessDirFiles = newErrFn(
-		"Missing permissions to access the specified path",
-		"Please ensure the specified path can be accessed",
-		"",
-	)
-
 	ErrSSLUnexpectedError = newErrFn(
 		"Invalid TLS certificate",
 		"Please check the content of your certificate data",
@@ -253,12 +247,6 @@ Example 1:
 		"",
 	)
 
-	ErrUnexpectedDataContent = newErrFn(
-		"Unexpected data content",
-		"Please contact MinIO at https://slack.min.io",
-		"",
-	)
-
 	ErrUnexpectedError = newErrFn(
 		"Unexpected error",
 		"Please contact MinIO at https://slack.min.io",

cmd/config/heal/heal.go

@@ -0,0 +1,67 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package heal
+
+import (
+	"errors"
+
+	"github.com/minio/minio/cmd/config"
+)
+
+// Compression environment variables
+const (
+	Bitrot = "bitrotscan"
+)
+
+// Config represents the heal settings.
+type Config struct {
+	// Bitrot will perform bitrot scan on local disk when checking objects.
+	Bitrot bool `json:"bitrotscan"`
+}
+
+var (
+	// DefaultKVS - default KV config for heal settings
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   Bitrot,
+			Value: config.EnableOff,
+		},
+	}
+
+	// Help provides help for config values
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         Bitrot,
+			Description: `perform bitrot scan on disks when checking objects during crawl`,
+			Optional:    true,
+			Type:        "on|off",
+		},
+	}
+)
+
+// LookupConfig - lookup config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+	bitrot := kvs.Get(Bitrot)
+	if bitrot != config.EnableOn && bitrot != config.EnableOff {
+		return cfg, errors.New(Bitrot + ": must be 'on' or 'off'")
+	}
+	cfg.Bitrot = bitrot == config.EnableOn
+	return cfg, nil
+}

cmd/config/identity/openid/jwks.go

@@ -22,11 +22,9 @@ import (
 	"crypto/elliptic"
 	"crypto/rsa"
 	"encoding/base64"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"math/big"
-	"strings"
 )
 
 // JWKS - https://tools.ietf.org/html/rfc7517
@@ -47,15 +45,6 @@ type JWKS struct {
 	K   string `json:"k,omitempty"`
 }
 
-func safeDecode(str string) ([]byte, error) {
-	lenMod4 := len(str) % 4
-	if lenMod4 > 0 {
-		str = str + strings.Repeat("=", 4-lenMod4)
-	}
-
-	return base64.URLEncoding.DecodeString(str)
-}
-
 var (
 	errMalformedJWKRSAKey = errors.New("malformed JWK RSA key")
 	errMalformedJWKECKey  = errors.New("malformed JWK EC key")
@@ -70,29 +59,24 @@ func (key *JWKS) DecodePublicKey() (crypto.PublicKey, error) {
 		}
 
 		// decode exponent
-		data, err := safeDecode(key.E)
+		ebuf, err := base64.RawURLEncoding.DecodeString(key.E)
 		if err != nil {
 			return nil, errMalformedJWKRSAKey
 		}
 
-		if len(data) < 4 {
-			ndata := make([]byte, 4)
-			copy(ndata[4-len(data):], data)
-			data = ndata
-		}
-
-		pubKey := &rsa.PublicKey{
-			N: &big.Int{},
-			E: int(binary.BigEndian.Uint32(data[:])),
-		}
-
-		data, err = safeDecode(key.N)
+		nbuf, err := base64.RawURLEncoding.DecodeString(key.N)
 		if err != nil {
 			return nil, errMalformedJWKRSAKey
 		}
-		pubKey.N.SetBytes(data)
 
-		return pubKey, nil
+		var n, e big.Int
+		n.SetBytes(nbuf)
+		e.SetBytes(ebuf)
+
+		return &rsa.PublicKey{
+			E: int(e.Int64()),
+			N: &n,
+		}, nil
 	case "EC":
 		if key.Crv == "" || key.X == "" || key.Y == "" {
 			return nil, errMalformedJWKECKey
@@ -112,25 +96,25 @@ func (key *JWKS) DecodePublicKey() (crypto.PublicKey, error) {
 			return nil, fmt.Errorf("Unknown curve type: %s", key.Crv)
 		}
 
-		pubKey := &ecdsa.PublicKey{
+		xbuf, err := base64.RawURLEncoding.DecodeString(key.X)
+		if err != nil {
+			return nil, errMalformedJWKECKey
+		}
+
+		ybuf, err := base64.RawURLEncoding.DecodeString(key.Y)
+		if err != nil {
+			return nil, errMalformedJWKECKey
+		}
+
+		var x, y big.Int
+		x.SetBytes(xbuf)
+		y.SetBytes(ybuf)
+
+		return &ecdsa.PublicKey{
 			Curve: curve,
-			X:     &big.Int{},
-			Y:     &big.Int{},
-		}
-
-		data, err := safeDecode(key.X)
-		if err != nil {
-			return nil, errMalformedJWKECKey
-		}
-		pubKey.X.SetBytes(data)
-
-		data, err = safeDecode(key.Y)
-		if err != nil {
-			return nil, errMalformedJWKECKey
-		}
-		pubKey.Y.SetBytes(data)
-
-		return pubKey, nil
+			X:     &x,
+			Y:     &y,
+		}, nil
 	default:
 		return nil, fmt.Errorf("Unknown JWK key type %s", key.Kty)
 	}

cmd/config/identity/openid/jwks_test.go

@@ -26,6 +26,29 @@ import (
 	"testing"
 )
 
+func TestAzurePublicKey(t *testing.T) {
+	const jsonkey = `{"keys":[{"kty":"RSA","use":"sig","kid":"SsZsBNhZcF3Q9S4trpQBTByNRRI","x5t":"SsZsBNhZcF3Q9S4trpQBTByNRRI","n":"uHPewhg4WC3eLVPkEFlj7RDtaKYWXCI5G-LPVzsMKOuIu7qQQbeytIA6P6HT9_iIRt8zNQvuw4P9vbNjgUCpI6vfZGsjk3XuCVoB_bAIhvuBcQh9ePH2yEwS5reR-NrG1PsqzobnZZuigKCoDmuOb_UDx1DiVyNCbMBlEG7UzTQwLf5NP6HaRHx027URJeZvPAWY7zjHlSOuKoS_d1yUveaBFIgZqPWLCg44ck4gvik45HsNVWT9zYfT74dvUSSrMSR-SHFT7Hy1XjbVXpHJHNNAXpPoGoWXTuc0BxMsB4cqjfJqoftFGOG4x32vEzakArLPxAKwGvkvu0jToAyvSQ","e":"AQAB","x5c":"MIIDBTCCAe2gAwIBAgIQWHw7h/Ysh6hPcXpnrJ0N8DANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDQyNzAwMDAwMFoXDTI1MDQyNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALhz3sIYOFgt3i1T5BBZY+0Q7WimFlwiORviz1c7DCjriLu6kEG3srSAOj+h0/f4iEbfMzUL7sOD/b2zY4FAqSOr32RrI5N17glaAf2wCIb7gXEIfXjx9shMEua3kfjaxtT7Ks6G52WbooCgqA5rjm/1A8dQ4lcjQmzAZRBu1M00MC3+TT+h2kR8dNu1ESXmbzwFmO84x5UjriqEv3dclL3mgRSIGaj1iwoOOHJOIL4pOOR7DVVk/c2H0++Hb1EkqzEkfkhxU+x8tV421V6RyRzTQF6T6BqFl07nNAcTLAeHKo3yaqH7RRjhuMd9rxM2pAKyz8QCsBr5L7tI06AMr0kCAwEAAaMhMB8wHQYDVR0OBBYEFOI7M+DDFMlP7Ac3aomPnWo1QL1SMA0GCSqGSIb3DQEBCwUAA4IBAQBv+8rBiDY8sZDBoUDYwFQM74QjqCmgNQfv5B0Vjwg20HinERjQeH24uAWzyhWN9++FmeY4zcRXDY5UNmB0nJz7UGlprA9s7voQ0Lkyiud0DO072RPBg38LmmrqoBsLb3MB9MZ2CGBaHftUHfpdTvrgmXSP0IJn7mCUq27g+hFk7n/MLbN1k8JswEODIgdMRvGqN+mnrPKkviWmcVAZccsWfcmS1pKwXqICTKzd6WmVdz+cL7ZSd9I2X0pY4oRwauoE2bS95vrXljCYgLArI3XB2QcnglDDBRYu3Z3aIJb26PTIyhkVKT7xaXhXl4OgrbmQon9/O61G2dzpjzzBPqNP","issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"huN95IvPfehq34GzBDZ1GXGirnM","x5t":"huN95IvPfehq34GzBDZ1GXGirnM","n":"6lldKm5Rc_vMKa1RM_TtUv3tmtj52wLRrJqu13yGM3_h0dwru2ZP53y65wDfz6_tLCjoYuRCuVsjoW37-0zXUORJvZ0L90CAX-58lW7NcE4bAzA1pXv7oR9kQw0X8dp0atU4HnHeaTU8LZxcjJO79_H9cxgwa-clKfGxllcos8TsuurM8xi2dx5VqwzqNMB2s62l3MTN7AzctHUiQCiX2iJArGjAhs-mxS1wmyMIyOSipdodhjQWRAcseW-aFVyRTFVi8okl2cT1HJjPXdx0b1WqYSOzeRdrrLUcA0oR2Tzp7xzOYJZSGNnNLQqa9f6h6h52XbX0iAgxKgEDlRpbJw","e":"AQAB","x5c":["MIIDBTCCAe2gAwIBAgIQPCxFbySVSLZOggeWRzBWOjANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDYwNzAwMDAwMFoXDTI1MDYwNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOpZXSpuUXP7zCmtUTP07VL97ZrY+dsC0ayartd8hjN/4dHcK7tmT+d8uucA38+v7Swo6GLkQrlbI6Ft+/tM11DkSb2dC/dAgF/ufJVuzXBOGwMwNaV7+6EfZEMNF/HadGrVOB5x3mk1PC2cXIyTu/fx/XMYMGvnJSnxsZZXKLPE7LrqzPMYtnceVasM6jTAdrOtpdzEzewM3LR1IkAol9oiQKxowIbPpsUtcJsjCMjkoqXaHYY0FkQHLHlvmhVckUxVYvKJJdnE9RyYz13cdG9VqmEjs3kXa6y1HANKEdk86e8czmCWUhjZzS0KmvX+oeoedl219IgIMSoBA5UaWycCAwEAAaMhMB8wHQYDVR0OBBYEFFXP0ODFhjf3RS6oRijM5Tb+yB8CMA0GCSqGSIb3DQEBCwUAA4IBAQB9GtVikLTbJWIu5x9YCUTTKzNhi44XXogP/v8VylRSUHI5YTMdnWwvDIt/Y1sjNonmSy9PrioEjcIiI1U8nicveafMwIq5VLn+gEY2lg6KDJAzgAvA88CXqwfHHvtmYBovN7goolp8TY/kddMTf6TpNzN3lCTM2MK4Ye5xLLVGdp4bqWCOJ/qjwDxpTRSydYIkLUDwqNjv+sYfOElJpYAB4rTL/aw3ChJ1iaA4MtXEt6OjbUtbOa21lShfLzvNRbYK3+ukbrhmRl9lemJEeUls51vPuIe+jg+Ssp43aw7PQjxt4/MpfNMS2BfZ5F8GVSVG7qNb352cLLeJg5rc398Z"],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"M6pX7RHoraLsprfJeRCjSxuURhc","x5t":"M6pX7RHoraLsprfJeRCjSxuURhc","n":"xHScZMPo8FifoDcrgncWQ7mGJtiKhrsho0-uFPXg-OdnRKYudTD7-Bq1MDjcqWRf3IfDVjFJixQS61M7wm9wALDj--lLuJJ9jDUAWTA3xWvQLbiBM-gqU0sj4mc2lWm6nPfqlyYeWtQcSC0sYkLlayNgX4noKDaXivhVOp7bwGXq77MRzeL4-9qrRYKjuzHfZL7kNBCsqO185P0NI2Jtmw-EsqYsrCaHsfNRGRrTvUHUq3hWa859kK_5uNd7TeY2ZEwKVD8ezCmSfR59ZzyxTtuPpkCSHS9OtUvS3mqTYit73qcvprjl3R8hpjXLb8oftfpWr3hFRdpxrwuoQEO4QQ","e":"AQAB","x5c":["MIIC8TCCAdmgAwIBAgIQfEWlTVc1uINEc9RBi6qHMjANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwHhcNMTgxMDE0MDAwMDAwWhcNMjAxMDE0MDAwMDAwWjAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEdJxkw+jwWJ+gNyuCdxZDuYYm2IqGuyGjT64U9eD452dEpi51MPv4GrUwONypZF/ch8NWMUmLFBLrUzvCb3AAsOP76Uu4kn2MNQBZMDfFa9AtuIEz6CpTSyPiZzaVabqc9+qXJh5a1BxILSxiQuVrI2BfiegoNpeK+FU6ntvAZervsxHN4vj72qtFgqO7Md9kvuQ0EKyo7Xzk/Q0jYm2bD4SypiysJoex81EZGtO9QdSreFZrzn2Qr/m413tN5jZkTApUPx7MKZJ9Hn1nPLFO24+mQJIdL061S9LeapNiK3vepy+muOXdHyGmNctvyh+1+laveEVF2nGvC6hAQ7hBAgMBAAGjITAfMB0GA1UdDgQWBBQ5TKadw06O0cvXrQbXW0Nb3M3h/DANBgkqhkiG9w0BAQsFAAOCAQEAI48JaFtwOFcYS/3pfS5+7cINrafXAKTL+/+he4q+RMx4TCu/L1dl9zS5W1BeJNO2GUznfI+b5KndrxdlB6qJIDf6TRHh6EqfA18oJP5NOiKhU4pgkF2UMUw4kjxaZ5fQrSoD9omjfHAFNjradnHA7GOAoF4iotvXDWDBWx9K4XNZHWvD11Td66zTg5IaEQDIZ+f8WS6nn/98nAVMDtR9zW7Te5h9kGJGfe6WiHVaGRPpBvqC4iypGHjbRwANwofZvmp5wP08hY1CsnKY5tfP+E2k/iAQgKKa6QoxXToYvP7rsSkglak8N5g/+FJGnq4wP6cOzgZpjdPMwaVt5432GA=="],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"}]}`
+
+	var jk JWKS
+	if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil {
+		t.Fatal("Unmarshal: ", err)
+	} else if len(jk.Keys) != 3 {
+		t.Fatalf("Expected 3 keys, got %d", len(jk.Keys))
+	}
+
+	var kids []string
+	for ii, jks := range jk.Keys {
+		_, err := jks.DecodePublicKey()
+		if err != nil {
+			t.Fatalf("Failed to decode key %d: %v", ii, err)
+		}
+		kids = append(kids, jks.Kid)
+	}
+	if len(kids) != 3 {
+		t.Fatalf("Failed to find the expected number of kids: 3, got %d", len(kids))
+	}
+}
+
 // A.1 - Example public keys
 func TestPublicKey(t *testing.T) {
 	const jsonkey = `{"keys":

cmd/config/identity/openid/jwt.go

@@ -25,6 +25,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
@@ -49,10 +50,14 @@ type Config struct {
 	publicKeys   map[string]crypto.PublicKey
 	transport    *http.Transport
 	closeRespFn  func(io.ReadCloser)
+	mutex        *sync.Mutex
 }
 
 // PopulatePublicKey - populates a new publickey from the JWKS URL.
 func (r *Config) PopulatePublicKey() error {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
 	if r.JWKS.URL == nil || r.JWKS.URL.String() == "" {
 		return nil
 	}
@@ -123,8 +128,8 @@ func GetDefaultExpiration(dsecs string) (time.Duration, error) {
 
 		// The duration, in seconds, of the role session.
 		// The value can range from 900 seconds (15 minutes)
-		// to 12 hours.
-		if expirySecs < 900 || expirySecs > 43200 {
+		// up to 7 days.
+		if expirySecs < 900 || expirySecs > 604800 {
 			return 0, auth.ErrInvalidDuration
 		}
 
@@ -185,6 +190,8 @@ func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 	var claims jwtgo.MapClaims
 	jwtToken, err := jp.ParseWithClaims(token, &claims, keyFuncCallback)
 	if err != nil {
+		// Re-populate the public key in-case the JWKS
+		// pubkeys are refreshed
 		if err = p.PopulatePublicKey(); err != nil {
 			return nil, err
 		}
@@ -203,7 +210,6 @@ func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 	}
 
 	return claims, nil
-
 }
 
 // ID returns the provider name and authentication type.
@@ -324,6 +330,7 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		ClientID:    env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
 		transport:   transport,
 		closeRespFn: closeRespFn,
+		mutex:       &sync.Mutex{}, // allocate for copying
 	}
 
 	configURL := env.Get(EnvIdentityOpenIDURL, kvs.Get(ConfigURL))