fix: data usage crawler env handling, usage-cache.bin location (#9163)

canonicalize the ENVs such that we can bring these ENVs 
as part of the config values, as a subsequent change.

- fix location of per bucket usage to `.minio.sys/buckets/<bucket_name>/usage-cache.bin`
- fix location of the overall usage in `json` at `.minio.sys/buckets/.usage.json`
  (avoid conflicts with a bucket named `usage.json` )
- fix location of the overall usage in `msgp` at `.minio.sys/buckets/.usage.bin`
  (avoid conflicts with a bucket named `usage.bin`

.jshintrc

@@ -1,4 +0,0 @@
-{
-  "asi": true,
-  "esnext": true   
-}

.mention-bot

@@ -1,4 +0,0 @@
-{
-    "numFilesToCheck": 10,
-    "requiredOrgs": ["minio"]
-}

CREDITS

@@ -11366,28 +11366,28 @@ SOFTWARE.
 github.com/klauspost/readahead
 https://github.com/klauspost/readahead
 ----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) 2015 Klaus Post
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
+The MIT License (MIT)
+
+Copyright (c) 2015 Klaus Post
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 
 ================================================================
 
@@ -12375,6 +12375,214 @@ https://github.com/minio/minio-go/v6
 
 ================================================================
 
+github.com/minio/minio-go/v6
+https://github.com/minio/minio-go/v6
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 github.com/minio/parquet-go
 https://github.com/minio/parquet-go
 ----------------------------------------------------------------
@@ -16208,7 +16416,7 @@ github.com/pascaldekloe/goe
 https://github.com/pascaldekloe/goe
 ----------------------------------------------------------------
 To the extent possible under law, Pascal S. de Kloe has waived all
-copyright and related or neighboring rights to Go Enterprise. This
+copyright and related or neighboring rights to Go Enterprice. This
 work is published from The Netherlands.
 
 https://creativecommons.org/publicdomain/zero/1.0/legalcode
@@ -16312,36 +16520,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-github.com/pkg/profile
-https://github.com/pkg/profile
-----------------------------------------------------------------
-Copyright (c) 2013 Dave Cheney. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-================================================================
-
 github.com/pmezard/go-difflib
 https://github.com/pmezard/go-difflib
 ----------------------------------------------------------------

Dockerfile

@@ -5,7 +5,6 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 ENV GOPATH /go
 ENV CGO_ENABLED 0
 ENV GO111MODULE on
-ENV GOPROXY https://proxy.golang.org
 
 RUN  \
      apk add --no-cache git && \
@@ -23,6 +22,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
+COPY CREDITS /third_party/
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \

Dockerfile.arm.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm32v7/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.arm64.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm64v8/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.dev

@@ -4,6 +4,7 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \

Dockerfile.release

@@ -13,6 +13,7 @@ FROM alpine:3.10
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \

Makefile

@@ -22,7 +22,7 @@ getdeps:
 ifeq ($(GOARCH),s390x)
 	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
 else
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2019.2.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2020.1.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
 endif
 	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
 

NOTICE

@@ -1,4 +1,4 @@
-MinIO Cloud Storage, (C) 2014,2015 MinIO, Inc.
+MinIO Cloud Storage, (C) 2014-2020 MinIO, Inc.
 
 This product includes software developed at MinIO, Inc.
 (https://min.io/).
@@ -6,4 +6,4 @@ This product includes software developed at MinIO, Inc.
 The MinIO project contains unmodified/modified subcomponents too with
 separate copyright notices and license terms. Your use of the source
 code for these subcomponents is subject to the terms and conditions
-of the following licenses.
+of Apache License Version 2.0

README.md

@@ -21,7 +21,7 @@ docker run -p 9000:9000 minio/minio:edge server /data
 > NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
-### Homebrew
+### Homebrew (recommended)
 Install minio packages using [Homebrew](http://brew.sh/)
 ```sh
 brew install minio/stable/minio
@@ -74,7 +74,7 @@ minio.exe server D:\Photos
 
 ## FreeBSD
 ### Port
-Install minio packages using [pkg](https://github.com/freebsd/pkg)
+Install minio packages using [pkg](https://github.com/freebsd/pkg), MinIO doesn't officially build FreeBSD binaries but is maintained by FreeBSD upstream [here](https://www.freshports.org/www/minio).
 
 ```sh
 pkg install minio

browser/app/js/browser/Login.js

@@ -148,7 +148,8 @@ export class Login extends React.Component {
                   <OpenIDLoginButton
                     className="btn openid-btn"
                     clientId={this.state.clientId}
-                    authorizationEndpoint={this.state.discoveryDoc.authorization_endpoint}
+                    authEp={this.state.discoveryDoc.authorization_endpoint}
+                    authScopes={this.state.discoveryDoc.scopes_supported}
                   >
                     Log in with OpenID
                   </OpenIDLoginButton>

browser/app/js/browser/OpenIDLogin.js

@@ -66,6 +66,7 @@ export class OpenIDLogin extends React.Component {
 
       const authURL = buildOpenIDAuthURL(
         this.state.discoveryDoc.authorization_endpoint,
+        this.state.discoveryDoc.scopes_supported,
         redirectURI,
         this.state.clientID,
         nonce

browser/app/js/browser/OpenIDLoginButton.js

@@ -27,7 +27,7 @@ export class OpenIDLoginButton extends React.Component {
 
   handleClick(event) {
     event.stopPropagation()
-    const { authorizationEndpoint, clientId } = this.props
+    const { authEp, authScopes, clientId } = this.props
 
     let redirectURI = window.location.href.split("#")[0]
     if (redirectURI.endsWith('/')) {
@@ -40,7 +40,7 @@ export class OpenIDLoginButton extends React.Component {
     const nonce = getRandomString(16)
     storage.setItem(OPEN_ID_NONCE_KEY, nonce)
 
-    const authURL = buildOpenIDAuthURL(authorizationEndpoint, redirectURI, clientId, nonce)
+    const authURL = buildOpenIDAuthURL(authEp, authScopes, redirectURI, clientId, nonce)
     window.location = authURL
   }
 

browser/app/js/browser/utils.js

@@ -16,14 +16,13 @@
 
 export const OPEN_ID_NONCE_KEY = 'openIDKey'
 
-export const buildOpenIDAuthURL = (authorizationEndpoint, redirectURI, clientID, nonce) => {
+export const buildOpenIDAuthURL = (authEp, authScopes, redirectURI, clientID, nonce) => {
   const params = new URLSearchParams()
   params.set("response_type", "id_token")
-  params.set("scope", "openid")
+  params.set("scope", authScopes.join(" "))
   params.set("client_id", clientID)
   params.set("redirect_uri", redirectURI)
   params.set("nonce", nonce)
 
-  return `${authorizationEndpoint}?${params.toString()}`
+  return `${authEp}?${params.toString()}`
 }
-

browser/staticcheck.conf

@@ -0,0 +1 @@
+checks = ["all", "-ST1005", "-ST1000", "-SA4000", "-SA9004", "-SA1019", "-SA1008", "-U1000", "-ST1003", "-ST1018"]

buildscripts/benchcmp.sh

@@ -1,67 +0,0 @@
-#!/bin/bash          
-
-# usage: ./benchcmp.sh <commit-sha1> <commit-sha2>
-# Exit on any non zero return value on execution of a command. 
-set -e 
-
-# path of benchcmp. 
-benchcmp=${GOPATH}/bin/benchcmp
-
-# function which runs the benchmark comparison. 
-RunBenchCmp () {
-	# Path for storing output of benchmark at commit 1. 
-	commit1Bench=/tmp/minio-$1.bench
-	# Path for storing output of benchmark at commit 2. 
-	commit2Bench=/tmp/minio-$2.bench
-	# switch to commit $1. 
-	git checkout $1
-	# Check if the benchmark results for given commit 1 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit1Bench  ]]
-	then
-
-		echo "Running benchmarks at $1"
-		go test -run=NONE -bench=. | tee $commit1Bench 
-	fi
-	# get back to the commit from which it was started. 
-	git checkout -
-	echo "Checking into commit $2"
-	# switch to commit $2 
-	git checkout $2
-	# Check if the benchmark results for given commit 2 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit2Bench  ]]
-	then
-	# Running benchmarks at $2.
-		echo "Running benchmarks at $2"
-		go test -run=NONE -bench=. | tee $commit2Bench 
-	fi
-	
-	# get back to the commit from which it was started. 
-	git checkout - 
-	# Comparing the benchmarks.
-	echo "Running benchmark comparison between $1 and $2 ..."
-	$benchcmp $commit1Bench $commit2Bench 
-	echo "Done."
-}
-
-# check if 2 commit SHA's of snapshots of code for which benchmp has to be done is provided. 
-if [ ! $# -eq 2 ]
-then
-	# exit if commit SHA's are not provided. 
-	echo $#
-	echo "Need Commit SHA's of 2 snapshots to be supplied to run benchmark comparison."
-	exit 1
-fi
-
-# check if benchcmp exists. 
-if [[ -x "$benchcmp" ]]
-then
-	RunBenchCmp $1 $2
-else
-	# install benchcmp if doesnt't exist. 
-	echo "fetching Benchcmp..."
-	go get -u golang.org/x/tools/cmd/benchcmp
-	echo "Done."
-	RunBenchCmp $1 $2
-fi

buildscripts/cross-compile.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
+set -e
 # Enable tracing if set.
-[ -n "$BASH_XTRACEFD" ] && set -ex
+[ -n "$BASH_XTRACEFD" ] && set -x
 
 function _init() {
     ## All binaries are static make sure to disable CGO.
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
 }
 
 function _build() {

buildscripts/verify-healing.sh

@@ -26,7 +26,7 @@ fi
 
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
 
 function start_minio_3_node() {
     declare -a minio_pids
@@ -34,16 +34,16 @@ function start_minio_3_node() {
     export MINIO_SECRET_KEY=minio123
 
     for i in $(seq 1 3); do
-        ARGS+=("http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/6/")
+        ARGS+=("http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/6/")
     done
 
-    "${MINIO[@]}" server --address ":9001" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9001.log" 2>&1 &
+    "${MINIO[@]}" --address ":8001" ${ARGS[@]} > "${WORK_DIR}/dist-minio-8001.log" 2>&1 &
     minio_pids[0]=$!
 
-    "${MINIO[@]}" server --address ":9002" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9002.log" 2>&1 &
+    "${MINIO[@]}" --address ":8002" ${ARGS[@]} > "${WORK_DIR}/dist-minio-8002.log" 2>&1 &
     minio_pids[1]=$!
 
-    "${MINIO[@]}" server --address ":9003" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9003.log" 2>&1 &
+    "${MINIO[@]}" --address ":8003" ${ARGS[@]} > "${WORK_DIR}/dist-minio-8003.log" 2>&1 &
     minio_pids[2]=$!
 
     sleep "$1"
@@ -53,7 +53,7 @@ function start_minio_3_node() {
 
 function check_online() {
     for i in $(seq 1 3); do
-        if grep -q 'Server switching to safe mode' ${WORK_DIR}/dist-minio-$[9000+$i].log; then
+        if grep -q 'Server switching to safe mode' ${WORK_DIR}/dist-minio-$[8000+$i].log; then
             echo "1"
         fi
     done
@@ -74,25 +74,41 @@ function __init__()
     echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
 }
 
-function perform_test_1() {
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
-
-    echo "Testing in Distributed Erasure setup healing test case 1"
-    echo "Remove the contents of the disks belonging to '2' erasure set"
-
-    rm -rf ${WORK_DIR}/2/*
-
+function perform_test() {
     minio_pids=( $(start_minio_3_node 60) )
     for pid in "${minio_pids[@]}"; do
         if ! kill "$pid"; then
-            cat "${WORK_DIR}/*.log"
+            for i in $(seq 1 3); do
+                echo "server$i log:"
+                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
+            done
             echo "FAILED"
             purge "$WORK_DIR"
             exit 1
         fi
+        # forcibly killing, to proceed further properly.
+        kill -9 "$pid"
+    done
+
+    echo "Testing Distributed Erasure setup healing of drives"
+    echo "Remove the contents of the disks belonging to '${1}' erasure set"
+
+    rm -rf ${WORK_DIR}/${1}/*/
+
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        if ! kill "$pid"; then
+            for i in $(seq 1 3); do
+                echo "server$i log:"
+                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
+            done
+            echo "FAILED"
+            purge "$WORK_DIR"
+            exit 1
+        fi
+        # forcibly killing, to proceed further properly.
+        # if the previous kill is taking time.
+        kill -9 "$pid"
     done
 
     rv=$(check_online)
@@ -102,81 +118,7 @@ function perform_test_1() {
         done
         for i in $(seq 1 3); do
             echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-$[9000+$i].log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
-}
-
-function perform_test_2() {
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
-
-    echo "Testing in Distributed Erasure setup healing test case 2"
-    echo "Remove the contents of the disks belonging to '1' erasure set"
-
-    rm -rf ${WORK_DIR}/1/*/
-
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            cat "${WORK_DIR}/*.log"
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
-    done
-
-    rv=$(check_online)
-
-    if [ "$rv" == "1" ]; then
-        for pid in "${minio_pids[@]}"; do
-            kill -9 "$pid"
-        done
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-$[9000+$i].log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
-}
-
-function perform_test_3() {
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
-
-    echo "Testing in Distributed Erasure setup healing test case 3"
-    echo "Remove the contents of the disks belonging to '3' erasure set"
-
-    rm -rf ${WORK_DIR}/3/*/
-
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            cat "${WORK_DIR}/*.log"
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
-    done
-
-    rv=$(check_online)
-
-    if [ "$rv" == "1" ]; then
-        for pid in "${minio_pids[@]}"; do
-            kill -9 "$pid"
-        done
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-$[9000+$i].log"
+            cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
         done
         echo "FAILED"
         purge "$WORK_DIR"
@@ -186,9 +128,9 @@ function perform_test_3() {
 
 function main()
 {
-    perform_test_1
-    perform_test_2
-    perform_test_3
+    perform_test "2"
+    perform_test "1"
+    perform_test "3"
 }
 
 ( __init__ "$@" && main "$@" )

cmd/acl-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,11 +18,14 @@ package cmd
 
 import (
 	"encoding/xml"
+	"io"
 	"net/http"
+	"net/url"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Data types used for returning dummy access control
@@ -50,6 +53,71 @@ type accessControlPolicy struct {
 	} `xml:"AccessControlList"`
 }
 
+// PutBucketACLHandler - PUT Bucket ACL
+// -----------------
+// This operation uses the ACL subresource
+// to set ACL for a bucket, this is a dummy call
+// only responds success if the ACL is private.
+func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketACL")
+
+	defer logger.AuditLog(w, r, "PutBucketACL", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow putBucketACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	aclHeader := r.Header.Get(xhttp.AmzACL)
+	if aclHeader == "" {
+		acl := &accessControlPolicy{}
+		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
+			if err == io.EOF {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
+					r.URL, guessIsBrowserReq(r))
+				return
+			}
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if len(acl.AccessControlList.Grants) == 0 {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if aclHeader != "" && aclHeader != "private" {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetBucketACLHandler - GET Bucket ACL
 // -----------------
 // This operation uses the ACL
@@ -99,6 +167,71 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	w.(http.Flusher).Flush()
 }
 
+// PutObjectACLHandler - PUT Object ACL
+// -----------------
+// This operation uses the ACL subresource
+// to set ACL for a bucket, this is a dummy call
+// only responds success if the ACL is private.
+func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutObjectACL")
+
+	defer logger.AuditLog(w, r, "PutObjectACL", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object, err := url.PathUnescape(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow putObjectACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Before proceeding validate if object exists.
+	_, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	aclHeader := r.Header.Get(xhttp.AmzACL)
+	if aclHeader == "" {
+		acl := &accessControlPolicy{}
+		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if len(acl.AccessControlList.Grants) == 0 {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if aclHeader != "" && aclHeader != "private" {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetObjectACLHandler - GET Object ACL
 // -----------------
 // This operation uses the ACL
@@ -110,7 +243,11 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	object := vars["object"]
+	object, err := url.PathUnescape(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -126,7 +263,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if object exists.
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	_, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/admin-handlers-users.go

@@ -378,6 +378,120 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// AddServiceAccount - PUT /minio/admin/v2/add-service-account?parentUser=<parent_user_accesskey>
+func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddServiceAccount")
+
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	var createReq madmin.AddServiceAccountReq
+	if err = json.Unmarshal(configBytes, &createReq); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	if createReq.Parent == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+
+	// Disallow creating service accounts for the root user
+	if createReq.Parent == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddServiceAccountInvalidArgument), r.URL)
+		return
+	}
+
+	creds, err := globalIAMSys.NewServiceAccount(ctx, createReq.Parent, createReq.Policy)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(creds.AccessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	var createResp = madmin.AddServiceAccountResp{
+		Credentials: creds,
+	}
+
+	data, err := json.Marshal(createResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// GetServiceAccount - GET /minio/admin/v2/get-service-account?accessKey=<access_key>
+func (a adminAPIHandlers) GetServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetServiceAccount")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	creds, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(creds)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
 // InfoCannedPolicy - GET /minio/admin/v2/info-canned-policy?name={policyName}
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")

cmd/admin-handlers.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"crypto/subtle"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -140,9 +141,9 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		if runtime.GOOS == "windows" {
-			u.Path = path.Dir(u.Path) + "minio.exe"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio.exe"
 		} else {
-			u.Path = path.Dir(u.Path) + "minio"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio"
 		}
 
 		updateURL = u.String()
@@ -270,12 +271,12 @@ type ServerInfo struct {
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
@@ -295,26 +296,87 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	dataUsageInfoJSON, err := json.Marshal(dataUsageInfo)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
+func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AccountingUsageInfo")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AccountingUsageInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var accountingUsageInfo = make(map[string]madmin.BucketAccountingUsage)
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		// writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	users, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Load the latest calculated data usage
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	// Calculate for each bucket, which users are allowed to access to it
+	for _, bucket := range buckets {
+		bucketUsageInfo := madmin.BucketAccountingUsage{}
+
+		// Fetch the data usage of the current bucket
+		if !dataUsageInfo.LastUpdate.IsZero() && dataUsageInfo.BucketsSizes != nil {
+			bucketUsageInfo.Size = dataUsageInfo.BucketsSizes[bucket.Name]
+		}
+
+		for user := range users {
+			rd, wr, custom := globalIAMSys.GetAccountAccess(user, bucket.Name)
+			if rd || wr || custom {
+				bucketUsageInfo.AccessList = append(bucketUsageInfo.AccessList, madmin.AccountAccess{
+					AccountName: user,
+					Read:        rd,
+					Write:       wr,
+					Custom:      custom,
+				})
+			}
+		}
+
+		accountingUsageInfo[bucket.Name] = bucketUsageInfo
+	}
+
+	usageInfoJSON, err := json.Marshal(accountingUsageInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, usageInfoJSON)
+}
+
 // ServerCPULoadInfo holds informantion about cpu utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
@@ -349,7 +411,7 @@ type ServerNetReadPerfInfo struct {
 func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PerfInfo")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.PerfInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -520,7 +582,7 @@ type PeerLocks struct {
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -565,7 +627,7 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -661,7 +723,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -835,7 +897,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// find number of disks in the setup
-	info := objectAPI.StorageInfo(ctx)
+	info := objectAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
 
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
@@ -920,13 +982,24 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{}
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
 	for _, state := range bgHealStates {
 		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
 			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
 		}
-
 	}
 
 	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
@@ -1051,7 +1124,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ListServerInfoAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1105,7 +1178,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1176,7 +1249,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatusHandler")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1208,20 +1281,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// 2. Check whether we can update / re-wrap the sealed key.
-	sealedKey, err = GlobalKMS.UpdateKey(keyID, sealedKey, kmsContext)
-	if err != nil {
-		response.UpdateErr = err.Error()
-		resp, err := json.Marshal(response)
-		if err != nil {
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
-			return
-		}
-		writeSuccessResponseJSON(w, resp)
-		return
-	}
-
-	// 3. Verify that we can indeed decrypt the (encrypted) key
+	// 2. Verify that we can indeed decrypt the (encrypted) key
 	decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
 	if err != nil {
 		response.DecryptionErr = err.Error()
@@ -1234,7 +1294,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// 4. Compare generated key with decrypted key
+	// 3. Compare generated key with decrypted key
 	if subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1 {
 		response.DecryptionErr = "The generated and the decrypted data key do not match"
 		resp, err := json.Marshal(response)
@@ -1261,7 +1321,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HardwareInfo")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerHardwareInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1316,7 +1376,7 @@ func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *ht
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, "")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1349,10 +1409,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		} else if ldapConn == nil {
 			ldap.Status = "Not Configured"
 		} else {
+			// Close ldap connection to avoid leaks.
+			ldapConn.Close()
 			ldap.Status = "online"
 		}
-		// Close ldap connection to avoid leaks.
-		defer ldapConn.Close()
 	}
 
 	log, audit := fetchLoggerInfo(cfg)
@@ -1361,7 +1421,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	notifyTarget := fetchLambdaInfo(cfg)
 
 	// Fetching the Storage information
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	var OnDisks int
 	var OffDisks int
@@ -1470,10 +1530,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 }
 
 func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
-	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
 	// Fetch the targets
-	targetList, _ := notify.RegisterNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport(), nil, true)
+	targetList, err := notify.RegisterNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport(), nil, true)
+	if err != nil {
+		return nil
+	}
+	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
 	for targetID, target := range targetList.TargetMap() {
 		targetIDStatus := make(map[string]madmin.Status)
@@ -1516,36 +1579,29 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 		return vault
 	}
 
-	if err := checkConnection(kmsInfo.Endpoint); err != nil {
+	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
 
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"
 
-		kmsContext := crypto.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
+		kmsContext := crypto.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
 		// 1. Generate a new key using the KMS.
 		key, sealedKey, err := GlobalKMS.GenerateKey(keyID, kmsContext)
 		if err != nil {
-			vault.Encrypt = "Encryption failed"
+			vault.Encrypt = fmt.Sprintf("Encryption failed: %v", err)
 		} else {
 			vault.Encrypt = "Ok"
 		}
 
-		// 2. Check whether we can update / re-wrap the sealed key.
-		sealedKey, err = GlobalKMS.UpdateKey(keyID, sealedKey, kmsContext)
-		if err != nil {
-			vault.Update = "Re-wrap failed:"
-		} else {
-			vault.Update = "Ok"
-		}
-
-		// 3. Verify that we can indeed decrypt the (encrypted) key
-		decryptedKey, decryptErr := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
-
-		// 4. Compare generated key with decrypted key
-		if subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1 || decryptErr != nil {
-			vault.Decrypt = "Re-wrap failed:"
-		} else {
+		// 2. Verify that we can indeed decrypt the (encrypted) key
+		decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
+		switch {
+		case err != nil:
+			vault.Decrypt = fmt.Sprintf("Decryption failed: %v", err)
+		case subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1:
+			vault.Decrypt = "Decryption failed: decrypted key does not match generated key"
+		default:
 			vault.Decrypt = "Ok"
 		}
 	}
@@ -1560,7 +1616,7 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 	var auditlogger []madmin.Audit
 	for log, l := range loggerCfg.HTTP {
 		if l.Enabled {
-			err := checkConnection(l.Endpoint)
+			err := checkConnection(l.Endpoint, 15*time.Second)
 			if err == nil {
 				mapLog := make(map[string]madmin.Status)
 				mapLog[log] = madmin.Status{Status: "Online"}
@@ -1575,7 +1631,7 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 
 	for audit, l := range loggerCfg.Audit {
 		if l.Enabled {
-			err := checkConnection(l.Endpoint)
+			err := checkConnection(l.Endpoint, 15*time.Second)
 			if err == nil {
 				mapAudit := make(map[string]madmin.Status)
 				mapAudit[audit] = madmin.Status{Status: "Online"}
@@ -1591,12 +1647,19 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 }
 
 // checkConnection - ping an endpoint , return err in case of no connection
-func checkConnection(endpointStr string) error {
+func checkConnection(endpointStr string, timeout time.Duration) error {
 	u, pErr := xnet.ParseURL(endpointStr)
 	if pErr != nil {
 		return pErr
 	}
-	if dErr := u.DialHTTP(); dErr != nil {
+
+	tr := newCustomHTTPTransport(
+		&tls.Config{RootCAs: globalRootCAs},
+		timeout,
+		0, /* Default value */
+	)()
+
+	if dErr := u.DialHTTP(tr); dErr != nil {
 		if urlErr, ok := dErr.(*url.Error); ok {
 			// To treat "connection refused" errors as un reachable endpoint.
 			if target.IsConnRefusedErr(urlErr.Err) {

cmd/admin-handlers_test.go

@@ -109,7 +109,7 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 		return nil, nil, err
 	}
 	endpoints := mustGetNewEndpoints(xlDirs...)
-	format, err := waitForFormatXL(true, endpoints, 1, 16, "")
+	format, err := waitForFormatXL(true, endpoints, 1, 1, 16, "")
 	if err != nil {
 		removeRoots(xlDirs)
 		return nil, nil, err
@@ -263,6 +263,7 @@ func TestAdminServerInfo(t *testing.T) {
 	if err != nil {
 		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
+
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.

cmd/admin-heal-ops.go

@@ -106,12 +106,12 @@ func initHealState() *allHealState {
 		healSeqMap: make(map[string]*healSequence),
 	}
 
-	go healState.periodicHealSeqsClean()
+	go healState.periodicHealSeqsClean(GlobalContext)
 
 	return healState
 }
 
-func (ahs *allHealState) periodicHealSeqsClean() {
+func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
 	ticker := time.NewTicker(time.Minute * 5)
@@ -127,7 +127,7 @@ func (ahs *allHealState) periodicHealSeqsClean() {
 				}
 			}
 			ahs.Unlock()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			// server could be restarting - need
 			// to exit immediately
 			return
@@ -369,7 +369,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	return &healSequence{
 		bucket:         bucket,
@@ -575,8 +575,6 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	h.lastHealActivity = UTCNow()
-
 	bucketsOnly := true // heal buckets only, not objects.
 	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)
@@ -605,7 +603,7 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			h.lastHealActivity = UTCNow()
 		case <-h.traverseAndHealDoneCh:
 			return nil
-		case <-GlobalServiceDoneCh:
+		case <-h.ctx.Done():
 			return nil
 		}
 	}
@@ -632,11 +630,6 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 		return err
 	}
 
-	// Start healing the background ops prefix.
-	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
-
 	// Heal buckets and objects
 	return h.healBuckets(bucketsOnly)
 }
@@ -673,7 +666,7 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, func(bucket string, object string) error {
+		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket string, object string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -767,7 +760,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.healObject); err != nil {
+	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.settings, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil
@@ -775,15 +768,15 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object string) error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	return h.queueHealTask(pathJoin(bucket, object), madmin.HealItemObject)
 }

cmd/admin-router.go

@@ -56,6 +56,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	// DataUsageInfo operations
 	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
+
 	if globalIsDistXL || globalIsXL {
 		/// Heal operations
 
@@ -108,6 +110,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
+		// Service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/get-service-account").HandlerFunc(httpTraceHdrs(adminAPI.GetServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+
 		// Info policy IAM
 		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 

cmd/admin-server-info.go

@@ -20,7 +20,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"strings"
 
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/cpu"
@@ -190,51 +189,49 @@ func getLocalNetworkInfo(endpointZones EndpointZones, r *http.Request) madmin.Se
 // getLocalServerProperty - returns ServerDrivesPerfInfo for only the
 // local endpoints from given list of endpoints
 func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
-	var di madmin.Disk
 	var disks []madmin.Disk
 	addr := r.Host
 	if globalIsDistXL {
 		addr = GetLocalPeer(endpointZones)
 	}
 	network := make(map[string]string)
-	hosts := set.NewStringSet()
 	for _, ep := range endpointZones {
 		for _, endpoint := range ep.Endpoints {
-
-			url := strings.Replace(endpoint.URL.String(), endpoint.Path, "", -1)
-			if url == "" {
-				url = r.Host
+			nodeName := endpoint.Host
+			if nodeName == "" {
+				nodeName = r.Host
 			}
-			hosts.Add(url)
-
-			// Only proceed for local endpoints
 			if endpoint.IsLocal {
-				url = fetchAddress(url)
-				network[url] = "online"
-				if _, err := os.Stat(endpoint.Path); err != nil {
-					continue
+				// Only proceed for local endpoints
+				network[nodeName] = "online"
+				var di = madmin.Disk{
+					DrivePath: endpoint.Path,
+				}
+				diInfo, err := disk.GetInfo(endpoint.Path)
+				if err != nil {
+					if os.IsNotExist(err) || isSysErrPathNotFound(err) {
+						di.State = madmin.DriveStateMissing
+					} else {
+						di.State = madmin.DriveStateCorrupt
+					}
+				} else {
+					di.State = madmin.DriveStateOk
+					di.DrivePath = endpoint.Path
+					di.TotalSpace = diInfo.Total
+					di.UsedSpace = diInfo.Total - diInfo.Free
+					di.Utilization = float64((diInfo.Total - diInfo.Free) / diInfo.Total * 100)
 				}
-
-				diInfo, _ := disk.GetInfo(endpoint.Path)
-				di.State = "ok"
-				di.DrivePath = endpoint.Path
-				di.TotalSpace = diInfo.Total
-				di.UsedSpace = diInfo.Total - diInfo.Free
-				di.Utilization = float64((diInfo.Total - diInfo.Free) / diInfo.Total * 100)
 				disks = append(disks, di)
-			}
-		}
-	}
-
-	for host := range hosts {
-		_, present := network[host]
-		if !present {
-			err := checkConnection(host)
-			host = fetchAddress(host)
-			if err != nil {
-				network[host] = "offline"
 			} else {
-				network[host] = "online"
+				_, present := network[nodeName]
+				if !present {
+					err := IsServerResolvable(endpoint)
+					if err == nil {
+						network[nodeName] = "online"
+					} else {
+						network[nodeName] = "offline"
+					}
+				}
 			}
 		}
 	}
@@ -249,13 +246,3 @@ func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin
 		Disks:    disks,
 	}
 }
-
-// Replaces http and https from address
-func fetchAddress(address string) string {
-	if strings.Contains(address, "http://") {
-		address = strings.Replace(address, "http://", "", -1)
-	} else if strings.Contains(address, "https://") {
-		address = strings.Replace(address, "https://", "", -1)
-	}
-	return address
-}

cmd/api-errors.go

@@ -21,6 +21,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
@@ -32,9 +33,12 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // APIError structure
@@ -91,9 +95,12 @@ const (
 	ErrMissingContentLength
 	ErrMissingContentMD5
 	ErrMissingRequestBodyError
+	ErrMissingSecurityHeader
 	ErrNoSuchBucket
 	ErrNoSuchBucketPolicy
 	ErrNoSuchBucketLifecycle
+	ErrNoSuchLifecycleConfiguration
+	ErrNoSuchBucketSSEConfig
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -115,7 +122,8 @@ const (
 	ErrMissingCredTag
 	ErrCredMalformed
 	ErrInvalidRegion
-	ErrInvalidService
+	ErrInvalidServiceS3
+	ErrInvalidServiceSTS
 	ErrInvalidRequestVersion
 	ErrMissingSignTag
 	ErrMissingSignHeadersTag
@@ -144,11 +152,13 @@ const (
 	ErrKeyTooLongError
 	ErrInvalidBucketObjectLockConfiguration
 	ErrObjectLockConfigurationNotAllowed
+	ErrNoSuchObjectLockConfiguration
 	ErrObjectLocked
 	ErrInvalidRetentionDate
 	ErrPastObjectLockRetainDate
 	ErrUnknownWORMModeDirective
 	ErrObjectLockInvalidHeaders
+	ErrInvalidTagDirective
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -323,6 +333,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrAddServiceAccountInvalidArgument
 	ErrPostPolicyConditionInvalidFormat
 )
 
@@ -464,6 +475,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Missing required header for this request: Content-Md5.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingSecurityHeader: {
+		Code:           "MissingSecurityHeader",
+		Description:    "Your request was missing a required header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrMissingRequestBodyError: {
 		Code:           "MissingRequestBodyError",
 		Description:    "Request body is empty.",
@@ -484,6 +500,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket lifecycle configuration does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchLifecycleConfiguration: {
+		Code:           "NoSuchLifecycleConfiguration",
+		Description:    "The lifecycle configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchBucketSSEConfig: {
+		Code:           "ServerSideEncryptionConfigurationNotFoundError",
+		Description:    "The server side encryption configuration was not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchKey: {
 		Code:           "NoSuchKey",
 		Description:    "The specified key does not exist.",
@@ -628,9 +654,14 @@ var errorCodes = errorCodeMap{
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
 	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
 	// Need changes to make sure variable messages can be constructed.
-	ErrInvalidService: {
-		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+	ErrInvalidServiceS3: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidServiceSTS: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential parameter; incorrect service. This endpoint belongs to \"sts\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
@@ -738,6 +769,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrNoSuchObjectLockConfiguration: {
+		Code:           "NoSuchObjectLockConfiguration",
+		Description:    "The specified object does not have a ObjectLock configuration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrObjectLocked: {
 		Code:           "InvalidRequest",
 		Description:    "Object is WORM protected and cannot be overwritten",
@@ -829,6 +865,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidTagDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown tag directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidEncryptionMethod: {
 		Code:           "InvalidRequest",
 		Description:    "The encryption method specified is not supported",
@@ -1539,6 +1580,12 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrAddServiceAccountInvalidArgument: {
+		Code:           "XMinioInvalidArgument",
+		Description:    "New service accounts for admin access key is not allowed",
+		HTTPStatusCode: http.StatusConflict,
+	},
+
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
 		Description:    "Invalid according to Policy: Policy Condition failed",
@@ -1607,18 +1654,20 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case crypto.ErrKMSAuthLogin:
 		apiErr = ErrKMSAuthFailure
-	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
+	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
 		apiErr = ErrSlowDown
-	case errInvalidRetentionDate:
+	case objectlock.ErrInvalidRetentionDate:
 		apiErr = ErrInvalidRetentionDate
-	case errPastObjectLockRetainDate:
+	case objectlock.ErrPastObjectLockRetainDate:
 		apiErr = ErrPastObjectLockRetainDate
-	case errUnknownWORMModeDirective:
+	case objectlock.ErrUnknownWORMModeDirective:
 		apiErr = ErrUnknownWORMModeDirective
-	case errObjectLockInvalidHeaders:
+	case objectlock.ErrObjectLockInvalidHeaders:
 		apiErr = ErrObjectLockInvalidHeaders
+	case objectlock.ErrMalformedXML:
+		apiErr = ErrMalformedXML
 	}
 
 	// Compression errors
@@ -1708,7 +1757,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case BucketPolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
 	case BucketLifecycleNotFound:
-		apiErr = ErrNoSuchBucketLifecycle
+		apiErr = ErrNoSuchLifecycleConfiguration
+	case BucketSSEConfigNotFound:
+		apiErr = ErrNoSuchBucketSSEConfig
 	case *event.ErrInvalidEventName:
 		apiErr = ErrEventNotification
 	case *event.ErrInvalidARN:
@@ -1731,6 +1782,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrOverlappingFilterNotification
 	case *event.ErrUnsupportedConfiguration:
 		apiErr = ErrUnsupportedNotification
+	case OperationTimedOut:
+		apiErr = ErrOperationTimedOut
 	case BackendDown:
 		apiErr = ErrBackendDown
 	case ObjectNameTooLong:
@@ -1777,6 +1830,32 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case *xml.SyntaxError:
+			apiErr = APIError{
+				Code: "MalformedXML",
+				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
+					e.Error()),
+				HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
+			}
+		case url.EscapeError:
+			apiErr = APIError{
+				Code: "XMinioInvalidObjectName",
+				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
+					e.Error()),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case lifecycle.Error:
+			apiErr = APIError{
+				Code:           "InvalidRequest",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case tagging.Error:
+			apiErr = APIError{
+				Code:           e.Code(),
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case policy.Error:
 			apiErr = APIError{
 				Code:           "MalformedPolicy",

cmd/api-headers.go

@@ -22,7 +22,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -80,6 +82,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
+	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
+		w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts)))
+	}
+
 	if objInfo.ContentType != "" {
 		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}
@@ -95,6 +101,14 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
 		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
 	}
+
+	// Set tag count if object has tags
+	tags, _ := url.ParseQuery(objInfo.UserTags)
+	tagCount := len(tags)
+	if tagCount != 0 {
+		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	}
+
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		if HasPrefix(k, ReservedMetadataPrefix) {

cmd/api-router.go

@@ -105,26 +105,33 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler))).Queries("acl", "")
-		// GetObjectTagging - this is a dummy call.
+		// PutObjectACL - this is a dummy call.
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler))).Queries("acl", "")
+		// GetObjectTagging
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler))).Queries("tagging", "")
+		// PutObjectTagging
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler))).Queries("tagging", "")
+		// DeleteObjectTagging
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler))).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceHdrs(api.GetObjectRetentionHandler))).Queries("retention", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler))).Queries("retention", "")
+		// GetObjectLegalHold
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
 		// GetObject
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)))
 		// CopyObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)))
 		// PutObjectRetention
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceHdrs(api.PutObjectRetentionHandler))).Queries("retention", "")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler))).Queries("retention", "")
+		// PutObjectLegalHold
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
+
 		// PutObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)))
 		// DeleteObject
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)))
-		// PutObjectLegalHold
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
-		// GetObjectLegalHold
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
 
 		/// Bucket operations
 		// GetBucketLocation
@@ -133,10 +140,14 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler))).Queries("policy", "")
 		// GetBucketLifecycle
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
+		// GetBucketEncryption
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler))).Queries("encryption", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler))).Queries("acl", "")
+		// PutBucketACL -- this is a dummy call.
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
@@ -178,6 +189,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler)))
 		// PutBucketLifecycle
 		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler))).Queries("lifecycle", "")
+		// PutBucketEncryption
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler))).Queries("encryption", "")
+
 		// PutBucketPolicy
 		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler))).Queries("policy", "")
 
@@ -199,6 +213,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler))).Queries("policy", "")
 		// DeleteBucketLifecycle
 		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler))).Queries("lifecycle", "")
+		// DeleteBucketEncryption
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler))).Queries("encryption", "")
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler)))
 	}

cmd/auth-handler.go

@@ -28,13 +28,13 @@ import (
 	"net/http"
 	"strings"
 
-	jwtgo "github.com/dgrijalva/jwt-go"
 	xhttp "github.com/minio/minio/cmd/http"
+	xjwt "github.com/minio/minio/cmd/jwt"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // Verify if request has JWT.
@@ -179,12 +179,14 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 
 // Fetch claims in the security token returned by the client.
 func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
-	claims := make(map[string]interface{})
+	claims := xjwt.NewMapClaims()
+
 	token := getSessionToken(r)
 	if token == "" {
-		return claims, nil
+		return claims.Map(), nil
 	}
-	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
+
+	stsTokenCallback := func(claims *xjwt.MapClaims) ([]byte, error) {
 		// JWT token for x-amz-security-token is signed with admin
 		// secret key, temporary credentials become invalid if
 		// server admin credentials change. This is done to ensure
@@ -195,66 +197,42 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 		// on the client side and is treated like an opaque value.
 		return []byte(globalActiveCred.SecretKey), nil
 	}
-	p := &jwtgo.Parser{
-		ValidMethods: []string{
-			jwtgo.SigningMethodHS256.Alg(),
-			jwtgo.SigningMethodHS512.Alg(),
-		},
-	}
-	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
-	if err != nil {
-		return nil, err
-	}
-	if !jtoken.Valid {
+
+	if err := xjwt.ParseWithClaims(token, claims, stsTokenCallback); err != nil {
 		return nil, errAuthentication
 	}
-	v, ok := claims["accessKey"]
-	if !ok {
-		return nil, errInvalidAccessKeyID
-	}
-	if _, ok = v.(string); !ok {
-		return nil, errInvalidAccessKeyID
-	}
 
 	if globalPolicyOPA == nil {
-		// If OPA is not set and if ldap claim key is set,
-		// allow the claim.
-		if _, ok := claims[ldapUser]; ok {
-			return claims, nil
+		// If OPA is not set and if ldap claim key is set, allow the claim.
+		if _, ok := claims.Lookup(ldapUser); ok {
+			return claims.Map(), nil
 		}
 
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.
-		p, pok := claims[iamPolicyClaimName()]
+		_, pok := claims.Lookup(iamPolicyClaimName())
 		if !pok {
 			return nil, errAuthentication
 		}
-		if _, pok = p.(string); !pok {
-			return nil, errAuthentication
-		}
-		sp, spok := claims[iampolicy.SessionPolicyName]
-		// Sub policy is optional, if not set return success.
+
+		sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
 		if !spok {
-			return claims, nil
-		}
-		// Sub policy is set but its not a string, reject such requests
-		spStr, spok := sp.(string)
-		if !spok {
-			return nil, errAuthentication
+			return claims.Map(), nil
 		}
 		// Looks like subpolicy is set and is a string, if set then its
 		// base64 encoded, decode it. Decoding fails reject such requests.
-		spBytes, err := base64.StdEncoding.DecodeString(spStr)
+		spBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
 			logger.LogIf(context.Background(), err, logger.Application)
 			return nil, errAuthentication
 		}
-		claims[iampolicy.SessionPolicyName] = string(spBytes)
+		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
-	return claims, nil
+
+	return claims.Map(), nil
 }
 
 // Fetch claims in the security token returned by the client and validate the token.
@@ -402,12 +380,11 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 		err                       error
 		contentMD5, contentSHA256 []byte
 	)
+
 	// Extract 'Content-Md5' if present.
-	if _, ok := r.Header[xhttp.ContentMD5]; ok {
-		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get(xhttp.ContentMD5))
-		if err != nil || len(contentMD5) == 0 {
-			return ErrInvalidDigest
-		}
+	contentMD5, err = checkValidMD5(r.Header)
+	if err != nil {
+		return ErrInvalidDigest
 	}
 
 	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)

cmd/background-heal-ops.go

@@ -52,6 +52,19 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
+func waitForLowHTTPReq(tolerance int32) {
+	if httpServer := newHTTPServerFn(); httpServer != nil {
+		// Wait at max 10 minute for an inprogress request before proceeding to heal
+		waitCount := 600
+		// Any requests in progress, delay the heal.
+		for (httpServer.GetRequestCount() >= tolerance) &&
+			waitCount > 0 {
+			waitCount--
+			time.Sleep(1 * time.Second)
+		}
+	}
+}
+
 // Wait for heal requests and process them
 func (h *healRoutine) run() {
 	ctx := context.Background()
@@ -62,20 +75,12 @@ func (h *healRoutine) run() {
 				break
 			}
 
-			if httpServer := newHTTPServerFn(); httpServer != nil {
-				// Wait at max 10 minute for an inprogress request before proceeding to heal
-				waitCount := 600
-				// Any requests in progress, delay the heal.
-				for (httpServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) &&
-					waitCount > 0 {
-					waitCount--
-					time.Sleep(1 * time.Second)
-				}
-			}
+			// Wait and proceed if there are active requests
+			waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
 
 			var res madmin.HealResultItem
 			var err error
-			bucket, object := urlPath2BucketObjectName(task.path)
+			bucket, object := path2BucketObject(task.path)
 			switch {
 			case bucket == "" && object == "":
 				res, err = bgHealDiskFormat(ctx, task.opts)
@@ -122,7 +127,7 @@ func startBackgroundHealing() {
 
 	// Launch the background healer sequence to track
 	// background healing operations
-	info := objAPI.StorageInfo(ctx)
+	info := objAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
 	nh := newBgHealSequence(numDisks)
 	globalBackgroundHealState.LaunchNewHealSequence(nh)
@@ -181,5 +186,5 @@ func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOp
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}
-	return objectAPI.HealObject(ctx, bucket, object, opts.DryRun, opts.Remove, opts.ScanMode)
+	return objectAPI.HealObject(ctx, bucket, object, opts)
 }

cmd/bucket-encryption-handlers.go

@@ -0,0 +1,181 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/policy"
+)
+
+const (
+	// Bucket Encryption configuration file name.
+	bucketSSEConfig = "bucket-encryption.xml"
+)
+
+// PutBucketEncryptionHandler - Stores given bucket encryption configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketEncryption.html
+func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketEncryption")
+
+	defer logger.AuditLog(w, r, "PutBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// PutBucketEncyrption API requires Content-Md5
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Parse bucket encryption xml
+	encConfig, err := validateBucketSSEConfig(io.LimitReader(r.Body, maxBucketSSEConfigSize))
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Return error if KMS is not initialized
+	if GlobalKMS == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Store the bucket encryption configuration in the object layer
+	if err = objAPI.SetBucketSSEConfig(ctx, bucket, encConfig); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Update the in-memory bucket encryption cache
+	globalBucketSSEConfigSys.Set(bucket, *encConfig)
+
+	// Update peer MinIO servers of the updated bucket encryption config
+	globalNotificationSys.SetBucketSSEConfig(ctx, bucket, encConfig)
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketEncryptionHandler - Returns bucket policy configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketEncryption.html
+func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketEncryption")
+
+	defer logger.AuditLog(w, r, "GetBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists
+	var err error
+	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Fetch bucket encryption configuration from object layer
+	var encConfig *bucketsse.BucketSSEConfig
+	if encConfig, err = objAPI.GetBucketSSEConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	var encConfigData []byte
+	if encConfigData, err = xml.Marshal(encConfig); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write bucket encryption configuration to client
+	writeSuccessResponseXML(w, encConfigData)
+}
+
+// DeleteBucketEncryptionHandler - Removes bucket encryption configuration
+func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketEncryption")
+
+	defer logger.AuditLog(w, r, "DeleteBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists
+	var err error
+	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Delete bucket encryption config from object layer
+	if err = objAPI.DeleteBucketSSEConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Remove entry from the in-memory bucket encryption cache
+	globalBucketSSEConfigSys.Remove(bucket)
+	// Update peer MinIO servers of the updated bucket encryption config
+	globalNotificationSys.RemoveBucketSSEConfig(ctx, bucket)
+
+	writeSuccessNoContent(w)
+}

cmd/bucket-encryption.go

@@ -0,0 +1,169 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/xml"
+	"errors"
+	"io"
+	"path"
+	"sync"
+
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+)
+
+// BucketSSEConfigSys - in-memory cache of bucket encryption config
+type BucketSSEConfigSys struct {
+	sync.RWMutex
+	bucketSSEConfigMap map[string]bucketsse.BucketSSEConfig
+}
+
+// NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache
+func NewBucketSSEConfigSys() *BucketSSEConfigSys {
+	return &BucketSSEConfigSys{
+		bucketSSEConfigMap: make(map[string]bucketsse.BucketSSEConfig),
+	}
+}
+
+// load - Loads the bucket encryption configuration for the given list of buckets
+func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
+	for _, bucket := range buckets {
+		config, err := objAPI.GetBucketSSEConfig(context.Background(), bucket.Name)
+		if err != nil {
+			if _, ok := err.(BucketSSEConfigNotFound); ok {
+				sys.Remove(bucket.Name)
+			}
+			continue
+		}
+		sys.Set(bucket.Name, *config)
+	}
+
+	return nil
+}
+
+// Init - Initializes in-memory bucket encryption config cache for the given list of buckets
+func (sys *BucketSSEConfigSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// We don't cache bucket encryption config in gateway mode, nothing to do.
+	if globalIsGateway {
+		return nil
+	}
+
+	// Load bucket encryption config cache once during boot.
+	return sys.load(buckets, objAPI)
+}
+
+// Get - gets bucket encryption config for the given bucket.
+func (sys *BucketSSEConfigSys) Get(bucket string) (config bucketsse.BucketSSEConfig, ok bool) {
+	// We don't cache bucket encryption config in gateway mode.
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return
+		}
+
+		cfg, err := objAPI.GetBucketSSEConfig(context.Background(), bucket)
+		if err != nil {
+			return
+		}
+		return *cfg, true
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+	config, ok = sys.bucketSSEConfigMap[bucket]
+	return
+}
+
+// Set - sets bucket encryption config to given bucket name.
+func (sys *BucketSSEConfigSys) Set(bucket string, config bucketsse.BucketSSEConfig) {
+	// We don't cache bucket encryption config in gateway mode.
+	if globalIsGateway {
+		return
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+	sys.bucketSSEConfigMap[bucket] = config
+}
+
+// Remove - removes bucket encryption config for given bucket.
+func (sys *BucketSSEConfigSys) Remove(bucket string) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.bucketSSEConfigMap, bucket)
+}
+
+// saveBucketSSEConfig - save bucket encryption config for given bucket.
+func saveBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string, config *bucketsse.BucketSSEConfig) error {
+	data, err := xml.Marshal(config)
+	if err != nil {
+		return err
+	}
+
+	// Path to store bucket encryption config for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+	return saveConfig(ctx, objAPI, configFile, data)
+}
+
+// getBucketSSEConfig - get bucket encryption config for given bucket.
+func getBucketSSEConfig(objAPI ObjectLayer, bucket string) (*bucketsse.BucketSSEConfig, error) {
+	// Path to bucket-encryption.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+	configData, err := readConfig(context.Background(), objAPI, configFile)
+	if err != nil {
+		if err == errConfigNotFound {
+			err = BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+
+	return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
+}
+
+// removeBucketSSEConfig - removes bucket encryption config for given bucket.
+func removeBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
+	// Path to bucket-encryption.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+
+	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return err
+	}
+	return nil
+}
+
+// validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.
+func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
+	encConfig, err := bucketsse.ParseBucketSSEConfig(r)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(encConfig.Rules) == 1 && encConfig.Rules[0].DefaultEncryptionAction.Algorithm == bucketsse.AES256 {
+		return encConfig, nil
+	}
+	return nil, errors.New("Unsupported bucket encryption configuration")
+}

cmd/bucket-encryption_test.go

@@ -0,0 +1,70 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+)
+
+func TestValidateBucketSSEConfig(t *testing.T) {
+	testCases := []struct {
+		inputXML    string
+		expectedErr error
+		shouldPass  bool
+	}{
+		// MinIO supported XML
+		{
+			inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Rule>
+			<ApplyServerSideEncryptionByDefault>
+			<SSEAlgorithm>AES256</SSEAlgorithm>
+			</ApplyServerSideEncryptionByDefault>
+			</Rule>
+			</ServerSideEncryptionConfiguration>`,
+			expectedErr: nil,
+			shouldPass:  true,
+		},
+		// Unsupported XML
+		{
+			inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Rule>
+			<ApplyServerSideEncryptionByDefault>
+                        <SSEAlgorithm>aws:kms</SSEAlgorithm>
+                        <KMSMasterKeyID>arn:aws:kms:us-east-1:1234/5678example</KMSMasterKeyID>
+			</ApplyServerSideEncryptionByDefault>
+			</Rule>
+			</ServerSideEncryptionConfiguration>`,
+			expectedErr: errors.New("Unsupported bucket encryption configuration"),
+			shouldPass:  false,
+		},
+	}
+
+	for i, tc := range testCases {
+		_, err := validateBucketSSEConfig(bytes.NewReader([]byte(tc.inputXML)))
+		if tc.shouldPass && err != nil {
+			t.Fatalf("Test case %d: Expected to succeed but got %s", i+1, err)
+		}
+
+		if !tc.shouldPass {
+			if err == nil || err != nil && err.Error() != tc.expectedErr.Error() {
+				t.Fatalf("Test case %d: Expected %s but got %s", i+1, tc.expectedErr, err)
+			}
+		}
+	}
+}

cmd/bucket-handlers-listobjects.go

@@ -24,7 +24,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode

cmd/bucket-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -36,11 +35,12 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -74,40 +74,45 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		return
 	}
 
-	bucketSet := set.NewStringSet()
+	bucketsSet := set.NewStringSet()
+	bucketsToBeUpdated := set.NewStringSet()
+	bucketsInConflict := set.NewStringSet()
+	for _, bucket := range buckets {
+		bucketsSet.Add(bucket.Name)
+		r, ok := dnsBuckets[bucket.Name]
+		if !ok {
+			bucketsToBeUpdated.Add(bucket.Name)
+			continue
+		}
+		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+			if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// No difference in terms of domainIPs and nothing
+				// has changed so we don't change anything on the etcd.
+				continue
+			}
+			// if domain IPs intersect then it won't be an empty set.
+			// such an intersection means that bucket exists on etcd.
+			// but if we do see a difference with local domain IPs with
+			// hostSlice from etcd then we should update with newer
+			// domainIPs, we proceed to do that here.
+			bucketsToBeUpdated.Add(bucket.Name)
+			continue
+		}
+		// No IPs seem to intersect, this means that bucket exists but has
+		// different IP addresses perhaps from a different deployment.
+		// bucket names are globally unique in federation at a given
+		// path prefix, name collision is not allowed. We simply log
+		// an error and continue.
+		bucketsInConflict.Add(bucket.Name)
+	}
 
-	// Add buckets that are not registered with the DNS
+	// Add/update buckets that are not registered with the DNS
 	g := errgroup.WithNErrs(len(buckets))
-	for index := range buckets {
-		bucketSet.Add(buckets[index].Name)
+	bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice()
+	for index := range bucketsToBeUpdatedSlice {
 		index := index
 		g.Go(func() error {
-			r, gerr := globalDNSConfig.Get(buckets[index].Name)
-			if gerr != nil {
-				if gerr == dns.ErrNoEntriesFound {
-					return globalDNSConfig.Put(buckets[index].Name)
-				}
-				return gerr
-			}
-			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-					// No difference in terms of domainIPs and nothing
-					// has changed so we don't change anything on the etcd.
-					return nil
-				}
-				// if domain IPs intersect then it won't be an empty set.
-				// such an intersection means that bucket exists on etcd.
-				// but if we do see a difference with local domain IPs with
-				// hostSlice from etcd then we should update with newer
-				// domainIPs, we proceed to do that here.
-				return globalDNSConfig.Put(buckets[index].Name)
-			}
-			// No IPs seem to intersect, this means that bucket exists but has
-			// different IP addresses perhaps from a different deployment.
-			// bucket names are globally unique in federation at a given
-			// path prefix, name collision is not allowed. We simply log
-			// an error and continue.
-			return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", buckets[index].Name, globalDomainIPs.ToSlice())
+			return globalDNSConfig.Put(bucketsToBeUpdatedSlice[index])
 		}, index)
 	}
 
@@ -117,36 +122,26 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		}
 	}
 
-	g = errgroup.WithNErrs(len(dnsBuckets))
-	// Remove buckets that are in DNS for this server, but aren't local
-	for index := range dnsBuckets {
-		index := index
-		g.Go(func() error {
-			// This is a local bucket that exists, so we can continue
-			if bucketSet.Contains(dnsBuckets[index].Key) {
-				return nil
-			}
-
-			// This is not for our server, so we can continue
-			hostPort := net.JoinHostPort(dnsBuckets[index].Host, string(dnsBuckets[index].Port))
-			if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
-				return nil
-			}
-
-			// We go to here, so we know the bucket no longer exists,
-			// but is registered in DNS to this server
-			if err := globalDNSConfig.DeleteRecord(dnsBuckets[index]); err != nil {
-				return fmt.Errorf("Failed to remove DNS entry for %s due to %w",
-					dnsBuckets[index].Key, err)
-			}
-
-			return nil
-		}, index)
+	for _, bucket := range bucketsInConflict.ToSlice() {
+		logger.LogIf(context.Background(), fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
-	for _, err := range g.Wait() {
-		if err != nil {
-			logger.LogIf(context.Background(), err)
+	// Remove buckets that are in DNS for this server, but aren't local
+	for bucket, records := range dnsBuckets {
+		if bucketsSet.Contains(bucket) {
+			continue
+		}
+
+		if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(records)...)).IsEmpty() {
+			// This is not for our server, so we can continue
+			continue
+		}
+
+		// We go to here, so we know the bucket no longer exists,
+		// but is registered in DNS to this server
+		if err = globalDNSConfig.Delete(bucket); err != nil {
+			logger.LogIf(context.Background(), fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+				bucket, err))
 		}
 	}
 }
@@ -284,16 +279,11 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		bucketSet := set.NewStringSet()
-		for _, dnsRecord := range dnsBuckets {
-			if bucketSet.Contains(dnsRecord.Key) {
-				continue
-			}
+		for _, dnsRecords := range dnsBuckets {
 			bucketsInfo = append(bucketsInfo, BucketInfo{
-				Name:    dnsRecord.Key,
-				Created: dnsRecord.CreationDate,
+				Name:    dnsRecords[0].Key,
+				Created: dnsRecords[0].CreationDate,
 			})
-			bucketSet.Add(dnsRecord.Key)
 		}
 	} else {
 		// Invoke the list buckets.
@@ -352,6 +342,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
@@ -359,34 +356,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Content-Md5 is requied should be set
-	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Allocate incoming content length bytes.
-	var deleteXMLBytes []byte
-	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
-	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
-		deleteXMLBytes = make([]byte, maxBodySize)
-	} else {
-		deleteXMLBytes = make([]byte, r.ContentLength)
-	}
-
-	// Read incoming body XML bytes.
-	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
+	// The max. XML contains 100000 object names (each at most 1024 bytes long) + XML overhead
+	const maxBodySize = 2 * 100000 * 1024
 
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
-	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
+	if err := xmlDecoder(r.Body, deleteObjects, maxBodySize); err != nil {
 		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -533,7 +510,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	if globalDNSConfig != nil {
-		if _, err := globalDNSConfig.Get(bucket); err != nil {
+		sr, err := globalDNSConfig.Get(bucket)
+		if err != nil {
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
@@ -566,7 +544,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			return
 
 		}
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBucketAlreadyOwnedByYou), r.URL, guessIsBrowserReq(r))
+		apiErr := ErrBucketAlreadyExists
+		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+			apiErr = ErrBucketAlreadyOwnedByYou
+		}
+		// No IPs seem to intersect, this means that bucket exists but has
+		// different IP addresses perhaps from a different deployment.
+		// bucket names are globally unique in federation at a given
+		// path prefix, name collision is not allowed. Return appropriate error.
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -577,14 +563,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if objectLockEnabled {
+	if objectLockEnabled && !globalIsGateway {
 		configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
 		if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		globalBucketObjectLockConfig.Set(bucket, Retention{})
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, Retention{})
+		globalBucketObjectLockConfig.Set(bucket, objectlock.Retention{})
+		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, objectlock.Retention{})
 	}
 
 	// Make sure to add Location information here only for bucket
@@ -758,8 +744,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	pReader := NewPutObjReader(rawReader, nil, nil)
 	var objectEncryptionKey []byte
 
+	// Check if bucket encryption is enabled
+	_, encEnabled := globalBucketSSEConfigSys.Get(bucket)
 	// This request header needs to be set prior to setting ObjectOptions
-	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+	if (globalAutoEncryption || encEnabled) && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
 	// get gateway encryption options
@@ -1005,7 +993,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	config, err := parseObjectLockConfig(r.Body)
+	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		apiErr.Description = err.Error()
@@ -1099,7 +1087,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 			return
 		}
 
-		if configData, err = xml.Marshal(newObjectLockConfig()); err != nil {
+		if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}

cmd/bucket-lifecycle-handler.go

@@ -22,9 +22,10 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/lifecycle"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 const (
@@ -48,6 +49,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	// PutBucketLifecycle always needs a Content-Md5
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -59,15 +66,9 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// PutBucketLifecycle always needs a Content-Md5
-	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
 	bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/bucket-notification-handlers.go

@@ -30,8 +30,8 @@ import (
 	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -84,8 +84,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 		config.SetRegion(globalServerRegion)
+		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 		notificationBytes, err := xml.Marshal(config)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/bucket-policy-handlers.go

@@ -24,7 +24,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 const (

cmd/bucket-policy-handlers_test.go

@@ -29,8 +29,8 @@ import (
 	"testing"
 
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/policy/condition"
 )
 
 func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {

cmd/common-main.go

@@ -17,9 +17,12 @@
 package cmd
 
 import (
+	"context"
 	"crypto/x509"
+	"encoding/gob"
 	"errors"
 	"net"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -34,6 +37,17 @@ import (
 	"github.com/minio/minio/pkg/env"
 )
 
+func init() {
+	logger.Init(GOPATH, GOROOT)
+	logger.RegisterError(config.FmtError)
+
+	// Initialize globalConsoleSys system
+	globalConsoleSys = NewConsoleLogger(context.Background())
+	logger.AddTarget(globalConsoleSys)
+
+	gob.Register(StorageErr(""))
+}
+
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 	if (globalAutoEncryption || GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
 		logger.Fatal(errInvalidArgument,
@@ -213,10 +227,22 @@ func handleCommonEnvVars() {
 		globalConfigEncrypted = true
 	}
 
+	if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
+		oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate the old credentials inherited from the shell environment")
+		}
+		globalOldCred = oldCred
+		os.Unsetenv(config.EnvAccessKeyOld)
+		os.Unsetenv(config.EnvSecretKeyOld)
+	}
+
 	globalWORMEnabled, err = config.LookupWorm()
 	if err != nil {
 		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
 	}
+
 }
 
 func logStartupMessage(msg string) {

cmd/config-current.go

@@ -26,7 +26,6 @@ import (
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
 	"github.com/minio/minio/cmd/config/etcd"
-	xetcd "github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
@@ -38,6 +37,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 func initHelp() {
@@ -303,13 +303,13 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	etcdCfg, err := xetcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
+	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 	}
 
 	if etcdCfg.Enabled {
-		globalEtcdClient, err = xetcd.New(etcdCfg)
+		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 		}
@@ -427,6 +427,10 @@ func lookupConfigs(s config.Config) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
+	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalServiceDoneCh, NewCustomHTTPTransport())
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
+	}
 }
 
 // Help - return sub-system level help
@@ -469,6 +473,20 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 
 	envHelp := config.HelpKVS{}
 	if envOnly {
+		// Only for multiple targets, make sure
+		// to list the ENV, for regular k/v EnableKey is
+		// implicit, for ENVs we cannot make it implicit.
+		if subSysHelp.MultipleTargets {
+			envK := config.EnvPrefix + strings.Join([]string{
+				strings.ToTitle(subSys), strings.ToTitle(madmin.EnableKey),
+			}, config.EnvWordDelimiter)
+			envHelp = append(envHelp, config.HelpKV{
+				Key:         envK,
+				Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
+				Optional:    false,
+				Type:        "on|off",
+			})
+		}
 		for _, hkv := range h {
 			envK := config.EnvPrefix + strings.Join([]string{
 				strings.ToTitle(subSys), strings.ToTitle(hkv.Key),

cmd/config-encrypted.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 	"unicode/utf8"
 
@@ -29,7 +28,6 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -88,11 +86,6 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	doneCh = make(chan struct{})
 	defer close(doneCh)
 
@@ -106,7 +99,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		select {
 		case <-retryTimerCh:
 			// Migrate IAM configuration
-			if err = migrateConfigPrefixToEncrypted(objAPI, activeCredOld, encrypted); err != nil {
+			if err = migrateConfigPrefixToEncrypted(objAPI, globalOldCred, encrypted); err != nil {
 				if err == errDiskNotFound ||
 					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
 					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
@@ -164,21 +157,6 @@ func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
 	return data, err
 }
 
-func getOldCreds() (activeCredOld auth.Credentials, err error) {
-	accessKeyOld := env.Get(config.EnvAccessKeyOld, "")
-	secretKeyOld := env.Get(config.EnvSecretKeyOld, "")
-	if accessKeyOld != "" && secretKeyOld != "" {
-		activeCredOld, err = auth.CreateCredentials(accessKeyOld, secretKeyOld)
-		if err != nil {
-			return activeCredOld, err
-		}
-		// Once we have obtained the rotating creds
-		os.Unsetenv(config.EnvAccessKeyOld)
-		os.Unsetenv(config.EnvSecretKeyOld)
-	}
-	return activeCredOld, nil
-}
-
 func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	defer cancel()
@@ -206,20 +184,15 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	if encrypted {
 		// No key rotation requested, and backend is
 		// already encrypted. We proceed without migration.
-		if !activeCredOld.IsValid() {
+		if !globalOldCred.IsValid() {
 			return nil
 		}
 
 		// No real reason to rotate if old and new creds are same.
-		if activeCredOld.Equal(globalActiveCred) {
+		if globalOldCred.Equal(globalActiveCred) {
 			return nil
 		}
 
@@ -254,8 +227,8 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 
 		var data []byte
 		// Is rotating of creds requested?
-		if activeCredOld.IsValid() {
-			data, err = decryptData(cdata, activeCredOld, globalActiveCred)
+		if globalOldCred.IsValid() {
+			data, err = decryptData(cdata, globalOldCred, globalActiveCred)
 			if err != nil {
 				if err == madmin.ErrMaliciousData {
 					return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
@@ -285,7 +258,7 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 	}
 
-	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
+	if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
 

cmd/config.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 	"path"
 	"sort"
 	"strings"
@@ -28,7 +27,6 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -212,33 +210,7 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	// Initializing configuration needs a retry mechanism for
-	// the following reasons:
-	//  - Read quorum is lost just after the initialization
-	//    of the object layer.
-	//  - Write quorum not met when upgrading configuration
-	//    version is needed.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case <-retryTimerCh:
-			if err := initConfig(objAPI); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for configuration to be initialized..")
-					continue
-				}
-				return err
-			}
-			return nil
-		case <-globalOSSignalCh:
-			return fmt.Errorf("Initializing config sub-system gracefully stopped")
-		}
-	}
+	return initConfig(objAPI)
 }
 
 // NewConfigSys - creates new config system object.

cmd/config/cache/config.go

@@ -28,12 +28,15 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled bool     `json:"-"`
-	Drives  []string `json:"drives"`
-	Expiry  int      `json:"expiry"`
-	MaxUse  int      `json:"maxuse"`
-	Quota   int      `json:"quota"`
-	Exclude []string `json:"exclude"`
+	Enabled       bool     `json:"-"`
+	Drives        []string `json:"drives"`
+	Expiry        int      `json:"expiry"`
+	MaxUse        int      `json:"maxuse"`
+	Quota         int      `json:"quota"`
+	Exclude       []string `json:"exclude"`
+	After         int      `json:"after"`
+	WatermarkLow  int      `json:"watermark_low"`
+	WatermarkHigh int      `json:"watermark_high"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -60,7 +63,18 @@ func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
 	if _cfg.Quota < 0 {
 		return errors.New("config quota value should not be null or negative")
 	}
-
+	if _cfg.After < 0 {
+		return errors.New("cache after value should not be less than 0")
+	}
+	if _cfg.WatermarkLow < 0 || _cfg.WatermarkLow > 100 {
+		return errors.New("config low watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkHigh < 0 || _cfg.WatermarkHigh > 100 {
+		return errors.New("config high watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkLow > 0 && (_cfg.WatermarkLow >= _cfg.WatermarkHigh) {
+		return errors.New("config low watermark value should be less than high watermark")
+	}
 	return nil
 }
 

cmd/config/cache/help.go

@@ -50,5 +50,23 @@ var (
 			Optional:    true,
 			Type:        "sentence",
 		},
+		config.HelpKV{
+			Key:         After,
+			Description: `minimum accesses before caching an object`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         WatermarkLow,
+			Description: `% of cache use at which to stop cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         WatermarkHigh,
+			Description: `% of cache use at which to start cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -26,21 +26,31 @@ import (
 
 // Cache ENVs
 const (
-	Drives  = "drives"
-	Exclude = "exclude"
-	Expiry  = "expiry"
-	MaxUse  = "maxuse"
-	Quota   = "quota"
+	Drives        = "drives"
+	Exclude       = "exclude"
+	Expiry        = "expiry"
+	MaxUse        = "maxuse"
+	Quota         = "quota"
+	After         = "after"
+	WatermarkLow  = "watermark_low"
+	WatermarkHigh = "watermark_high"
+
+	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry        = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse        = "MINIO_CACHE_MAXUSE"
+	EnvCacheQuota         = "MINIO_CACHE_QUOTA"
+	EnvCacheAfter         = "MINIO_CACHE_AFTER"
+	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
+	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 
-	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
-	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
-	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
-	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
-	EnvCacheQuota               = "MINIO_CACHE_QUOTA"
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
-	DefaultExpiry = "90"
-	DefaultQuota  = "80"
+	DefaultExpiry        = "90"
+	DefaultQuota         = "80"
+	DefaultAfter         = "0"
+	DefaultWaterMarkLow  = "70"
+	DefaultWaterMarkHigh = "80"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -62,6 +72,18 @@ var (
 			Key:   Quota,
 			Value: DefaultQuota,
 		},
+		config.KV{
+			Key:   After,
+			Value: DefaultAfter,
+		},
+		config.KV{
+			Key:   WatermarkLow,
+			Value: DefaultWaterMarkLow,
+		},
+		config.KV{
+			Key:   WatermarkHigh,
+			Value: DefaultWaterMarkHigh,
+		},
 	}
 )
 
@@ -79,7 +101,6 @@ func Enabled(kvs config.KVS) bool {
 // variables and merge them with provided CacheConfiguration.
 func LookupConfig(kvs config.KVS) (Config, error) {
 	cfg := Config{}
-
 	if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
@@ -134,5 +155,45 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		cfg.MaxUse = cfg.Quota
 	}
 
+	if afterStr := env.Get(EnvCacheAfter, kvs.Get(After)); afterStr != "" {
+		cfg.After, err = strconv.Atoi(afterStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheAfter(err)
+		}
+		// after should be a valid value >= 0.
+		if cfg.After < 0 {
+			err := errors.New("cache after value cannot be less than 0")
+			return cfg, config.ErrInvalidCacheAfter(err)
+		}
+	}
+
+	if lowWMStr := env.Get(EnvCacheWatermarkLow, kvs.Get(WatermarkLow)); lowWMStr != "" {
+		cfg.WatermarkLow, err = strconv.Atoi(lowWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+		// WatermarkLow should be a valid percentage.
+		if cfg.WatermarkLow < 0 || cfg.WatermarkLow > 100 {
+			err := errors.New("config min watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+	}
+
+	if highWMStr := env.Get(EnvCacheWatermarkHigh, kvs.Get(WatermarkHigh)); highWMStr != "" {
+		cfg.WatermarkHigh, err = strconv.Atoi(highWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+
+		// MaxWatermark should be a valid percentage.
+		if cfg.WatermarkHigh < 0 || cfg.WatermarkHigh > 100 {
+			err := errors.New("config high watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+	}
+	if cfg.WatermarkLow > cfg.WatermarkHigh {
+		err := errors.New("config high watermark value should be greater than low watermark value")
+		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+	}
 	return cfg, nil
 }

cmd/config/errors-utils.go

@@ -46,7 +46,7 @@ func (u Err) Clone() Err {
 	}
 }
 
-// Return the error message
+// Error returns the error message
 func (u Err) Error() string {
 	if u.detail == "" {
 		if u.msg != "" {

cmd/config/errors.go

@@ -66,6 +66,24 @@ var (
 		"MINIO_CACHE_QUOTA: Valid cache quota value must be between 0-100",
 	)
 
+	ErrInvalidCacheAfter = newErrFn(
+		"Invalid cache after value",
+		"Please check the passed value",
+		"MINIO_CACHE_AFTER: Valid cache after value must be 0 or greater",
+	)
+
+	ErrInvalidCacheWatermarkLow = newErrFn(
+		"Invalid cache low watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_LOW: Valid cache low watermark value must be between 0-100",
+	)
+
+	ErrInvalidCacheWatermarkHigh = newErrFn(
+		"Invalid cache high watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_HIGH: Valid cache high watermark value must be between 0-100",
+	)
+
 	ErrInvalidCacheEncryptionKey = newErrFn(
 		"Invalid cache encryption master key value",
 		"Please check the passed value",

cmd/config/etcd/dns/etcd_dns.go

@@ -38,18 +38,18 @@ var ErrNoEntriesFound = errors.New("No entries found for this key")
 const etcdPathSeparator = "/"
 
 // create a new coredns service record for the bucket.
-func newCoreDNSMsg(ip string, port string, ttl uint32) ([]byte, error) {
+func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, error) {
 	return json.Marshal(&SrvRecord{
 		Host:         ip,
 		Port:         json.Number(port),
 		TTL:          ttl,
-		CreationDate: time.Now().UTC(),
+		CreationDate: t,
 	})
 }
 
 // List - Retrieves list of DNS entries for the domain.
-func (c *CoreDNS) List() ([]SrvRecord, error) {
-	var srvRecords []SrvRecord
+func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
+	var srvRecords = map[string][]SrvRecord{}
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
 		records, err := c.list(key)
@@ -60,7 +60,7 @@ func (c *CoreDNS) List() ([]SrvRecord, error) {
 			if record.Key == "" {
 				continue
 			}
-			srvRecords = append(srvRecords, record)
+			srvRecords[record.Key] = append(srvRecords[record.Key], record)
 		}
 	}
 	return srvRecords, nil
@@ -150,8 +150,9 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 func (c *CoreDNS) Put(bucket string) error {
 	c.Delete(bucket) // delete any existing entries.
 
+	t := time.Now().UTC()
 	for ip := range c.domainIPs {
-		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL)
+		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL, t)
 		if err != nil {
 			return err
 		}

cmd/config/identity/openid/jwt.go

@@ -251,6 +251,7 @@ func parseDiscoveryDoc(u *xnet.URL, transport *http.Transport, closeRespFn func(
 	}
 	resp, err := clnt.Do(req)
 	if err != nil {
+		clnt.CloseIdleConnections()
 		return d, err
 	}
 	defer closeRespFn(resp.Body)

cmd/config/notify/help.go

@@ -213,6 +213,12 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         target.KafkaVersion,
+			Description: "specify the version of the Kafka cluster",
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/notify/parse.go

@@ -42,7 +42,14 @@ const (
 func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport,
 	targetIDs []event.TargetID) error {
 	test := true
-	_, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test)
+	targets, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test)
+	if err == nil {
+		// Close all targets since we are only testing connections.
+		for _, t := range targets.TargetMap() {
+			_ = t.Close()
+		}
+	}
+
 	return err
 }
 
@@ -58,9 +65,19 @@ func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool) (*event.TargetList, error) {
+func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool) (_ *event.TargetList, err error) {
 	targetList := event.NewTargetList()
-	if err := checkValidNotificationKeys(cfg); err != nil {
+
+	defer func() {
+		// Automatically close all connections to targets when an error occur
+		if err != nil {
+			for _, t := range targetList.TargetMap() {
+				_ = t.Close()
+			}
+		}
+	}()
+
+	if err = checkValidNotificationKeys(cfg); err != nil {
 		return nil, err
 	}
 
@@ -125,9 +142,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range esTargets {
@@ -142,9 +156,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range kafkaTargets {
@@ -159,9 +170,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range mqttTargets {
@@ -176,9 +184,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range mysqlTargets {
@@ -192,9 +197,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range natsTargets {
@@ -208,9 +210,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range nsqTargets {
@@ -224,9 +223,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range postgresTargets {
@@ -240,9 +236,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range redisTargets {
@@ -256,9 +249,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range webhookTargets {
@@ -272,10 +262,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err := targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-			continue
-		}
 	}
 
 	if test {
@@ -398,6 +384,10 @@ var (
 			Key:   target.KafkaQueueDir,
 			Value: "",
 		},
+		config.KV{
+			Key:   target.KafkaVersion,
+			Value: "",
+		},
 	}
 )
 
@@ -465,12 +455,18 @@ func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs
 			queueDirEnv = queueDirEnv + config.Default + k
 		}
 
+		versionEnv := target.EnvKafkaVersion
+		if k != config.Default {
+			versionEnv = versionEnv + config.Default + k
+		}
+
 		kafkaArgs := target.KafkaArgs{
 			Enable:     enabled,
 			Brokers:    brokers,
 			Topic:      env.Get(topicEnv, kv.Get(target.KafkaTopic)),
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.KafkaQueueDir)),
 			QueueLimit: queueLimit,
+			Version:    env.Get(versionEnv, kv.Get(target.KafkaVersion)),
 		}
 
 		tlsEnableEnv := target.EnvKafkaTLS

cmd/config/policy/opa/config.go

@@ -223,5 +223,6 @@ func (o *Opa) IsAllowed(args iampolicy.Args) (bool, error) {
 		}
 		return resultAllow.Result.Allow, nil
 	}
+
 	return result.Result, nil
 }

cmd/daily-lifecycle-ops.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/lifecycle"
 )
 
 const (
@@ -104,12 +104,12 @@ func startDailyLifecycle() {
 	}
 }
 
-var lifecycleTimeout = newDynamicTimeout(60*time.Second, time.Second)
+var lifecycleLockTimeout = newDynamicTimeout(60*time.Second, time.Second)
 
 func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 	// Lock to avoid concurrent lifecycle ops from other nodes
 	sweepLock := objAPI.NewNSLock(ctx, "system", "daily-lifecycle-ops")
-	if err := sweepLock.GetLock(lifecycleTimeout); err != nil {
+	if err := sweepLock.GetLock(lifecycleLockTimeout); err != nil {
 		return err
 	}
 	defer sweepLock.Unlock()
@@ -129,29 +129,39 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 		// Calculate the common prefix of all lifecycle rules
 		var prefixes []string
 		for _, rule := range l.Rules {
-			prefixes = append(prefixes, rule.Filter.Prefix)
+			prefixes = append(prefixes, rule.Prefix())
 		}
 		commonPrefix := lcp(prefixes)
 
-		// List all objects and calculate lifecycle action based on object name & object modtime
-		marker := ""
+		// Allocate new results channel to receive ObjectInfo.
+		objInfoCh := make(chan ObjectInfo)
+
+		// Walk through all objects
+		if err := objAPI.Walk(ctx, bucket.Name, commonPrefix, objInfoCh); err != nil {
+			return err
+		}
+
 		for {
-			res, err := objAPI.ListObjects(ctx, bucket.Name, commonPrefix, marker, "", maxObjectList)
-			if err != nil {
-				continue
-			}
 			var objects []string
-			for _, obj := range res.Objects {
+			for obj := range objInfoCh {
+				if len(objects) == maxObjectList {
+					// Reached maximum delete requests, attempt a delete for now.
+					break
+				}
+
 				// Find the action that need to be executed
-				action := l.ComputeAction(obj.Name, obj.ModTime)
-				switch action {
-				case lifecycle.DeleteAction:
+				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
 					objects = append(objects, obj.Name)
-				default:
-					// Do nothing, for now.
 				}
 			}
 
+			// Nothing to do.
+			if len(objects) == 0 {
+				break
+			}
+
+			waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
+
 			// Deletes a list of objects.
 			deleteErrs, err := objAPI.DeleteObjects(ctx, bucket.Name, objects)
 			if err != nil {
@@ -173,12 +183,6 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 					})
 				}
 			}
-
-			if !res.IsTruncated {
-				// We are done here, proceed to next bucket.
-				break
-			}
-			marker = res.NextMarker
 		}
 	}
 

cmd/data-usage-cache.go

@@ -0,0 +1,517 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/tinylib/msgp/msgp"
+)
+
+const dataUsageHashLen = 8
+
+//go:generate msgp -file $GOFILE -unexported
+
+// dataUsageHash is the hash type used.
+type dataUsageHash uint64
+
+// sizeHistogram is a size histogram.
+type sizeHistogram [dataUsageBucketLen]uint64
+
+//msgp:tuple dataUsageEntry
+type dataUsageEntry struct {
+	// These fields do no include any children.
+	Size     int64
+	Objects  uint64
+	ObjSizes sizeHistogram
+
+	Children dataUsageHashMap
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageEntryInfo struct {
+	Name   string
+	Parent string
+	Entry  dataUsageEntry
+}
+
+type dataUsageCacheInfo struct {
+	// Name of the bucket. Also root element.
+	Name       string
+	LastUpdate time.Time
+	NextCycle  uint8
+}
+
+// merge other data usage entry into this, excluding children.
+func (e *dataUsageEntry) merge(other dataUsageEntry) {
+	e.Objects += other.Objects
+	e.Size += other.Size
+	for i, v := range other.ObjSizes[:] {
+		e.ObjSizes[i] += v
+	}
+}
+
+// mod returns true if the hash mod cycles == cycle.
+func (h dataUsageHash) mod(cycle uint8, cycles uint8) bool {
+	return uint8(h)%cycles == cycle%cycles
+}
+
+// addChildString will add a child based on its name.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChildString(name string) {
+	e.addChild(hashPath(name))
+}
+
+// addChild will add a child based on its hash.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChild(hash dataUsageHash) {
+	if _, ok := e.Children[hash]; ok {
+		return
+	}
+	if e.Children == nil {
+		e.Children = make(dataUsageHashMap, 1)
+	}
+	e.Children[hash] = struct{}{}
+}
+
+// find a path in the cache.
+// Returns nil if not found.
+func (d *dataUsageCache) find(path string) *dataUsageEntry {
+	due, ok := d.Cache[hashPath(path)]
+	if !ok {
+		return nil
+	}
+	return &due
+}
+
+// dui converts the flattened version of the path to DataUsageInfo.
+func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
+	e := d.find(path)
+	if e == nil {
+		return DataUsageInfo{LastUpdate: UTCNow()}
+	}
+	flat := d.flatten(*e)
+	return DataUsageInfo{
+		LastUpdate:            d.Info.LastUpdate,
+		ObjectsCount:          flat.Objects,
+		ObjectsTotalSize:      uint64(flat.Size),
+		ObjectsSizesHistogram: flat.ObjSizes.asMap(),
+		BucketsCount:          uint64(len(e.Children)),
+		BucketsSizes:          d.pathSizes(buckets),
+	}
+}
+
+// replace will add or replace an entry in the cache.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
+	hash := hashPath(path)
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != "" {
+		phash := hashPath(parent)
+		p := d.Cache[phash]
+		p.addChild(hash)
+		d.Cache[phash] = p
+	}
+}
+
+// replaceHashed add or replaces an entry to the cache based on its hash.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) {
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != nil {
+		p := d.Cache[*parent]
+		p.addChild(hash)
+		d.Cache[*parent] = p
+	}
+}
+
+// StringAll returns a detailed string representation of all entries in the cache.
+func (d *dataUsageCache) StringAll() string {
+	s := fmt.Sprintf("info:%+v\n", d.Info)
+	for k, v := range d.Cache {
+		s += fmt.Sprintf("\t%v: %+v\n", k, v)
+	}
+	return strings.TrimSpace(s)
+}
+
+// String returns a human readable representation of the string.
+func (h dataUsageHash) String() string {
+	return fmt.Sprintf("%x", uint64(h))
+}
+
+// flatten all children of the root into the root element and return it.
+func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
+	for id := range root.Children {
+		e := d.Cache[id]
+		if len(e.Children) > 0 {
+			e = d.flatten(e)
+		}
+		root.merge(e)
+	}
+	root.Children = nil
+	return root
+}
+
+// add a size to the histogram.
+func (h *sizeHistogram) add(size int64) {
+	// Fetch the histogram interval corresponding
+	// to the passed object size.
+	for i, interval := range ObjectsHistogramIntervals {
+		if size >= interval.start && size <= interval.end {
+			h[i]++
+			break
+		}
+	}
+}
+
+// asMap returns the map as a map[string]uint64.
+func (h *sizeHistogram) asMap() map[string]uint64 {
+	res := make(map[string]uint64, 7)
+	for i, count := range h {
+		res[ObjectsHistogramIntervals[i].name] = count
+	}
+	return res
+}
+
+// pathSizes returns the path sizes as a map.
+func (d *dataUsageCache) pathSizes(buckets []BucketInfo) map[string]uint64 {
+	var dst = make(map[string]uint64, len(buckets))
+	for _, bucket := range buckets {
+		e := d.find(bucket.Name)
+		if e == nil {
+			continue
+		}
+		flat := d.flatten(*e)
+		dst[bucket.Name] = uint64(flat.Size)
+	}
+	return dst
+}
+
+// sizeRecursive returns the path as a flattened entry.
+func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
+	root := d.find(path)
+	if root == nil || len(root.Children) == 0 {
+		return root
+	}
+	flat := d.flatten(*root)
+	return &flat
+}
+
+// dataUsageCache contains a cache of data usage entries.
+//msgp:ignore dataUsageCache
+type dataUsageCache struct {
+	Info  dataUsageCacheInfo
+	Cache map[dataUsageHash]dataUsageEntry
+}
+
+// root returns the root of the cache.
+func (d *dataUsageCache) root() *dataUsageEntry {
+	return d.find(d.Info.Name)
+}
+
+// rootHash returns the root of the cache.
+func (d *dataUsageCache) rootHash() dataUsageHash {
+	return hashPath(d.Info.Name)
+}
+
+// clone returns a copy of the cache with no references to the existing.
+func (d *dataUsageCache) clone() dataUsageCache {
+	clone := dataUsageCache{
+		Info:  d.Info,
+		Cache: make(map[dataUsageHash]dataUsageEntry, len(d.Cache)),
+	}
+	for k, v := range d.Cache {
+		clone.Cache[k] = v
+	}
+	return clone
+}
+
+// merge root of other into d.
+// children of root will be flattened before being merged.
+// Last update time will be set to the last updated.
+func (d *dataUsageCache) merge(other dataUsageCache) {
+	existingRoot := d.root()
+	otherRoot := other.root()
+	if existingRoot == nil && otherRoot == nil {
+		return
+	}
+	if otherRoot == nil {
+		return
+	}
+	if existingRoot == nil {
+		*d = other.clone()
+		return
+	}
+	if other.Info.LastUpdate.After(d.Info.LastUpdate) {
+		d.Info.LastUpdate = other.Info.LastUpdate
+	}
+	existingRoot.merge(*otherRoot)
+	eHash := d.rootHash()
+	for key := range otherRoot.Children {
+		entry := other.Cache[key]
+		flat := other.flatten(entry)
+		existing := d.Cache[key]
+		// If not found, merging simply adds.
+		existing.merge(flat)
+		d.replaceHashed(key, &eHash, existing)
+	}
+}
+
+// load the cache content with name from minioMetaBackgroundOpsBucket.
+// Only backend errors are returned as errors.
+// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name string) error {
+	var buf bytes.Buffer
+	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
+	if err != nil {
+		if !isErrObjectNotFound(err) {
+			return toObjectErr(err, dataUsageBucket, name)
+		}
+		*d = dataUsageCache{}
+		return nil
+	}
+	err = d.deserialize(buf.Bytes())
+	if err != nil {
+		*d = dataUsageCache{}
+		logger.LogIf(ctx, err)
+	}
+	return nil
+}
+
+// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
+func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name string) error {
+	b := d.serialize()
+	size := int64(len(b))
+	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)
+	if err != nil {
+		return err
+	}
+
+	_, err = store.PutObject(ctx,
+		dataUsageBucket,
+		name,
+		NewPutObjReader(r, nil, nil),
+		ObjectOptions{})
+	return err
+}
+
+// dataUsageCacheVer indicates the cache version.
+// Bumping the cache version will drop data from previous versions
+// and write new data with the new version.
+const dataUsageCacheVer = 1
+
+// serialize the contents of the cache.
+func (d *dataUsageCache) serialize() []byte {
+	// Alloc pessimistically
+	// dataUsageCacheVer
+	due := dataUsageEntry{}
+	msgLen := 1
+	msgLen += d.Info.Msgsize()
+	// len(d.Cache)
+	msgLen += binary.MaxVarintLen64
+	// Hashes (one for key, assume 1 child/node)
+	msgLen += len(d.Cache) * dataUsageHashLen * 2
+	msgLen += len(d.Cache) * due.Msgsize()
+
+	// Create destination buffer...
+	dst := make([]byte, 0, msgLen)
+
+	var n int
+	tmp := make([]byte, 1024)
+	// byte: version.
+	dst = append(dst, dataUsageCacheVer)
+	// Info...
+	dst, err := d.Info.MarshalMsg(dst)
+	if err != nil {
+		panic(err)
+	}
+	n = binary.PutUvarint(tmp, uint64(len(d.Cache)))
+	dst = append(dst, tmp[:n]...)
+
+	for k, v := range d.Cache {
+		// Put key
+		binary.LittleEndian.PutUint64(tmp[:dataUsageHashLen], uint64(k))
+		dst = append(dst, tmp[:8]...)
+		tmp, err = v.MarshalMsg(tmp[:0])
+		if err != nil {
+			panic(err)
+		}
+		// key, value pairs.
+		dst = append(dst, tmp...)
+
+	}
+	return dst
+}
+
+// deserialize the supplied byte slice into the cache.
+func (d *dataUsageCache) deserialize(b []byte) error {
+	if len(b) < 1 {
+		return io.ErrUnexpectedEOF
+	}
+	switch b[0] {
+	case 1:
+	default:
+		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
+	}
+	b = b[1:]
+
+	// Info...
+	b, err := d.Info.UnmarshalMsg(b)
+	if err != nil {
+		return err
+	}
+	cacheLen, n := binary.Uvarint(b)
+	if n <= 0 {
+		return fmt.Errorf("dataUsageCache: reading cachelen, n <= 0 ")
+	}
+	b = b[n:]
+	d.Cache = make(map[dataUsageHash]dataUsageEntry, cacheLen)
+
+	for i := 0; i < int(cacheLen); i++ {
+		if len(b) <= dataUsageHashLen {
+			return io.ErrUnexpectedEOF
+		}
+		k := binary.LittleEndian.Uint64(b[:dataUsageHashLen])
+		b = b[dataUsageHashLen:]
+		var v dataUsageEntry
+		b, err = v.UnmarshalMsg(b)
+		if err != nil {
+			return err
+		}
+		d.Cache[dataUsageHash(k)] = v
+	}
+	return nil
+}
+
+// Trim this from start+end of hashes.
+var hashPathCutSet = dataUsageRoot
+
+func init() {
+	if dataUsageRoot != string(filepath.Separator) {
+		hashPathCutSet = dataUsageRoot + string(filepath.Separator)
+	}
+}
+
+// hashPath calculates a hash of the provided string.
+func hashPath(data string) dataUsageHash {
+	if data != dataUsageRoot {
+		data = strings.Trim(data, hashPathCutSet)
+	}
+	data = path.Clean(data)
+	return dataUsageHash(xxhash.Sum64String(data))
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageHashMap map[dataUsageHash]struct{}
+
+// MarshalMsg implements msgp.Marshaler
+func (d dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, d.Msgsize())
+
+	// Write bin header manually
+	const mbin32 uint8 = 0xc6
+	sz := uint32(len(d)) * dataUsageHashLen
+	o = append(o, mbin32, byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz))
+
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		o = append(o, tmp[:]...)
+	}
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (d dataUsageHashMap) Msgsize() (s int) {
+	s = 5 + len(d)*dataUsageHashLen
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (d *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var hashes []byte
+	hashes, bts, err = msgp.ReadBytesZC(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "dataUsageHashMap")
+		return
+	}
+
+	var dst = make(dataUsageHashMap, len(hashes)/dataUsageHashLen)
+	for len(hashes) >= dataUsageHashLen {
+		dst[dataUsageHash(binary.LittleEndian.Uint64(hashes[:dataUsageHashLen]))] = struct{}{}
+		hashes = hashes[dataUsageHashLen:]
+	}
+	*d = dst
+	o = bts
+	return
+}
+
+func (d *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadBytesHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var dst = make(dataUsageHashMap, zb0001)
+	var tmp [8]byte
+	for i := uint32(0); i < zb0001; i++ {
+		_, err = io.ReadFull(dc, tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err, "dataUsageHashMap")
+			return
+		}
+		dst[dataUsageHash(binary.LittleEndian.Uint64(tmp[:]))] = struct{}{}
+	}
+	return nil
+}
+func (d dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteBytesHeader(uint32(len(d)) * dataUsageHashLen)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		_, err = en.Write(tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+	}
+	return nil
+}

cmd/data-usage-cache_gen.go

@@ -0,0 +1,447 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, err = dc.ReadUint8()
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "Name"
+	err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "LastUpdate"
+	err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.LastUpdate)
+	if err != nil {
+		err = msgp.WrapError(err, "LastUpdate")
+		return
+	}
+	// write "NextCycle"
+	err = en.Append(0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(z.NextCycle)
+	if err != nil {
+		err = msgp.WrapError(err, "NextCycle")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "Name"
+	o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "LastUpdate"
+	o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	o = msgp.AppendTime(o, z.LastUpdate)
+	// string "NextCycle"
+	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	o = msgp.AppendUint8(o, z.NextCycle)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, bts, err = msgp.ReadUint8Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageCacheInfo) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint8Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, err = dc.ReadInt64()
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.DecodeMsg(dc)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// array header, size 4
+	err = en.Append(0x94)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	err = en.WriteUint64(z.Objects)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		err = en.WriteUint64(z.ObjSizes[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// array header, size 4
+	o = append(o, 0x94)
+	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.Objects)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z.ObjSizes {
+		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
+	}
+	o, err = z.Children.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	bts, err = z.Children.UnmarshalMsg(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntry) Msgsize() (s int) {
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 uint64
+		zb0001, err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteUint64(uint64(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint64(o, uint64(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 uint64
+		zb0001, bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageHash) Msgsize() (s int) {
+	s = msgp.Uint64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *sizeHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *sizeHistogram) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *sizeHistogram) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *sizeHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *sizeHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size))
+	return
+}

cmd/data-usage-cache_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageCacheInfo Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageCacheInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshaldataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageEntry Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageEntry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodesizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodesizeHistogram Msgsize() is inaccurate")
+	}
+
+	vn := sizeHistogram{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage.go

@@ -20,22 +20,42 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"os"
+	"path"
+	"strconv"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
 )
 
 const (
-	dataUsageObjName       = "data-usage"
-	dataUsageCrawlInterval = 12 * time.Hour
+	dataUsageObjName         = ".usage.json"
+	dataUsageCacheName       = ".usage-cache.bin"
+	envDataUsageCrawlConf    = "MINIO_DISK_USAGE_CRAWL_ENABLE"
+	envDataUsageCrawlDelay   = "MINIO_DISK_USAGE_CRAWL_DELAY"
+	envDataUsageCrawlDebug   = "MINIO_DISK_USAGE_CRAWL_DEBUG"
+	dataUsageSleepPerFolder  = 1 * time.Millisecond
+	dataUsageSleepDefMult    = 10.0
+	dataUsageUpdateDirCycles = 16
+	dataUsageRoot            = SlashSeparator
+	dataUsageBucket          = minioMetaBucket + SlashSeparator + bucketMetaPrefix
+	dataUsageStartDelay      = 5 * time.Minute // Time to wait on startup and between cycles.
 )
 
+// initDataUsageStats will start the crawler unless disabled.
 func initDataUsageStats() {
-	go runDataUsageInfoUpdateRoutine()
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+		go runDataUsageInfoUpdateRoutine()
+	}
 }
 
+// runDataUsageInfoUpdateRoutine will contain the main crawler.
 func runDataUsageInfoUpdateRoutine() {
 	// Wait until the object layer is ready
 	var objAPI ObjectLayer
@@ -48,89 +68,69 @@ func runDataUsageInfoUpdateRoutine() {
 		break
 	}
 
-	ctx := context.Background()
-
-	switch v := objAPI.(type) {
-	case *xlZones:
-		runDataUsageInfoForXLZones(ctx, v, GlobalServiceDoneCh)
-	case *FSObjects:
-		runDataUsageInfoForFS(ctx, v, GlobalServiceDoneCh)
-	default:
-		return
-	}
+	runDataUsageInfo(GlobalContext, objAPI)
 }
 
-func runDataUsageInfoForFS(ctx context.Context, fsObj *FSObjects, endCh <-chan struct{}) {
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
-	for {
-		// Get data usage info of the FS Object
-		usageInfo := fsObj.crawlAndGetDataUsageInfo(ctx, endCh)
-		// Save the data usage in the disk
-		err := storeDataUsageInBackend(ctx, fsObj, usageInfo)
-		if err != nil {
-			logger.LogIf(ctx, err)
-		}
-		select {
-		case <-endCh:
-			return
-		// Wait until the next crawl interval
-		case <-t.C:
-		}
-	}
-}
+var dataUsageLockTimeout = lifecycleLockTimeout
 
-func runDataUsageInfoForXLZones(ctx context.Context, z *xlZones, endCh <-chan struct{}) {
-	locker := z.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
+func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only 1 crawler is running on the cluster.
+	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
 	for {
-		err := locker.GetLock(newDynamicTimeout(time.Millisecond, time.Millisecond))
+		err := locker.GetLock(dataUsageLockTimeout)
 		if err != nil {
 			time.Sleep(5 * time.Minute)
 			continue
 		}
-		// Break without locking
+		// Break without unlocking, this node will acquire
+		// data usage calculator role for its lifetime.
 		break
 	}
-
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
 	for {
-		usageInfo := z.crawlAndGetDataUsage(ctx, endCh)
-		err := storeDataUsageInBackend(ctx, z, usageInfo)
-		if err != nil {
-			logger.LogIf(ctx, err)
-		}
 		select {
-		case <-endCh:
+		case <-ctx.Done():
 			locker.Unlock()
 			return
-		case <-t.C:
+			// Wait before starting next cycle and wait on startup.
+		case <-time.NewTimer(dataUsageStartDelay).C:
+			results := make(chan DataUsageInfo, 1)
+			go storeDataUsageInBackend(ctx, objAPI, results)
+			err := objAPI.CrawlAndGetDataUsage(ctx, results)
+			close(results)
+			logger.LogIf(ctx, err)
 		}
 	}
 }
 
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
-	dataUsageJSON, err := json.Marshal(dataUsageInfo)
-	if err != nil {
-		return err
-	}
+// storeDataUsageInBackend will store all objects sent on the gui channel until closed.
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
+	for dataUsageInfo := range gui {
+		dataUsageJSON, err := json.Marshal(dataUsageInfo)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		size := int64(len(dataUsageJSON))
+		r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
 
-	size := int64(len(dataUsageJSON))
-	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
-	if err != nil {
-		return err
+		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+		logger.LogIf(ctx, err)
 	}
-
-	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
-	return err
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	var dataUsageInfoJSON bytes.Buffer
 
-	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
+	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
 	if err != nil {
-		return DataUsageInfo{}, nil
+		if isErrObjectNotFound(err) {
+			return DataUsageInfo{}, nil
+		}
+		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
 	}
 
 	var dataUsageInfo DataUsageInfo
@@ -142,3 +142,324 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	return dataUsageInfo, nil
 }
+
+// Item represents each file while walking.
+type Item struct {
+	Path string
+	Typ  os.FileMode
+}
+
+type getSizeFn func(item Item) (int64, error)
+
+type cachedFolder struct {
+	name   string
+	parent *dataUsageHash
+}
+
+type folderScanner struct {
+	root               string
+	getSize            getSizeFn
+	oldCache           dataUsageCache
+	newCache           dataUsageCache
+	waitForLowActiveIO func()
+
+	dataUsageCrawlMult  float64
+	dataUsageCrawlDebug bool
+
+	newFolders      []cachedFolder
+	existingFolders []cachedFolder
+}
+
+// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
+// sleep is limited to max 1 second.
+func sleepDuration(d time.Duration, x float64) {
+	// Don't sleep for really small amount of time
+	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
+		if d > time.Second {
+			d = time.Second
+		}
+		time.Sleep(d)
+	}
+}
+
+// scanQueuedLevels will scan the provided folders.
+// Files found in the folders will be added to f.newCache.
+// If final is provided folders will be put into f.newFolders or f.existingFolders.
+// If final is not provided the folders found are returned from the function.
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+	var nextFolders []cachedFolder
+	done := ctx.Done()
+	for _, folder := range folders {
+		select {
+		case <-done:
+			return nil, ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+
+		cache := dataUsageEntry{}
+		thisHash := hashPath(folder.name)
+
+		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+			// Parse
+			entName = path.Clean(path.Join(folder.name, entName))
+			bucket, _ := path2BucketObjectWithBasePath(f.root, entName)
+			if bucket == "" {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
+				}
+				return nil
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
+				}
+				return nil
+			}
+
+			select {
+			case <-done:
+				return ctx.Err()
+			default:
+			}
+
+			if typ&os.ModeDir != 0 {
+				h := hashPath(entName)
+				_, exists := f.oldCache.Cache[h]
+				cache.addChildString(entName)
+
+				this := cachedFolder{name: entName, parent: &thisHash}
+				cache.addChild(h)
+				if final {
+					if exists {
+						f.existingFolders = append(f.existingFolders, this)
+					} else {
+						f.newFolders = append(f.newFolders, this)
+					}
+				} else {
+					nextFolders = append(nextFolders, this)
+				}
+				return nil
+			}
+			f.waitForLowActiveIO()
+			// Dynamic time delay.
+			t := UTCNow()
+
+			// Get file size, ignore errors.
+			size, err := f.getSize(Item{Path: path.Join(f.root, entName), Typ: typ})
+
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			if err == errSkipFile {
+				return nil
+			}
+			logger.LogIf(ctx, err)
+			cache.Size += size
+			cache.Objects++
+			cache.ObjSizes.add(size)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		f.newCache.replaceHashed(thisHash, folder.parent, cache)
+	}
+	return nextFolders, nil
+}
+
+// deepScanFolder will deep scan a folder and return the size if no error occurs.
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
+	var cache dataUsageEntry
+
+	done := ctx.Done()
+
+	var addDir func(entName string, typ os.FileMode) error
+	var dirStack = []string{f.root, folder}
+
+	addDir = func(entName string, typ os.FileMode) error {
+		select {
+		case <-done:
+			return ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		if typ&os.ModeDir != 0 {
+			dirStack = append(dirStack, entName)
+			err := readDirFn(path.Join(dirStack...), addDir)
+			dirStack = dirStack[:len(dirStack)-1]
+			sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+			return err
+		}
+
+		// Dynamic time delay.
+		t := UTCNow()
+
+		// Get file size, ignore errors.
+		dirStack = append(dirStack, entName)
+		fileName := path.Join(dirStack...)
+		dirStack = dirStack[:len(dirStack)-1]
+
+		size, err := f.getSize(Item{Path: fileName, Typ: typ})
+
+		// Don't sleep for really small amount of time
+		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
+		if err == errSkipFile {
+			return nil
+		}
+		logger.LogIf(ctx, err)
+		cache.Size += size
+		cache.Objects++
+		cache.ObjSizes.add(size)
+		return nil
+	}
+	err := readDirFn(path.Join(dirStack...), addDir)
+	if err != nil {
+		return nil, err
+	}
+	return &cache, nil
+}
+
+// updateUsage will crawl the basepath+cache.Info.Name and return an updated cache.
+// The returned cache will always be valid, but may not be updated from the existing.
+// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// If the supplied context is canceled the function will return at the first chance.
+func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+	t := UTCNow()
+
+	dataUsageDebug := env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn
+	defer func() {
+		if dataUsageDebug {
+			logger.Info(color.Green("updateUsage")+" Crawl time at %s: %v", basePath, time.Since(t))
+		}
+	}()
+
+	if cache.Info.Name == "" {
+		cache.Info.Name = dataUsageRoot
+	}
+
+	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		delayMult = dataUsageSleepDefMult
+	}
+
+	s := folderScanner{
+		root:                basePath,
+		getSize:             getSize,
+		oldCache:            cache,
+		newCache:            dataUsageCache{Info: cache.Info},
+		waitForLowActiveIO:  waitForLowActiveIO,
+		newFolders:          nil,
+		existingFolders:     nil,
+		dataUsageCrawlMult:  delayMult,
+		dataUsageCrawlDebug: dataUsageDebug,
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(color.Green("runDataUsageInfo:") + " Starting crawler master")
+	}
+
+	done := ctx.Done()
+	var flattenLevels = 3
+
+	// If we are scanning inside a bucket reduce depth by 1.
+	if cache.Info.Name != dataUsageRoot {
+		flattenLevels--
+	}
+
+	var logPrefix, logSuffix string
+	if s.dataUsageCrawlDebug {
+		logPrefix = color.Green("data-usage: ")
+		logSuffix = color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Cycle: %v"+logSuffix, cache.Info.NextCycle)
+	}
+
+	// Always scan flattenLevels deep. Cache root is level 0.
+	todo := []cachedFolder{{name: cache.Info.Name}}
+	for i := 0; i < flattenLevels; i++ {
+		if s.dataUsageCrawlDebug {
+			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
+		}
+		select {
+		case <-done:
+			return cache, ctx.Err()
+		default:
+		}
+		var err error
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
+		if err != nil {
+			// No useful information...
+			return cache, err
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+	}
+
+	// Add new folders first
+	for _, folder := range s.newFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replace(folder.name, "", *du)
+		// Add to parent manually
+		if folder.parent != nil {
+			parent := s.newCache.Cache[*folder.parent]
+			parent.addChildString(folder.name)
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+	}
+
+	// Do selective scanning of existing folders.
+	for _, folder := range s.existingFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		h := hashPath(folder.name)
+		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
+			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
+			continue
+		}
+
+		// Update on this cycle...
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replaceHashed(h, folder.parent, *du)
+	}
+
+	s.newCache.Info.LastUpdate = UTCNow()
+	s.newCache.Info.NextCycle++
+	return s.newCache, nil
+}

cmd/data-usage_test.go

@@ -0,0 +1,651 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type usageTestFile struct {
+	name string
+	size int
+}
+
+func TestDataUsageUpdate(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageUpdate")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+
+	got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1322310,
+			flatten: true,
+			objs:    8,
+			oSizes:  sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:    "/dir1/dira",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:    "/dir1/dira/",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "/dir1/dira",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:   "/dir1/dira/",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1363315,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    1300010,
+			flatten: true,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    363515,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    300210,
+			objs:    4,
+			flatten: true,
+			oSizes:  sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func TestDataUsageUpdatePrefix(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUpdateUsagePrefix")
+	if err != nil {
+		t.Skip(err)
+	}
+	base = filepath.Join(base, "bucket")
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "bucket/rootfile", size: 10000},
+		{name: "bucket/rootfile2", size: 10000},
+		{name: "bucket/dir1/d1file", size: 2000},
+		{name: "bucket/dir2/d2file", size: 300},
+		{name: "bucket/dir1/dira/dafile", size: 100000},
+		{name: "bucket/dir1/dira/dbfile", size: 200000},
+		{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1322310,
+			objs:   8,
+			oSizes: sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "bucket/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "bucket/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "bucket/dir1/dira/",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "bucket/newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "bucket/newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "bucket/dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "bucket/dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "bucket/rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1363315,
+			objs:   14,
+			oSizes: sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "bucket/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "bucket/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   363515,
+			objs:   14,
+			oSizes: sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   300210,
+			objs:   4,
+			oSizes: sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) {
+	for _, f := range files {
+		err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestDataUsageCacheSerialize(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageCacheSerialize")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := want.serialize()
+	var got dataUsageCache
+	err = got.deserialize(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got.Info.LastUpdate.IsZero() {
+		t.Error("lastupdate not set")
+	}
+
+	if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) {
+		t.Fatalf("deserialize mismatch\nwant: %+v\ngot:  %+v", want, got)
+	}
+}

cmd/disk-cache-backend.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"reflect"
+	"strings"
 	"sync"
 	"time"
 
@@ -37,7 +37,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/sio"
-	"github.com/ncw/directio"
+	"go.uber.org/atomic"
 )
 
 const (
@@ -45,7 +45,7 @@ const (
 	cacheMetaJSONFile = "cache.json"
 	cacheDataFile     = "part.1"
 	cacheMetaVersion  = "1.0.0"
-
+	cacheExpiryDays   = time.Duration(90 * time.Hour * 24) // defaults to 90 days
 	// SSECacheEncrypted is the metadata key indicating that the object
 	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
 	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
@@ -68,6 +68,8 @@ type cacheMeta struct {
 	Meta map[string]string `json:"meta,omitempty"`
 	// Ranges maps cached range to associated filename.
 	Ranges map[string]string `json:"ranges,omitempty"`
+	// Hits is a counter on the number of times this object has been accessed so far.
+	Hits int `json:"hits,omitempty"`
 }
 
 // RangeInfo has the range, file and range length information for a cached range.
@@ -125,44 +127,54 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 type diskCache struct {
 	dir      string // caching directory
 	quotaPct int    // max usage in %
-	expiry   int    // cache expiry in days
 	// mark false if drive is offline
 	online bool
 	// mutex to protect updates to online variable
-	onlineMutex *sync.RWMutex
-	// purge() listens on this channel to start the cache-purge process
-	purgeChan chan struct{}
-	pool      sync.Pool
+	onlineMutex   *sync.RWMutex
+	pool          sync.Pool
+	after         int // minimum accesses before an object is cached.
+	lowWatermark  int
+	highWatermark int
+	gcCounter     atomic.Uint64
+	// nsMutex namespace lock
+	nsMutex *nsLockMap
+	// Object functions pointing to the corresponding functions of backend implementation.
+	NewNSLockFn func(ctx context.Context, cachePath string) RWLocker
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(dir string, expiry int, quotaPct int) (*diskCache, error) {
+func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
-		dir:         dir,
-		expiry:      expiry,
-		quotaPct:    quotaPct,
-		purgeChan:   make(chan struct{}),
-		online:      true,
-		onlineMutex: &sync.RWMutex{},
+		dir:           dir,
+		quotaPct:      quotaPct,
+		after:         after,
+		lowWatermark:  lowWatermark,
+		highWatermark: highWatermark,
+		online:        true,
+		onlineMutex:   &sync.RWMutex{},
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := directio.AlignedBlock(int(cacheBlkSize))
+				b := disk.AlignedBlock(int(cacheBlkSize))
 				return &b
 			},
 		},
+		nsMutex: newNSLock(false),
+	}
+	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
+		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
 	}
 	return &cache, nil
 }
 
-// Returns if the disk usage is low.
-// Disk usage is low if usage is < 80% of cacheMaxDiskUsagePct
-// Ex. for a 100GB disk, if maxUsage is configured as 70% then cacheMaxDiskUsagePct is 70G
-// hence disk usage is low if the disk usage is less than 56G (because 80% of 70G is 56G)
+// diskUsageLow() returns true if disk usage falls below the low watermark w.r.t configured cache quota.
+// Ex. for a 100GB disk, if quota is configured as 70%  and watermark_low = 80% and
+// watermark_high = 90% then garbage collection starts when 63% of disk is used and
+// stops when disk usage drops to 56%
 func (c *diskCache) diskUsageLow() bool {
-	minUsage := c.quotaPct * 80 / 100
+	gcStopPct := c.quotaPct * c.lowWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
@@ -171,21 +183,22 @@ func (c *diskCache) diskUsageLow() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < minUsage
+	return int(usedPercent) < gcStopPct
 }
 
-// Return if the disk usage is high.
-// Disk usage is high if disk used is > cacheMaxDiskUsagePct
+// Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
+// gc starts if high water mark reached.
 func (c *diskCache) diskUsageHigh() bool {
+	gcTriggerPct := c.quotaPct * c.highWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
 		logger.LogIf(ctx, err)
-		return true
+		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) > c.quotaPct
+	return int(usedPercent) >= gcTriggerPct
 }
 
 // Returns if size space can be allocated without exceeding
@@ -202,12 +215,42 @@ func (c *diskCache) diskAvailable(size int64) bool {
 	return int(usedPercent) < c.quotaPct
 }
 
+// toClear returns how many bytes should be cleared to reach the low watermark quota.
+// returns 0 if below quota.
+func (c *diskCache) toClear() uint64 {
+	di, err := disk.GetInfo(c.dir)
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		return 0
+	}
+	return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark))
+}
+
 // Purge cache entries that were not accessed.
-func (c *diskCache) purge() {
+func (c *diskCache) purge(ctx context.Context, doneCh <-chan struct{}) {
+	if c.diskUsageLow() {
+		return
+	}
+	toFree := c.toClear()
+	if toFree == 0 {
+		return
+	}
+	// expiry for cleaning up old cache.json files that
+	// need to be cleaned up.
+	expiry := UTCNow().Add(-cacheExpiryDays)
+	// defaulting max hits count to 100
+	scorer, err := newFileScorer(int64(toFree), time.Now().Unix(), 100)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+
 	// this function returns FileInfo for cached range files and cache data file.
 	fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
 		fm := make(map[string]os.FileInfo)
-		fname := pathJoin(pathPrefix, cacheDataFile)
+		fname := pathJoin(pathPrefix, dataFile)
 		if fi, err := os.Stat(fname); err == nil {
 			fm[fname] = fi
 		}
@@ -220,65 +263,73 @@ func (c *diskCache) purge() {
 		}
 		return fm
 	}
-	ctx := context.Background()
-	for {
-		olderThan := c.expiry * 24
-		for !c.diskUsageLow() {
-			// delete unaccessed objects older than expiry duration
-			expiry := UTCNow().Add(time.Hour * time.Duration(-1*olderThan))
-			olderThan /= 2
-			if olderThan < 1 {
-				break
-			}
-			deletedCount := 0
+	objDirs, err := ioutil.ReadDir(c.dir)
+	if err != nil {
+		log.Fatal(err)
+	}
 
-			objDirs, err := ioutil.ReadDir(c.dir)
-			if err != nil {
-				log.Fatal(err)
-			}
+	for _, obj := range objDirs {
+		if obj.Name() == minioMetaBucket {
+			continue
+		}
 
-			for _, obj := range objDirs {
-				if obj.Name() == minioMetaBucket {
-					continue
-				}
-				meta, _, err := c.statCachedMeta(pathJoin(c.dir, obj.Name()))
-				if err != nil {
-					// delete any partially filled cache entry left behind.
-					removeAll(pathJoin(c.dir, obj.Name()))
-					continue
-				}
-				// stat all cached file ranges and cacheDataFile.
-				fis := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
-				objInfo := meta.ToObjectInfo("", "")
-				cc := cacheControlOpts(objInfo)
-
-				for fname, fi := range fis {
-					if atime.Get(fi).Before(expiry) ||
-						cc.isStale(objInfo.ModTime) {
-						if err = removeAll(fname); err != nil {
-							logger.LogIf(ctx, err)
-						}
-						deletedCount++
-						// break early if sufficient disk space reclaimed.
-						if !c.diskUsageLow() {
-							break
-						}
+		cacheDir := pathJoin(c.dir, obj.Name())
+		meta, _, numHits, err := c.statCachedMeta(ctx, cacheDir)
+		if err != nil {
+			// delete any partially filled cache entry left behind.
+			removeAll(cacheDir)
+			continue
+		}
+		// stat all cached file ranges and cacheDataFile.
+		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
+		objInfo := meta.ToObjectInfo("", "")
+		cc := cacheControlOpts(objInfo)
+		for fname, fi := range cachedFiles {
+			if cc != nil {
+				if cc.isStale(objInfo.ModTime) {
+					if err = removeAll(fname); err != nil {
+						logger.LogIf(ctx, err)
+					}
+					scorer.adjustSaveBytes(-fi.Size())
+					// break early if sufficient disk space reclaimed.
+					if c.diskUsageLow() {
+						return
 					}
 				}
+				continue
 			}
-			if deletedCount == 0 {
-				break
-			}
+			scorer.addFile(fname, atime.Get(fi), fi.Size(), numHits)
 		}
-		lastRunTime := time.Now()
-		for {
-			<-c.purgeChan
-			timeElapsed := time.Since(lastRunTime)
-			if timeElapsed > time.Hour {
-				break
-			}
+		// clean up stale cache.json files for objects that never got cached but access count was maintained in cache.json
+		fi, err := os.Stat(pathJoin(cacheDir, cacheMetaJSONFile))
+		if err != nil || (fi.ModTime().Before(expiry) && len(cachedFiles) == 0) {
+			removeAll(cacheDir)
+			scorer.adjustSaveBytes(-fi.Size())
+			continue
+		}
+		if c.diskUsageLow() {
+			return
 		}
 	}
+	for _, path := range scorer.fileNames() {
+		removeAll(path)
+		slashIdx := strings.LastIndex(path, SlashSeparator)
+		pathPrefix := path[0:slashIdx]
+		fname := path[slashIdx+1:]
+		if fname == cacheDataFile {
+			removeAll(pathPrefix)
+		}
+	}
+}
+
+func (c *diskCache) incGCCounter() {
+	c.gcCounter.Add(uint64(1))
+}
+func (c *diskCache) resetGCCounter() {
+	c.gcCounter.Store(uint64(0))
+}
+func (c *diskCache) gcCount() uint64 {
+	return c.gcCounter.Load()
 }
 
 // sets cache drive status
@@ -296,63 +347,67 @@ func (c *diskCache) IsOnline() bool {
 }
 
 // Stat returns ObjectInfo from disk cache
-func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectInfo, err error) {
+func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectInfo, numHits int, err error) {
+	var partial bool
+	var meta *cacheMeta
+
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	oi, err = c.statCache(cacheObjPath)
+	// Stat the file to get file size.
+	meta, partial, numHits, err = c.statCachedMeta(ctx, cacheObjPath)
 	if err != nil {
 		return
 	}
+	if partial {
+		return oi, numHits, errFileNotFound
+	}
+	oi = meta.ToObjectInfo("", "")
 	oi.Bucket = bucket
 	oi.Name = object
 
 	if err = decryptCacheObjectETag(&oi); err != nil {
-		return oi, err
+		return
 	}
 	return
 }
 
 // statCachedMeta returns metadata from cache - including ranges cached, partial to indicate
 // if partial object is cached.
-func (c *diskCache) statCachedMeta(cacheObjPath string) (meta *cacheMeta, partial bool, err error) {
-	// Stat the file to get file size.
-	metaPath := pathJoin(cacheObjPath, cacheMetaJSONFile)
-	f, err := os.Open(metaPath)
-	if err != nil {
-		return meta, partial, err
+func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
+
+	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	if err = cLock.GetRLock(globalObjectTimeout); err != nil {
+		return
 	}
-	defer f.Close()
-	meta = &cacheMeta{Version: cacheMetaVersion}
-	if err := jsonLoad(f, meta); err != nil {
-		return meta, partial, err
-	}
-	// get metadata of part.1 if full file has been cached.
-	partial = true
-	fi, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile))
-	if err == nil {
-		meta.Stat.ModTime = atime.Get(fi)
-		partial = false
-	}
-	return meta, partial, nil
+
+	defer cLock.RUnlock()
+	return c.statCache(ctx, cacheObjPath)
 }
 
 // statRange returns ObjectInfo and RangeInfo from disk cache
-func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HTTPRangeSpec) (oi ObjectInfo, rngInfo RangeInfo, err error) {
+func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HTTPRangeSpec) (oi ObjectInfo, rngInfo RangeInfo, numHits int, err error) {
 	// Stat the file to get file size.
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
+	var meta *cacheMeta
+	var partial bool
 
-	if rs == nil {
-		oi, err = c.statCache(cacheObjPath)
-		return oi, rngInfo, err
-	}
-	meta, _, err := c.statCachedMeta(cacheObjPath)
+	meta, partial, numHits, err = c.statCachedMeta(ctx, cacheObjPath)
 	if err != nil {
-		return oi, rngInfo, err
+		return
+	}
+
+	oi = meta.ToObjectInfo("", "")
+	oi.Bucket = bucket
+	oi.Name = object
+	if !partial {
+		err = decryptCacheObjectETag(&oi)
+		return
 	}
 
 	actualSize := uint64(meta.Stat.Size)
-	_, length, err := rs.GetOffsetLength(int64(actualSize))
+	var length int64
+	_, length, err = rs.GetOffsetLength(int64(actualSize))
 	if err != nil {
-		return oi, rngInfo, err
+		return
 	}
 
 	actualRngSize := uint64(length)
@@ -363,38 +418,61 @@ func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HT
 	rng := rs.String(int64(actualSize))
 	rngFile, ok := meta.Ranges[rng]
 	if !ok {
-		return oi, rngInfo, ObjectNotFound{Bucket: bucket, Object: object}
+		return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
+	}
+	if _, err = os.Stat(pathJoin(cacheObjPath, rngFile)); err != nil {
+		return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
 	}
 	rngInfo = RangeInfo{Range: rng, File: rngFile, Size: int64(actualRngSize)}
 
-	oi = meta.ToObjectInfo("", "")
-	oi.Bucket = bucket
-	oi.Name = object
-
-	if err = decryptCacheObjectETag(&oi); err != nil {
-		return oi, rngInfo, err
-	}
+	err = decryptCacheObjectETag(&oi)
 	return
 }
 
 // statCache is a convenience function for purge() to get ObjectInfo for cached object
-func (c *diskCache) statCache(cacheObjPath string) (oi ObjectInfo, e error) {
+func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
 	// Stat the file to get file size.
-	meta, partial, err := c.statCachedMeta(cacheObjPath)
+	metaPath := pathJoin(cacheObjPath, cacheMetaJSONFile)
+	f, err := os.Open(metaPath)
 	if err != nil {
-		return oi, err
+		return meta, partial, 0, err
 	}
-	if partial {
-		return oi, errFileNotFound
+	defer f.Close()
+	meta = &cacheMeta{Version: cacheMetaVersion}
+	if err := jsonLoad(f, meta); err != nil {
+		return meta, partial, 0, err
 	}
-	return meta.ToObjectInfo("", ""), nil
+	// get metadata of part.1 if full file has been cached.
+	partial = true
+	fi, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile))
+	if err == nil {
+		meta.Stat.ModTime = atime.Get(fi)
+		partial = false
+	}
+	return meta, partial, meta.Hits, nil
 }
 
 // saves object metadata to disk cache
-func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string) error {
-	fileName := getCacheSHADir(c.dir, bucket, object)
-	metaPath := pathJoin(fileName, cacheMetaJSONFile)
+// incHitsOnly is true if metadata update is incrementing only the hit counter
+func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
+	cachedPath := getCacheSHADir(c.dir, bucket, object)
+	cLock := c.NewNSLockFn(ctx, cachedPath)
+	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer cLock.Unlock()
+	return c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly)
+}
 
+// saves object metadata to disk cache
+// incHitsOnly is true if metadata update is incrementing only the hit counter
+func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
+	cachedPath := getCacheSHADir(c.dir, bucket, object)
+	metaPath := pathJoin(cachedPath, cacheMetaJSONFile)
+	// Create cache directory if needed
+	if err := os.MkdirAll(cachedPath, 0777); err != nil {
+		return err
+	}
 	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0666)
 	if err != nil {
 		return err
@@ -405,6 +483,7 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	if err := jsonLoad(f, m); err != nil && err != io.EOF {
 		return err
 	}
+	// increment hits
 	if rs != nil {
 		if m.Ranges == nil {
 			m.Ranges = make(map[string]string)
@@ -413,46 +492,30 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	} else {
 		// this is necessary cleanup of range files if entire object is cached.
 		for _, f := range m.Ranges {
-			removeAll(pathJoin(fileName, f))
+			removeAll(pathJoin(cachedPath, f))
 		}
 		m.Ranges = nil
 	}
 	m.Stat.Size = actualSize
 	m.Stat.ModTime = UTCNow()
-	m.Meta = meta
+	if !incHitsOnly {
+		// reset meta
+		m.Meta = meta
+	} else {
+		if m.Meta == nil {
+			m.Meta = make(map[string]string)
+		}
+		if etag, ok := meta["etag"]; ok {
+			m.Meta["etag"] = etag
+		}
+	}
+
+	m.Hits++
+
 	m.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
-
 	return jsonSave(f, m)
 }
 
-// Backend metadata could have changed through server side copy - reset cache metadata if that is the case
-func (c *diskCache) updateMetadataIfChanged(ctx context.Context, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
-
-	bkMeta := make(map[string]string)
-	cacheMeta := make(map[string]string)
-	for k, v := range bkObjectInfo.UserDefined {
-		if HasPrefix(k, ReservedMetadataPrefix) {
-			// Do not need to send any internal metadata
-			continue
-		}
-		bkMeta[http.CanonicalHeaderKey(k)] = v
-	}
-	for k, v := range cacheObjInfo.UserDefined {
-		if HasPrefix(k, ReservedMetadataPrefix) {
-			// Do not need to send any internal metadata
-			continue
-		}
-		cacheMeta[http.CanonicalHeaderKey(k)] = v
-	}
-	if !reflect.DeepEqual(bkMeta, cacheMeta) ||
-		bkObjectInfo.ETag != cacheObjInfo.ETag ||
-		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
-		bkObjectInfo.Expires != cacheObjInfo.Expires {
-		return c.saveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "")
-	}
-	return nil
-}
-
 func getCacheSHADir(dir, bucket, object string) string {
 	return pathJoin(dir, getSHA256Hash([]byte(pathJoin(bucket, object))))
 }
@@ -548,21 +611,40 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 }
 
 // Caches the object to disk
-func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions) error {
+func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
 	if c.diskUsageHigh() {
-		select {
-		case c.purgeChan <- struct{}{}:
-		default:
-		}
+		c.incGCCounter()
+		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}
+	cachePath := getCacheSHADir(c.dir, bucket, object)
+	cLock := c.NewNSLockFn(ctx, cachePath)
+	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer cLock.Unlock()
+
+	meta, _, numHits, err := c.statCache(ctx, cachePath)
+	// Case where object not yet cached
+	if os.IsNotExist(err) && c.after >= 1 {
+		return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+	}
+	// Case where object already has a cache metadata entry but not yet cached
+	if err == nil && numHits < c.after {
+		cETag := extractETag(meta.Meta)
+		bETag := extractETag(opts.UserDefined)
+		if cETag == bETag {
+			return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+		}
+		incHitsOnly = true
+	}
+
 	if rs != nil {
 		return c.putRange(ctx, bucket, object, data, size, rs, opts)
 	}
 	if !c.diskAvailable(size) {
 		return errDiskFull
 	}
-	cachePath := getCacheSHADir(c.dir, bucket, object)
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return err
 	}
@@ -572,7 +654,6 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	}
 	var reader = data
 	var actualSize = uint64(size)
-	var err error
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
@@ -588,11 +669,12 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 		removeAll(cachePath)
 		return err
 	}
+
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
 		return IncompleteBody{}
 	}
-	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "")
+	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
 }
 
 // Caches the range to disk
@@ -638,7 +720,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 		removeAll(cachePath)
 		return IncompleteBody{}
 	}
-	return c.saveMetadata(ctx, bucket, object, metadata, int64(objSize), rs, cacheFile)
+	return c.saveMetadata(ctx, bucket, object, metadata, int64(objSize), rs, cacheFile, false)
 }
 
 // checks streaming bitrot checksum of cached object before returning data
@@ -738,13 +820,18 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 }
 
 // Get returns ObjectInfo and reader for object from disk cache
-func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	var objInfo ObjectInfo
+func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	var rngInfo RangeInfo
+	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
+		return nil, numHits, err
+	}
 
-	if objInfo, rngInfo, err = c.statRange(ctx, bucket, object, rs); err != nil {
-		return nil, toObjectErr(err, bucket, object)
+	defer cLock.RUnlock()
+	var objInfo ObjectInfo
+	var rngInfo RangeInfo
+	if objInfo, rngInfo, numHits, err = c.statRange(ctx, bucket, object, rs); err != nil {
+		return nil, numHits, toObjectErr(err, bucket, object)
 	}
 	cacheFile := cacheDataFile
 	objSize := objInfo.Size
@@ -760,12 +847,13 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	if HasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
 		// objReader.Close() is called by the caller.
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+		gr, gerr := NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+		return gr, numHits, gerr
 	}
 
 	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
 	if nErr != nil {
-		return nil, nErr
+		return nil, numHits, nErr
 	}
 	filePath := pathJoin(cacheObjPath, cacheFile)
 	pr, pw := io.Pipe()
@@ -782,7 +870,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 
 	gr, gerr := fn(pr, h, opts.CheckCopyPrecondFn, pipeCloser)
 	if gerr != nil {
-		return gr, gerr
+		return gr, numHits, gerr
 	}
 	if globalCacheKMS != nil {
 		// clean up internal SSE cache metadata
@@ -792,15 +880,24 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		// overlay Size with actual object size and not the range size
 		gr.ObjInfo.Size = objSize
 	}
-	return gr, nil
+	return gr, numHits, nil
 
 }
 
 // Deletes the cached object
-func (c *diskCache) Delete(ctx context.Context, bucket, object string) (err error) {
-	cachePath := getCacheSHADir(c.dir, bucket, object)
-	return removeAll(cachePath)
+func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
+	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer cLock.Unlock()
+	return removeAll(cacheObjPath)
+}
 
+// Deletes the cached object
+func (c *diskCache) Delete(ctx context.Context, bucket, object string) (err error) {
+	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
+	return c.delete(ctx, cacheObjPath)
 }
 
 // convenience function to check if object is cached on this diskCache

cmd/disk-cache-utils.go

@@ -17,8 +17,12 @@
 package cmd
 
 import (
+	"container/list"
 	"encoding/hex"
+	"errors"
+	"fmt"
 	"io"
+	"math"
 	"os"
 	"path"
 	"strconv"
@@ -57,13 +61,8 @@ type cacheControl struct {
 	noCache      bool
 }
 
-func (c cacheControl) isEmpty() bool {
-	return c == cacheControl{}
-
-}
-
-func (c cacheControl) isStale(modTime time.Time) bool {
-	if c.isEmpty() {
+func (c *cacheControl) isStale(modTime time.Time) bool {
+	if c == nil {
 		return false
 	}
 	// response will never be stale if only-if-cached is set
@@ -100,7 +99,8 @@ func (c cacheControl) isStale(modTime time.Time) bool {
 }
 
 // returns struct with cache-control settings from user metadata.
-func cacheControlOpts(o ObjectInfo) (c cacheControl) {
+func cacheControlOpts(o ObjectInfo) *cacheControl {
+	c := cacheControl{}
 	m := o.UserDefined
 	if o.Expires != timeSentinel {
 		c.expiry = o.Expires
@@ -114,7 +114,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 
 	}
 	if headerVal == "" {
-		return
+		return nil
 	}
 	headerVal = strings.ToLower(headerVal)
 	headerVal = strings.TrimSpace(headerVal)
@@ -146,7 +146,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			p[0] == "max-stale" {
 			i, err := strconv.Atoi(p[1])
 			if err != nil {
-				return cacheControl{}
+				return nil
 			}
 			if p[0] == "max-age" {
 				c.maxAge = i
@@ -162,7 +162,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			}
 		}
 	}
-	return c
+	return &c
 }
 
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
@@ -265,3 +265,179 @@ func decryptCacheObjectETag(info *ObjectInfo) error {
 
 	return nil
 }
+
+func isMetadataSame(m1, m2 map[string]string) bool {
+	if m1 == nil && m2 == nil {
+		return true
+	}
+	if (m1 == nil && m2 != nil) || (m2 == nil && m1 != nil) {
+		return false
+	}
+	if len(m1) != len(m2) {
+		return false
+	}
+	for k1, v1 := range m1 {
+		if v2, ok := m2[k1]; !ok || (v1 != v2) {
+			return false
+		}
+	}
+	return true
+}
+
+type fileScorer struct {
+	saveBytes int64
+	now       int64
+	maxHits   int
+	// 1/size for consistent score.
+	sizeMult float64
+
+	// queue is a linked list of files we want to delete.
+	// The list is kept sorted according to score, highest at top, lowest at bottom.
+	queue       list.List
+	queuedBytes int64
+}
+
+type queuedFile struct {
+	name  string
+	size  int64
+	score float64
+}
+
+// newFileScorer allows to collect files to save a specific number of bytes.
+// Each file is assigned a score based on its age, size and number of hits.
+// A list of files is maintained
+func newFileScorer(saveBytes int64, now int64, maxHits int) (*fileScorer, error) {
+	if saveBytes <= 0 {
+		return nil, errors.New("newFileScorer: saveBytes <= 0")
+	}
+	if now < 0 {
+		return nil, errors.New("newFileScorer: now < 0")
+	}
+	if maxHits <= 0 {
+		return nil, errors.New("newFileScorer: maxHits <= 0")
+	}
+	f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
+	f.queue.Init()
+	return &f, nil
+}
+
+func (f *fileScorer) addFile(name string, lastAccess time.Time, size int64, hits int) {
+	// Calculate how much we want to delete this object.
+	file := queuedFile{
+		name: name,
+		size: size,
+	}
+	score := float64(f.now - lastAccess.Unix())
+	// Size as fraction of how much we want to save, 0->1.
+	szWeight := math.Max(0, (math.Min(1, float64(size)*f.sizeMult)))
+	// 0 at f.maxHits, 1 at 0.
+	hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
+	file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
+	// If we still haven't saved enough, just add the file
+	if f.queuedBytes < f.saveBytes {
+		f.insertFile(file)
+		f.trimQueue()
+		return
+	}
+	// If we score less than the worst, don't insert.
+	worstE := f.queue.Back()
+	if worstE != nil && file.score < worstE.Value.(queuedFile).score {
+		return
+	}
+	f.insertFile(file)
+	f.trimQueue()
+}
+
+// adjustSaveBytes allows to adjust the number of bytes to save.
+// This can be used to adjust the count on the fly.
+// Returns true if there still is a need to delete files (saveBytes >0),
+// false if no more bytes needs to be saved.
+func (f *fileScorer) adjustSaveBytes(n int64) bool {
+	f.saveBytes += n
+	if f.saveBytes <= 0 {
+		f.queue.Init()
+		f.saveBytes = 0
+		return false
+	}
+	if n < 0 {
+		f.trimQueue()
+	}
+	return true
+}
+
+// insertFile will insert a file into the list, sorted by its score.
+func (f *fileScorer) insertFile(file queuedFile) {
+	e := f.queue.Front()
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if v.score < file.score {
+			break
+		}
+		e = e.Next()
+	}
+	f.queuedBytes += file.size
+	// We reached the end.
+	if e == nil {
+		f.queue.PushBack(file)
+		return
+	}
+	f.queue.InsertBefore(file, e)
+}
+
+// trimQueue will trim the back of queue and still keep below wantSave.
+func (f *fileScorer) trimQueue() {
+	for {
+		e := f.queue.Back()
+		if e == nil {
+			return
+		}
+		v := e.Value.(queuedFile)
+		if f.queuedBytes-v.size < f.saveBytes {
+			return
+		}
+		f.queue.Remove(e)
+		f.queuedBytes -= v.size
+	}
+}
+
+// fileNames returns all queued file names.
+func (f *fileScorer) fileNames() []string {
+	res := make([]string, 0, f.queue.Len())
+	e := f.queue.Front()
+	for e != nil {
+		res = append(res, e.Value.(queuedFile).name)
+		e = e.Next()
+	}
+	return res
+}
+
+func (f *fileScorer) reset() {
+	f.queue.Init()
+	f.queuedBytes = 0
+}
+
+func (f *fileScorer) queueString() string {
+	var res strings.Builder
+	e := f.queue.Front()
+	i := 0
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if i > 0 {
+			res.WriteByte('\n')
+		}
+		res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
+		i++
+		e = e.Next()
+	}
+	return res.String()
+}
+
+// bytesToClear() returns the number of bytes to clear to reach low watermark
+// w.r.t quota given disk total and free space, quota in % allocated to cache
+// and low watermark % w.r.t allowed quota.
+func bytesToClear(total, free int64, quotaPct, lowWatermark uint64) uint64 {
+	used := (total - free)
+	quotaAllowed := total * (int64)(quotaPct) / 100
+	lowWMUsage := (total * (int64)(lowWatermark*quotaPct) / (100 * 100))
+	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
+}

cmd/disk-cache-utils_test.go

@@ -29,16 +29,16 @@ func TestGetCacheControlOpts(t *testing.T) {
 	testCases := []struct {
 		cacheControlHeaderVal string
 		expiryHeaderVal       time.Time
-		expectedCacheControl  cacheControl
+		expectedCacheControl  *cacheControl
 		expectedErr           bool
 	}{
-		{"", timeSentinel, cacheControl{}, false},
-		{"max-age=2592000, public", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"max-age=2592000, no-store", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
-		{"must-revalidate, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", expiry, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
-		{"s-maxAge=2500, max-age=600s", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
+		{"", timeSentinel, nil, false},
+		{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
+		{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
+		{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
 	}
 
 	for _, testCase := range testCases {
@@ -49,7 +49,7 @@ func TestGetCacheControlOpts(t *testing.T) {
 				m["expires"] = testCase.expiryHeaderVal.String()
 			}
 			c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
-			if testCase.expectedErr && (c != cacheControl{}) {
+			if testCase.expectedErr && (c != nil) {
 				t.Errorf("expected err, got <nil>")
 			}
 			if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
@@ -58,3 +58,115 @@ func TestGetCacheControlOpts(t *testing.T) {
 		})
 	}
 }
+
+func TestIsMetadataSame(t *testing.T) {
+
+	testCases := []struct {
+		m1       map[string]string
+		m2       map[string]string
+		expected bool
+	}{
+		{nil, nil, true},
+		{nil, map[string]string{}, false},
+		{map[string]string{"k": "v"}, map[string]string{"k": "v"}, true},
+		{map[string]string{"k": "v"}, map[string]string{"a": "b"}, false},
+		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v1"}, false},
+		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, true},
+		{map[string]string{"K1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
+		{map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
+	}
+
+	for i, testCase := range testCases {
+		actual := isMetadataSame(testCase.m1, testCase.m2)
+		if testCase.expected != actual {
+			t.Errorf("test %d expected %v, got %v", i, testCase.expected, actual)
+		}
+	}
+}
+
+func TestNewFileScorer(t *testing.T) {
+	fs, err := newFileScorer(1000, time.Now().Unix(), 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fs.fileNames()) != 0 {
+		t.Fatal("non zero files??")
+	}
+	now := time.Now()
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	fs.addFile("older", now.Add(-time.Hour), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("less", now.Add(-time.Minute), 1000, 5)
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("small", now.Add(-time.Minute), 200, 10)
+	fs.addFile("medium", now.Add(-time.Minute), 300, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.addFile("large", now.Add(-time.Minute), 700, 10)
+	fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+
+	fs.reset()
+	fs.addFile("less", now.Add(-time.Minute), 500, 5)
+	fs.addFile("recent", now.Add(-time.Minute), 500, 10)
+	if !fs.adjustSaveBytes(-500) {
+		t.Fatal("we should still need more bytes, got false")
+	}
+	// We should only need 500 bytes now.
+	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	if fs.adjustSaveBytes(-500) {
+		t.Fatal("we shouldn't need any more bytes, got true")
+	}
+	fs, err = newFileScorer(1000, time.Now().Unix(), 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
+	// sorting should be consistent after adjusting savebytes.
+	fs.adjustSaveBytes(-800)
+	fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+}
+func TestBytesToClear(t *testing.T) {
+	testCases := []struct {
+		total        int64
+		free         int64
+		quotaPct     uint64
+		watermarkLow uint64
+		expected     uint64
+	}{
+		{1000, 800, 40, 90, 0},
+		{1000, 200, 40, 90, 400},
+		{1000, 400, 40, 90, 240},
+		{1000, 600, 40, 90, 40},
+		{1000, 600, 40, 70, 120},
+		{1000, 1000, 90, 70, 0},
+		{1000, 0, 90, 70, 370},
+	}
+	for i, tc := range testCases {
+		toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow)
+		if tc.expected != toClear {
+			t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
+		}
+	}
+}

cmd/disk-cache.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019,2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,13 +31,15 @@ import (
 	"github.com/djherbis/atime"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/logger"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
-	cacheBlkSize = int64(1 * 1024 * 1024)
+	cacheBlkSize    = int64(1 * 1024 * 1024)
+	cacheGCInterval = time.Minute * 30
 )
 
 // CacheStorageInfo - represents total, free capacity of
@@ -55,6 +57,7 @@ type CacheObjectLayer interface {
 	DeleteObject(ctx context.Context, bucket, object string) error
 	DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	// Storage operations.
 	StorageInfo(ctx context.Context) CacheStorageInfo
 	CacheStats() *CacheStats
@@ -66,74 +69,57 @@ type cacheObjects struct {
 	cache []*diskCache
 	// file path patterns to exclude from cache
 	exclude []string
-
+	// number of accesses after which to cache an object
+	after int
 	// if true migration is in progress from v1 to v2
 	migrating bool
 	// mutex to protect migration bool
 	migMutex sync.Mutex
 
-	// nsMutex namespace lock
-	nsMutex *nsLockMap
-
 	// Cache stats
 	cacheStats *CacheStats
 
-	// Object functions pointing to the corresponding functions of backend implementation.
-	NewNSLockFn      func(ctx context.Context, bucket, object string) RWLocker
 	GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn   func(ctx context.Context, bucket, object string) error
 	DeleteObjectsFn  func(ctx context.Context, bucket string, objects []string) ([]error, error)
 	PutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	CopyObjectFn     func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 }
 
-func (c *cacheObjects) delete(ctx context.Context, dcache *diskCache, bucket, object string) (err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
-		return err
-	}
-	defer cLock.Unlock()
-	return dcache.Delete(ctx, bucket, object)
+func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string) error {
+	metadata := make(map[string]string)
+	metadata["etag"] = eTag
+	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, nil, "", true)
 }
 
-func (c *cacheObjects) put(ctx context.Context, dcache *diskCache, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions) error {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
-		return err
-	}
-	defer cLock.Unlock()
-	return dcache.Put(ctx, bucket, object, data, size, rs, opts)
-}
+// Backend metadata could have changed through server side copy - reset cache metadata if that is the case
+func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *diskCache, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
 
-func (c *cacheObjects) get(ctx context.Context, dcache *diskCache, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
-		return nil, err
+	bkMeta := make(map[string]string)
+	cacheMeta := make(map[string]string)
+	for k, v := range bkObjectInfo.UserDefined {
+		if HasPrefix(k, ReservedMetadataPrefix) {
+			// Do not need to send any internal metadata
+			continue
+		}
+		bkMeta[http.CanonicalHeaderKey(k)] = v
+	}
+	for k, v := range cacheObjInfo.UserDefined {
+		if HasPrefix(k, ReservedMetadataPrefix) {
+			// Do not need to send any internal metadata
+			continue
+		}
+		cacheMeta[http.CanonicalHeaderKey(k)] = v
 	}
 
-	defer cLock.RUnlock()
-	return dcache.Get(ctx, bucket, object, rs, h, opts)
-}
-
-func (c *cacheObjects) stat(ctx context.Context, dcache *diskCache, bucket, object string) (oi ObjectInfo, err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
-		return oi, err
+	if !isMetadataSame(bkMeta, cacheMeta) ||
+		bkObjectInfo.ETag != cacheObjInfo.ETag ||
+		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
+		!bkObjectInfo.Expires.Equal(cacheObjInfo.Expires) {
+		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false)
 	}
-
-	defer cLock.RUnlock()
-	return dcache.Stat(ctx, bucket, object)
-}
-
-func (c *cacheObjects) statRange(ctx context.Context, dcache *diskCache, bucket, object string, rs *HTTPRangeSpec) (oi ObjectInfo, err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
-		return oi, err
-	}
-
-	defer cLock.RUnlock()
-	oi, _, err = dcache.statRange(ctx, bucket, object, rs)
-	return oi, err
+	return c.incHitsToMeta(ctx, dcache, bucket, object, cacheObjInfo.Size, cacheObjInfo.ETag)
 }
 
 // DeleteObject clears cache entry if backend delete operation succeeds
@@ -149,9 +135,7 @@ func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string)
 	if cerr != nil {
 		return
 	}
-	if dcache.Exists(ctx, bucket, object) {
-		c.delete(ctx, dcache, bucket, object)
-	}
+	dcache.Delete(ctx, bucket, object)
 	return
 }
 
@@ -191,7 +175,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
-	var cc cacheControl
+	var cc *cacheControl
 	var cacheObjSize int64
 	// fetch diskCache if object is currently cached or nearest available cache drive
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
@@ -199,7 +183,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 
-	cacheReader, cacheErr := c.get(ctx, dcache, bucket, object, rs, h, opts)
+	cacheReader, numCacheHits, cacheErr := dcache.Get(ctx, bucket, object, rs, h, opts)
 	if cacheErr == nil {
 		cacheObjSize = cacheReader.ObjInfo.Size
 		if rs != nil {
@@ -208,8 +192,8 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			}
 		}
 		cc = cacheControlOpts(cacheReader.ObjInfo)
-		if (!cc.isEmpty() && !cc.isStale(cacheReader.ObjInfo.ModTime)) ||
-			cc.onlyIfCached {
+		if cc != nil && (!cc.isStale(cacheReader.ObjInfo.ModTime) ||
+			cc.onlyIfCached) {
 			// This is a cache hit, mark it so
 			bytesServed := cacheReader.ObjInfo.Size
 			if rs != nil {
@@ -219,9 +203,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			}
 			c.cacheStats.incHit()
 			c.cacheStats.incBytesServed(bytesServed)
+			c.incHitsToMeta(ctx, dcache, bucket, object, cacheReader.ObjInfo.Size, cacheReader.ObjInfo.ETag)
 			return cacheReader, nil
 		}
-		if cc.noStore {
+		if cc != nil && cc.noStore {
 			c.cacheStats.incMiss()
 			bReader, err := c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 			bReader.ObjInfo.CacheLookupStatus = CacheHit
@@ -251,32 +236,46 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		c.cacheStats.incMiss()
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
-
+	// skip cache for objects with locks
+	objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
+	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
+		c.cacheStats.incMiss()
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	}
 	if cacheErr == nil {
 		// if ETag matches for stale cache entry, serve from cache
 		if cacheReader.ObjInfo.ETag == objInfo.ETag {
 			// Update metadata in case server-side copy might have changed object metadata
-			dcache.updateMetadataIfChanged(ctx, bucket, object, objInfo, cacheReader.ObjInfo)
+			c.updateMetadataIfChanged(ctx, dcache, bucket, object, objInfo, cacheReader.ObjInfo)
 			c.incCacheStats(cacheObjSize)
 			return cacheReader, nil
 		}
 		cacheReader.Close()
 		// Object is stale, so delete from cache
-		c.delete(ctx, dcache, bucket, object)
+		dcache.Delete(ctx, bucket, object)
 	}
 
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
 	// Since we got here, we are serving the request from backend,
 	// and also adding the object to the cache.
-	if !dcache.diskUsageLow() {
-		select {
-		case dcache.purgeChan <- struct{}{}:
-		default:
-		}
+	if dcache.diskUsageHigh() {
+		dcache.incGCCounter()
 	}
 
 	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+
+	if bkErr != nil {
+		return bkReader, bkErr
+	}
+	// If object has less hits than configured cache after, just increment the hit counter
+	// but do not cache it.
+	if numCacheHits < c.after {
+		c.incHitsToMeta(ctx, dcache, bucket, object, objInfo.Size, objInfo.ETag)
+		return bkReader, bkErr
+	}
+
 	// Record if cache has a hit that was invalidated by ETag verification
 	if cacheErr == nil {
 		bkReader.ObjInfo.CacheLookupStatus = CacheHit
@@ -293,10 +292,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 				return
 			}
 			defer bReader.Close()
-			oi, err := c.statRange(ctx, dcache, bucket, object, rs)
+			oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
+				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
 			}
 		}()
 		return bkReader, bkErr
@@ -306,7 +305,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	go func() {
-		putErr := c.put(ctx, dcache, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)})
+		putErr := dcache.Put(ctx, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)}, false)
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
@@ -329,12 +328,12 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	if err != nil {
 		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
-	var cc cacheControl
+	var cc *cacheControl
 	// if cache control setting is valid, avoid HEAD operation to backend
-	cachedObjInfo, cerr := c.stat(ctx, dcache, bucket, object)
+	cachedObjInfo, _, cerr := dcache.Stat(ctx, bucket, object)
 	if cerr == nil {
 		cc = cacheControlOpts(cachedObjInfo)
-		if !cc.isStale(cachedObjInfo.ModTime) {
+		if cc == nil || (cc != nil && !cc.isStale(cachedObjInfo.ModTime)) {
 			// This is a cache hit, mark it so
 			c.cacheStats.incHit()
 			return cachedObjInfo, nil
@@ -345,7 +344,7 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
-			c.delete(ctx, dcache, bucket, object)
+			dcache.Delete(ctx, bucket, object)
 			c.cacheStats.incMiss()
 			return ObjectInfo{}, err
 		}
@@ -369,11 +368,35 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	}
 	if cachedObjInfo.ETag != objInfo.ETag {
 		// Delete the cached entry if the backend object was replaced.
-		c.delete(ctx, dcache, bucket, object)
+		dcache.Delete(ctx, bucket, object)
 	}
 	return objInfo, nil
 }
 
+// CopyObject reverts to backend after evicting any stale cache entries
+func (c *cacheObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+	copyObjectFn := c.CopyObjectFn
+	if c.isCacheExclude(srcBucket, srcObject) || c.skipCache() {
+		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+	}
+	if srcBucket != dstBucket || srcObject != dstObject {
+		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+	}
+	// fetch diskCache if object is currently cached or nearest available cache drive
+	dcache, err := c.getCacheToLoc(ctx, srcBucket, srcObject)
+	if err != nil {
+		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+	}
+	// if currently cached, evict old entry and revert to backend.
+	if cachedObjInfo, _, cerr := dcache.Stat(ctx, srcBucket, srcObject); cerr == nil {
+		cc := cacheControlOpts(cachedObjInfo)
+		if cc == nil || !cc.isStale(cachedObjInfo.ModTime) {
+			dcache.Delete(ctx, srcBucket, srcObject)
+		}
+	}
+	return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+}
+
 // StorageInfo - returns underlying storage statistics.
 func (c *cacheObjects) StorageInfo(ctx context.Context) (cInfo CacheStorageInfo) {
 	var total, free uint64
@@ -497,16 +520,10 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		if quota == 0 {
 			quota = config.Quota
 		}
-
-		cache, err := newDiskCache(dir, config.Expiry, quota)
+		cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
 		if err != nil {
 			return nil, false, err
 		}
-		// Start the purging go-routine for entries that have expired if no migration in progress
-		if !migrating {
-			go cache.purge()
-		}
-
 		caches = append(caches, cache)
 	}
 	return caches, migrating, nil
@@ -553,13 +570,12 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	}
 
 	errCnt := 0
-	for index, err := range g.Wait() {
+	for _, err := range g.Wait() {
 		if err != nil {
 			errCnt++
 			logger.LogIf(ctx, err)
 			continue
 		}
-		go c.cache[index].purge()
 	}
 
 	if errCnt > 0 {
@@ -596,8 +612,9 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	}
 
 	// skip cache for objects with locks
-	objRetention := getObjectRetentionMeta(opts.UserDefined)
-	if objRetention.Mode == Governance || objRetention.Mode == Compliance {
+	objRetention := objectlock.GetObjectRetentionMeta(opts.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(opts.UserDefined)
+	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
@@ -619,10 +636,10 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 				return
 			}
 			defer bReader.Close()
-			oi, err := c.stat(ctx, dcache, bucket, object)
+			oi, _, err := dcache.Stat(ctx, bucket, object)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
+				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
 			}
 		}()
 	}
@@ -638,13 +655,12 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 	if err != nil {
 		return nil, err
 	}
-
 	c := &cacheObjects{
 		cache:      cache,
 		exclude:    config.Exclude,
+		after:      config.After,
 		migrating:  migrateSw,
 		migMutex:   sync.Mutex{},
-		nsMutex:    newNSLock(false),
 		cacheStats: newCacheStats(),
 		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
@@ -665,13 +681,43 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
 		},
-	}
-	c.NewNSLockFn = func(ctx context.Context, bucket, object string) RWLocker {
-		return c.nsMutex.NewNSLock(ctx, nil, bucket, object)
+		CopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
+		},
 	}
 
 	if migrateSw {
 		go c.migrateCacheFromV1toV2(ctx)
 	}
+	go c.gc(ctx, GlobalServiceDoneCh)
 	return c, nil
 }
+
+func (c *cacheObjects) gc(ctx context.Context, doneCh <-chan struct{}) {
+	ticker := time.NewTicker(cacheGCInterval)
+
+	defer ticker.Stop()
+	for {
+		select {
+		case <-doneCh:
+			return
+		case <-ticker.C:
+			if c.migrating {
+				continue
+			}
+			var wg sync.WaitGroup
+			for _, dcache := range c.cache {
+				if dcache.gcCount() == 0 {
+					continue
+				}
+				wg.Add(1)
+				go func(d *diskCache) {
+					defer wg.Done()
+					d.resetGCCounter()
+					d.purge(ctx, doneCh)
+				}(dcache)
+			}
+			wg.Wait()
+		}
+	}
+}

cmd/disk-cache_test.go

@@ -27,15 +27,15 @@ import (
 )
 
 // Initialize cache objects.
-func initCacheObjects(disk string, cacheMaxUse int) (*diskCache, error) {
-	return newDiskCache(disk, 80, cacheMaxUse)
+func initCacheObjects(disk string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int) (*diskCache, error) {
+	return newDiskCache(disk, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) ([]*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int, t *testing.T) ([]*diskCache, error) {
 	var cb []*diskCache
 	for _, d := range drives {
-		obj, err := initCacheObjects(d, cacheMaxUse)
+		obj, err := initCacheObjects(d, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
 		if err != nil {
 			return nil, err
 		}
@@ -70,7 +70,7 @@ func TestGetCachedLoc(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, 100, t)
+		d, err := initDiskCaches(fsDirs, 100, 1, 80, 90, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -109,7 +109,7 @@ func TestGetCacheMaxUse(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, 80, t)
+		d, err := initDiskCaches(fsDirs, 80, 1, 80, 90, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -180,7 +180,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, 100, t)
+	d, err := initDiskCaches(fsDirs, 100, 0, 80, 90, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -212,11 +212,11 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta})
+	err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta}, false)
 	if err != nil {
 		t.Fatal(err)
 	}
-	cReader, err := cache.Get(ctx, bucketName, objectName, nil, http.Header{
+	cReader, _, err := cache.Get(ctx, bucketName, objectName, nil, http.Header{
 		"Content-Type": []string{"application/json"},
 	}, opts)
 	if err != nil {
@@ -258,7 +258,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, 80, t)
+	d, err := initDiskCaches(fsDirs, 90, 0, 90, 99, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -290,16 +290,16 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta}, false)
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta}, false)
 		if err != nil {
 			t.Fatal(err)
 		}
-		cReader, err := cache.Get(ctx, bucketName, objectName, nil, nil, opts)
+		cReader, _, err := cache.Get(ctx, bucketName, objectName, nil, nil, opts)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/dummy-handlers.go

@@ -21,27 +21,14 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Data types used for returning dummy tagging XML.
 // These variables shouldn't be used elsewhere.
 // They are only defined to be used in this file alone.
 
-type tagging struct {
-	XMLName xml.Name `xml:"Tagging"`
-	TagSet  tagSet   `xml:"TagSet"`
-}
-
-type tagSet struct {
-	Tag []tagElem `xml:"Tag"`
-}
-
-type tagElem struct {
-	Key   string `xml:"Key"`
-	Value string `xml:"Value"`
-}
-
 // GetBucketWebsite  - GET bucket website, a dummy api
 func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseHeadersOnly(w)
@@ -171,47 +158,8 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	tags := &tagging{}
-	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
-
-	if err := xml.NewEncoder(w).Encode(tags); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	w.(http.Flusher).Flush()
-}
-
-// GetObjectTaggingHandler - GET object tagging, a dummy api
-func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetObjectTagging")
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	object := vars["object"]
-
-	objAPI := api.ObjectAPI()
-	if objAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Allow getObjectTagging if policy action is set, since this is a dummy call
-	// we are simply re-purposing the bucketPolicyAction.
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Validate if object exists, before proceeding further...
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	tags := &tagging{}
-	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
+	tags := &tagging.Tagging{}
+	tags.TagSet.Tags = append(tags.TagSet.Tags, tagging.Tag{})
 
 	if err := xml.NewEncoder(w).Encode(tags); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/encryption-v1_test.go

@@ -280,15 +280,11 @@ func TestGetDecryptedRange_Issue50(t *testing.T) {
 		Parts: []ObjectPartInfo{
 			{
 				Number:     1,
-				Name:       "part.1",
-				ETag:       "etag1",
 				Size:       297580380,
 				ActualSize: 297435132,
 			},
 			{
 				Number:     2,
-				Name:       "part.2",
-				ETag:       "etag2",
 				Size:       297580380,
 				ActualSize: 297435132,
 			},

cmd/endpoint-ellipses.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -108,12 +108,24 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		return ss
 	}
 
+	setCounts := possibleSetCounts(commonSize)
+	if len(setCounts) == 0 {
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
+	}
+
 	if customSetDriveCount > 0 {
-		msg := fmt.Sprintf("Invalid set drive count, leads to non-uniform distribution for the given number of disks. Possible values for custom set count are %d", possibleSetCounts(setSize))
+		msg := fmt.Sprintf("Invalid set drive count. Acceptable values for %d number drives are %d", commonSize, setCounts)
 		if customSetDriveCount > setSize {
 			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
-		if setSize%customSetDriveCount != 0 {
+		var found bool
+		for _, ss := range setCounts {
+			if ss == customSetDriveCount {
+				found = true
+			}
+		}
+		if !found {
 			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
 		setSize = customSetDriveCount
@@ -121,7 +133,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		msg := fmt.Sprintf("Incorrect number of endpoints provided %s", args)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
 		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 
@@ -261,9 +273,6 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, -1, config.ErrInvalidErasureSetSize(err)
 		}
-		if !isValidSetSize(uint64(setDriveCount)) {
-			return nil, -1, -1, config.ErrInvalidErasureSetSize(nil)
-		}
 	}
 
 	if !ellipses.HasEllipses(args...) {

cmd/endpoint-ellipses_test.go

@@ -104,6 +104,36 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
 			8,
 			true,
 		},
+		{
+			[]string{"http://host{1...12}/data{1...12}"},
+			[]uint64{144},
+			[][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16}},
+			16,
+			true,
+		},
+		{
+			[]string{"http://host{0...5}/data{1...28}"},
+			[]uint64{168},
+			[][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}},
+			12,
+			true,
+		},
+		// Incorrect custom set drive count.
+		{
+			[]string{"http://host{0...5}/data{1...28}"},
+			[]uint64{168},
+			nil,
+			10,
+			false,
+		},
+		// Failure not divisible number of disks.
+		{
+			[]string{"http://host{1...11}/data{1...11}"},
+			[]uint64{121},
+			nil,
+			11,
+			false,
+		},
 		{
 			[]string{"data{1...60}"},
 			nil,

cmd/fastwalk.go

@@ -11,216 +11,25 @@ package cmd
 import (
 	"errors"
 	"os"
-	"path/filepath"
-	"runtime"
 	"strings"
-	"sync"
 )
 
-// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the
-// symlink named in the call may be traversed.
-var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory")
+var errSkipFile = errors.New("fastwalk: skip this file")
 
-// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the
-// callback should not be called for any other files in the current directory.
-// Child directories will still be traversed.
-var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
-
-// Walk is a faster implementation of filepath.Walk.
-//
-// filepath.Walk's design necessarily calls os.Lstat on each file,
-// even if the caller needs less info.
-// Many tools need only the type of each file.
-// On some platforms, this information is provided directly by the readdir
-// system call, avoiding the need to stat each file individually.
-// fastwalk_unix.go contains a fork of the syscall routines.
-//
-// See golang.org/issue/16399
-//
-// Walk walks the file tree rooted at root, calling walkFn for
-// each file or directory in the tree, including root.
-//
-// If fastWalk returns filepath.SkipDir, the directory is skipped.
-//
-// Unlike filepath.Walk:
-//   * file stat calls must be done by the user.
-//     The only provided metadata is the file type, which does not include
-//     any permission bits.
-//   * multiple goroutines stat the filesystem concurrently. The provided
-//     walkFn must be safe for concurrent use.
-//   * fastWalk can follow symlinks if walkFn returns the TraverseLink
-//     sentinel error. It is the walkFn's responsibility to prevent
-//     fastWalk from going into symlink cycles.
-func fastWalk(root string, walkFn func(path string, typ os.FileMode) error) error {
-	// TODO(bradfitz): make numWorkers configurable? We used a
-	// minimum of 4 to give the kernel more info about multiple
-	// things we want, in hopes its I/O scheduling can take
-	// advantage of that. Hopefully most are in cache. Maybe 4 is
-	// even too low of a minimum. Profile more.
-	numWorkers := 4
-	if n := runtime.NumCPU(); n > numWorkers {
-		numWorkers = n
-	}
-
-	// Make sure to wait for all workers to finish, otherwise
-	// walkFn could still be called after returning. This Wait call
-	// runs after close(e.donec) below.
-	var wg sync.WaitGroup
-	defer wg.Wait()
-
-	w := &walker{
-		fn:       walkFn,
-		enqueuec: make(chan walkItem, numWorkers), // buffered for performance
-		workc:    make(chan walkItem, numWorkers), // buffered for performance
-		donec:    make(chan struct{}),
-
-		// buffered for correctness & not leaking goroutines:
-		resc: make(chan error, numWorkers),
-	}
-	defer close(w.donec)
-
-	for i := 0; i < numWorkers; i++ {
-		wg.Add(1)
-		go w.doWork(&wg)
-	}
-	todo := []walkItem{{dir: root}}
-	out := 0
-	for {
-		workc := w.workc
-		var workItem walkItem
-		if len(todo) == 0 {
-			workc = nil
-		} else {
-			workItem = todo[len(todo)-1]
-		}
-		select {
-		case workc <- workItem:
-			todo = todo[:len(todo)-1]
-			out++
-		case it := <-w.enqueuec:
-			todo = append(todo, it)
-		case err := <-w.resc:
-			out--
-			if err != nil {
-				return err
-			}
-			if out == 0 && len(todo) == 0 {
-				// It's safe to quit here, as long as the buffered
-				// enqueue channel isn't also readable, which might
-				// happen if the worker sends both another unit of
-				// work and its result before the other select was
-				// scheduled and both w.resc and w.enqueuec were
-				// readable.
-				select {
-				case it := <-w.enqueuec:
-					todo = append(todo, it)
-				default:
-					return nil
-				}
-			}
-		}
-	}
-}
-
-// doWork reads directories as instructed (via workc) and runs the
-// user's callback function.
-func (w *walker) doWork(wg *sync.WaitGroup) {
-	defer wg.Done()
-	for {
-		select {
-		case <-w.donec:
-			return
-		case it := <-w.workc:
-			select {
-			case <-w.donec:
-				return
-			case w.resc <- w.walk(it.dir, !it.callbackDone):
-			}
-		}
-	}
-}
-
-type walker struct {
-	fn func(path string, typ os.FileMode) error
-
-	donec    chan struct{} // closed on fastWalk's return
-	workc    chan walkItem // to workers
-	enqueuec chan walkItem // from workers
-	resc     chan error    // from workers
-}
-
-type walkItem struct {
-	dir          string
-	callbackDone bool // callback already called; don't do it again
-}
-
-func (w *walker) enqueue(it walkItem) {
-	select {
-	case w.enqueuec <- it:
-	case <-w.donec:
-	}
-}
-
-func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
-	joined := dirName + string(os.PathSeparator) + baseName
-	if typ == os.ModeDir {
-		w.enqueue(walkItem{dir: joined})
-		return nil
-	}
-
-	err := w.fn(joined, typ)
-	if typ == os.ModeSymlink {
-		if err == ErrTraverseLink {
-			// Set callbackDone so we don't call it twice for both the
-			// symlink-as-symlink and the symlink-as-directory later:
-			w.enqueue(walkItem{dir: joined, callbackDone: true})
-			return nil
-		}
-		if err == filepath.SkipDir {
-			// Permit SkipDir on symlinks too.
-			return nil
-		}
-	}
-	return err
-}
-
-func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error {
+func readDirFn(dirName string, fn func(entName string, typ os.FileMode) error) error {
 	fis, err := readDir(dirName)
 	if err != nil {
 		return err
 	}
-	skipFiles := false
 	for _, fi := range fis {
 		var mode os.FileMode
 		if strings.HasSuffix(fi, SlashSeparator) {
 			mode |= os.ModeDir
 		}
 
-		if mode == 0 && skipFiles {
-			continue
-		}
-
-		if err := fn(dirName, fi, mode); err != nil {
-			if err == ErrSkipFiles {
-				skipFiles = true
-				continue
-			}
+		if err = fn(fi, mode); err != nil {
 			return err
 		}
 	}
 	return nil
 }
-
-func (w *walker) walk(root string, runUserCallback bool) error {
-	if runUserCallback {
-		err := w.fn(root, os.ModeDir)
-		if err == filepath.SkipDir {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-	}
-
-	return readDirFn(root, w.onDirEnt)
-}

cmd/format-xl.go

@@ -426,7 +426,7 @@ func checkFormatXLValue(formatXL *formatXLV3) error {
 }
 
 // Check all format values.
-func checkFormatXLValues(formats []*formatXLV3) error {
+func checkFormatXLValues(formats []*formatXLV3, drivesPerSet int) error {
 	for i, formatXL := range formats {
 		if formatXL == nil {
 			continue
@@ -438,6 +438,9 @@ func checkFormatXLValues(formats []*formatXLV3) error {
 			return fmt.Errorf("%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)",
 				humanize.Ordinal(i+1), len(formats), humanize.Ordinal(i+1), len(formatXL.XL.Sets)*len(formatXL.XL.Sets[0]))
 		}
+		if len(formatXL.XL.Sets[0]) != drivesPerSet {
+			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatXL.XL.Sets[0]), drivesPerSet)
+		}
 	}
 	return nil
 }
@@ -643,7 +646,7 @@ func initXLMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*formatX
 		// goroutine will return its own instance of index variable.
 		index := index
 		g.Go(func() error {
-			return makeFormatXLMetaVolumes(storageDisks[index])
+			return makeFormatXLMetaVolumes(disksToInit[index])
 		}, index)
 	}
 
@@ -825,7 +828,7 @@ func ecDrivesNoConfig(drivesPerSet int) int {
 // Make XL backend meta volumes.
 func makeFormatXLMetaVolumes(disk StorageAPI) error {
 	// Attempt to create MinIO internal buckets.
-	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
+	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
 }
 
 var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

cmd/fs-v1-helpers.go

@@ -429,7 +429,7 @@ func fsDeleteFile(ctx context.Context, basePath, deletePath string) error {
 		return err
 	}
 
-	if err := deleteFile(basePath, deletePath); err != nil {
+	if err := deleteFile(basePath, deletePath, false); err != nil {
 		if err != errFileNotFound {
 			logger.LogIf(ctx, err)
 		}

cmd/fs-v1-metadata.go

@@ -182,9 +182,14 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 			objInfo.Expires = t.UTC()
 		}
 	}
+
+	// Add user tags to the object info
+	objInfo.UserTags = m.Meta[xhttp.AmzObjectTagging]
+
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
+	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(m.Meta)
 
 	// All the parts per object.

cmd/fs-v1-multipart.go

@@ -575,7 +575,6 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 		fsMeta.Parts[i] = ObjectPartInfo{
 			Number:     part.PartNumber,
-			ETag:       part.ETag,
 			Size:       fi.Size(),
 			ActualSize: actualSize,
 		}
@@ -758,7 +757,7 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
-func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/fs-v1-multipart_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 )
@@ -34,33 +35,38 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	obj := initFSObjects(disk, t)
 	fs := obj.(*FSObjects)
 
-	// Close the go-routine, we are going to
-	// manually start it and test in this test case.
-	GlobalServiceDoneCh <- struct{}{}
-
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{})
+	// Create a context we can cancel.
+	ctx, cancel := context.WithCancel(context.Background())
+	obj.MakeBucketWithLocation(ctx, bucketName, "")
+
+	uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
-	go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, GlobalServiceDoneCh)
+	var cleanupWg sync.WaitGroup
+	cleanupWg.Add(1)
+	go func() {
+		defer cleanupWg.Done()
+		fs.cleanupStaleMultipartUploads(context.Background(), time.Millisecond, 0, ctx.Done())
+	}()
 
-	// Wait for 40ms such that - we have given enough time for
-	// cleanup routine to kick in.
-	time.Sleep(40 * time.Millisecond)
-
-	// Close the routine we do not need it anymore.
-	GlobalServiceDoneCh <- struct{}{}
+	// Wait for 100ms such that - we have given enough time for
+	// cleanup routine to kick in. Flaky on slow systems...
+	time.Sleep(100 * time.Millisecond)
+	cancel()
+	cleanupWg.Wait()
 
 	// Check if upload id was already purged.
 	if err = obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
+	} else {
+		t.Error("Item was not cleaned up.")
 	}
 }
 

cmd/fs-v1.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,7 +26,6 @@ import (
 	"os"
 	"os/user"
 	"path"
-	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -37,13 +35,16 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/lifecycle"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
 	"github.com/minio/minio/pkg/mountinfo"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // Default etag is used for pre-existing objects.
@@ -109,8 +110,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 		return err
 	}
 
-	metaStatsPath := pathJoin(fsPath, minioMetaBackgroundOpsBucket, fsUUID)
-	if err := os.MkdirAll(metaStatsPath, 0777); err != nil {
+	if err := os.MkdirAll(pathJoin(fsPath, dataUsageBucket), 0777); err != nil {
 		return err
 	}
 
@@ -186,9 +186,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
 	// lockers are explicitly 'nil' for FS mode since there are only local lockers
-	return fs.nsMutex.NewNSLock(ctx, nil, bucket, object)
+	return fs.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
 }
 
 // Shutdown - should be called when process shuts down.
@@ -200,7 +200,7 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo - returns underlying storage statistics.
-func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
+func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo {
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
@@ -226,96 +226,44 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
 	return storageInfo
 }
 
-func (fs *FSObjects) waitForLowActiveIO() error {
-	t := time.NewTicker(lowActiveIOWaitTick)
-	defer t.Stop()
-	for {
-		if atomic.LoadInt64(&fs.activeIOCount) >= fs.maxActiveIOCount {
-			select {
-			case <-GlobalServiceDoneCh:
-				return errors.New("forced exit")
-			case <-t.C:
-				continue
-			}
-		}
-		break
+func (fs *FSObjects) waitForLowActiveIO() {
+	for atomic.LoadInt64(&fs.activeIOCount) >= fs.maxActiveIOCount {
+		time.Sleep(lowActiveIOWaitTick)
 	}
-
-	return nil
-
 }
 
-// crawlAndGetDataUsageInfo returns data usage stats of the current FS deployment
-func (fs *FSObjects) crawlAndGetDataUsageInfo(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-
-	var dataUsageInfoMu sync.Mutex
-	var dataUsageInfo = DataUsageInfo{
-		BucketsSizes:          make(map[string]uint64),
-		ObjectsSizesHistogram: make(map[string]uint64),
+// CrawlAndGetDataUsage returns data usage stats of the current FS deployment
+func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+	// Load bucket totals
+	var oldCache dataUsageCache
+	err := oldCache.load(ctx, fs, dataUsageCacheName)
+	if err != nil {
+		return err
 	}
-
-	walkFn := func(origPath string, typ os.FileMode) error {
-
-		select {
-		case <-GlobalServiceDoneCh:
-			return filepath.SkipDir
-		default:
-		}
-
-		if err := fs.waitForLowActiveIO(); err != nil {
-			return filepath.SkipDir
-		}
-
-		path := strings.TrimPrefix(origPath, fs.fsPath)
-		path = strings.TrimPrefix(path, SlashSeparator)
-
-		splits := splitN(path, SlashSeparator, 2)
-		bucket := splits[0]
-		prefix := splits[1]
-
-		if bucket == "" {
-			return nil
-		}
-
-		if isReservedOrInvalidBucket(bucket, false) {
-			return filepath.SkipDir
-		}
-
-		if prefix == "" {
-			dataUsageInfoMu.Lock()
-			dataUsageInfo.BucketsCount++
-			dataUsageInfo.BucketsSizes[bucket] = 0
-			dataUsageInfoMu.Unlock()
-			return nil
-		}
-
-		if typ&os.ModeDir != 0 {
-			return nil
-		}
-
-		// Get file size
-		fi, err := os.Stat(origPath)
+	if oldCache.Info.Name == "" {
+		oldCache.Info.Name = dataUsageRoot
+	}
+	buckets, err := fs.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+	cache, err := updateUsage(ctx, fs.fsPath, oldCache, fs.waitForLowActiveIO, func(item Item) (int64, error) {
+		// Get file size, symlinks which cannot be
+		// followed are automatically filtered by fastwalk.
+		fi, err := os.Stat(item.Path)
 		if err != nil {
-			return nil
+			return 0, errSkipFile
 		}
-		size := fi.Size()
+		return fi.Size(), nil
+	})
 
-		dataUsageInfoMu.Lock()
-		dataUsageInfo.ObjectsCount++
-		dataUsageInfo.ObjectsTotalSize += uint64(size)
-		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
-		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
-		dataUsageInfoMu.Unlock()
-
-		return nil
+	// Even if there was an error, the new cache may have better info.
+	if cache.Info.LastUpdate.After(oldCache.Info.LastUpdate) {
+		logger.LogIf(ctx, cache.save(ctx, fs, dataUsageCacheName))
+		updates <- cache.dui(dataUsageRoot, buckets)
 	}
 
-	fastWalk(fs.fsPath, walkFn)
-
-	dataUsageInfo.LastUpdate = UTCNow()
-	atomic.StoreUint64(&fs.totalUsed, dataUsageInfo.ObjectsTotalSize)
-
-	return dataUsageInfo
+	return err
 }
 
 /// Bucket operations
@@ -556,7 +504,6 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 // GetObjectNInfo - returns object info and a reader for object
 // content.
 func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-
 	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
 		return nil, err
 	}
@@ -1125,15 +1072,18 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // is a leaf or non-leaf entry.
 func (fs *FSObjects) listDirFactory() ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
-			return
+			return false, nil
+		}
+		if len(entries) == 0 {
+			return true, nil
 		}
 		sort.Strings(entries)
-		return filterMatchingPrefix(entries, prefixEntry)
+		return false, filterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -1235,6 +1185,58 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
 }
 
+// GetObjectTag - get object tags from an existing object
+func (fs *FSObjects) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	oi, err := fs.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+
+	tags, err := tagging.FromString(oi.UserTags)
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+
+	return tags, nil
+}
+
+// PutObjectTag - replace or add tags to an existing object
+func (fs *FSObjects) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+	fsMeta := fsMetaV1{}
+	wlk, err := fs.rwPool.Write(fsMetaPath)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return toObjectErr(err, bucket, object)
+	}
+	// This close will allow for locks to be synchronized on `fs.json`.
+	defer wlk.Close()
+
+	// Read objects' metadata in `fs.json`.
+	if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil {
+		// For any error to read fsMeta, set default ETag and proceed.
+		fsMeta = fs.defaultFsJSON(object)
+	}
+
+	// clean fsMeta.Meta of tag key, before updating the new tags
+	delete(fsMeta.Meta, xhttp.AmzObjectTagging)
+
+	// Do not update for empty tags
+	if tags != "" {
+		fsMeta.Meta[xhttp.AmzObjectTagging] = tags
+	}
+
+	if _, err = fsMeta.WriteTo(wlk); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+	return nil
+}
+
+// DeleteObjectTag - delete object tags from an existing object
+func (fs *FSObjects) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	return fs.PutObjectTag(ctx, bucket, object, "")
+}
+
 // ReloadFormat - no-op for fs, Valid only for XL.
 func (fs *FSObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
 	logger.LogIf(ctx, NotImplemented{})
@@ -1248,7 +1250,7 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
@@ -1261,8 +1263,17 @@ func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remo
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 
+// Walk a bucket, optionally prefix recursively, until we have returned
+// all the content to objectInfo channel, it is callers responsibility
+// to allocate a receive channel for ObjectInfo, upon any unhandled
+// error walker returns error. Optionally if context.Done() is received
+// then Walk() stops the walker.
+func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfo, fs.getObjectInfo)
+}
+
 // HealObjects - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
+func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) (e error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }
@@ -1273,12 +1284,6 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 	return []BucketInfo{}, NotImplemented{}
 }
 
-// ListObjectsHeal - list all objects to be healed. Valid only for XL
-func (fs *FSObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return ListObjectsInfo{}, NotImplemented{}
-}
-
 // GetMetrics - no op
 func (fs *FSObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
@@ -1315,6 +1320,21 @@ func (fs *FSObjects) DeleteBucketLifecycle(ctx context.Context, bucket string) e
 	return removeLifecycleConfig(ctx, fs, bucket)
 }
 
+// GetBucketSSEConfig returns bucket encryption config on given bucket
+func (fs *FSObjects) GetBucketSSEConfig(ctx context.Context, bucket string) (*bucketsse.BucketSSEConfig, error) {
+	return getBucketSSEConfig(fs, bucket)
+}
+
+// SetBucketSSEConfig sets bucket encryption config on given bucket
+func (fs *FSObjects) SetBucketSSEConfig(ctx context.Context, bucket string, config *bucketsse.BucketSSEConfig) error {
+	return saveBucketSSEConfig(ctx, fs, bucket, config)
+}
+
+// DeleteBucketSSEConfig deletes bucket encryption config on given bucket
+func (fs *FSObjects) DeleteBucketSSEConfig(ctx context.Context, bucket string) error {
+	return removeBucketSSEConfig(ctx, fs, bucket)
+}
+
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
 	marker := continuationToken

cmd/fs-v1_test.go

@@ -280,7 +280,7 @@ func TestFSDeleteObject(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with invalid object name
-	if err := fs.DeleteObject(context.Background(), bucketName, "\\"); !isSameType(err, ObjectNameInvalid{}) {
+	if err := fs.DeleteObject(context.Background(), bucketName, "\\"); !isSameType(err, ObjectNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with object does not exist.
@@ -356,8 +356,6 @@ func TestFSListBuckets(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	GlobalServiceDoneCh <- struct{}{}
-
 	// Create a bucket with invalid name
 	if err := os.MkdirAll(pathJoin(fs.fsPath, "vo^"), 0777); err != nil {
 		t.Fatal("Unexpected error: ", err)
@@ -392,7 +390,7 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false, madmin.HealDeepScan)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", madmin.HealOpts{})
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
@@ -404,7 +402,7 @@ func TestFSHealObjects(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	err := obj.HealObjects(context.Background(), "bucket", "prefix", nil)
+	err := obj.HealObjects(context.Background(), "bucket", "prefix", madmin.HealOpts{}, nil)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-common.go

@@ -304,9 +304,12 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
 	if err != nil {
 		return false
 	}
-	if _, err = clnt.Do(req); err != nil {
+	resp, err := clnt.Do(req)
+	if err != nil {
+		clnt.CloseIdleConnections()
 		return !xnet.IsNetworkOrHostDown(err)
 	}
+	xhttp.DrainBody(resp.Body)
 	return true
 }
 
@@ -343,7 +346,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = BucketNotEmpty{}
 	case "NoSuchBucketPolicy":
 		err = BucketPolicyNotFound{}
-	case "NoSuchBucketLifecycle":
+	case "NoSuchLifecycleConfiguration":
 		err = BucketLifecycleNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
@@ -414,3 +417,36 @@ func gatewayHandleEnvVars() {
 		}
 	}
 }
+
+// shouldMeterRequest checks whether incoming request should be added to prometheus gateway metrics
+func shouldMeterRequest(req *http.Request) bool {
+	return !(guessIsBrowserReq(req) || guessIsHealthCheckReq(req) || guessIsMetricsReq(req))
+}
+
+// MetricsTransport is a custom wrapper around Transport to track metrics
+type MetricsTransport struct {
+	Transport *http.Transport
+	Metrics   *Metrics
+}
+
+// RoundTrip implements the RoundTrip method for MetricsTransport
+func (m MetricsTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	metered := shouldMeterRequest(r)
+	if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
+		m.Metrics.IncRequests(r.Method)
+		if r.ContentLength > 0 {
+			m.Metrics.IncBytesSent(uint64(r.ContentLength))
+		}
+	}
+	// Make the request to the server.
+	resp, err := m.Transport.RoundTrip(r)
+	if err != nil {
+		return nil, err
+	}
+	if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
+		if r.ContentLength > 0 {
+			m.Metrics.IncBytesReceived(uint64(resp.ContentLength))
+		}
+	}
+	return resp, nil
+}

cmd/gateway-main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,15 +35,6 @@ import (
 	"github.com/minio/minio/pkg/env"
 )
 
-func init() {
-	logger.Init(GOPATH, GOROOT)
-	logger.RegisterError(config.FmtError)
-
-	// Initialize globalConsoleSys system
-	globalConsoleSys = NewConsoleLogger(context.Background())
-	logger.AddTarget(globalConsoleSys)
-}
-
 var (
 	gatewayCmd = cli.Command{
 		Name:            "gateway",
@@ -165,7 +156,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalServerConfig = srvCfg
 	globalServerConfigMu.Unlock()
 
-	router := mux.NewRouter().SkipClean(true)
+	// Initialize router. `SkipClean(true)` stops gorilla/mux from
+	// normalizing URL path minio/minio#3256
+	// avoid URL path encoding minio/minio#8950
+	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
 
 	if globalEtcdClient != nil {
 		// Enable STS router if etcd is enabled.
@@ -243,12 +237,16 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// sub-systems, make sure that we do not move the above codeblock elsewhere.
 	if enableConfigOps {
 		logger.FatalIf(globalConfigSys.Init(newObject), "Unable to initialize config system")
+		buckets, err := newObject.ListBuckets(context.Background())
+		if err != nil {
+			logger.Fatal(err, "Unable to list buckets")
+		}
 
+		logger.FatalIf(globalNotificationSys.Init(buckets, newObject), "Unable to initialize notification system")
 		// Start watching disk for reloading config, this
 		// is only enabled for "NAS" gateway.
 		globalConfigSys.WatchConfigNASDisk(newObject)
 	}
-
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
 	logger.SetDeploymentID(globalDeploymentID)
@@ -311,8 +309,5 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
 	}
 
-	// Set uptime time after object layer has initialized.
-	globalBootTime = UTCNow()
-
 	handleSignals()
 }

cmd/gateway-metrics.go

@@ -37,8 +37,8 @@ type Metrics struct {
 }
 
 // IncBytesReceived - Increase total bytes received from gateway backend
-func (s *Metrics) IncBytesReceived(n int64) {
-	s.bytesReceived.Add(uint64(n))
+func (s *Metrics) IncBytesReceived(n uint64) {
+	s.bytesReceived.Add(n)
 }
 
 // GetBytesReceived - Get total bytes received from gateway backend
@@ -47,8 +47,8 @@ func (s *Metrics) GetBytesReceived() uint64 {
 }
 
 // IncBytesSent - Increase total bytes sent to gateway backend
-func (s *Metrics) IncBytesSent(n int64) {
-	s.bytesSent.Add(uint64(n))
+func (s *Metrics) IncBytesSent(n uint64) {
+	s.bytesSent.Add(n)
 }
 
 // GetBytesSent - Get total bytes received from gateway backend

cmd/gateway-unsupported.go

@@ -21,9 +21,13 @@ import (
 	"errors"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/lifecycle"
+
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
+
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // GatewayLocker implements custom NeNSLock implementation
@@ -33,8 +37,8 @@ type GatewayLocker struct {
 }
 
 // NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, object)
+func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
 }
 
 // NewGatewayLayerWithLocker - initialize gateway with locker.
@@ -45,8 +49,14 @@ func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
 // GatewayUnsupported list of unsupported call stubs for gateway.
 type GatewayUnsupported struct{}
 
+// CrawlAndGetDataUsage - crawl is not implemented for gateway
+func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+	logger.CriticalIf(ctx, errors.New("not implemented"))
+	return NotImplemented{}
+}
+
 // NewNSLock is a dummy stub for gateway.
-func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
 	return nil
 }
@@ -121,6 +131,21 @@ func (a GatewayUnsupported) DeleteBucketLifecycle(ctx context.Context, bucket st
 	return NotImplemented{}
 }
 
+// GetBucketSSEConfig returns bucket encryption config on given bucket
+func (a GatewayUnsupported) GetBucketSSEConfig(ctx context.Context, bucket string) (*bucketsse.BucketSSEConfig, error) {
+	return nil, NotImplemented{}
+}
+
+// SetBucketSSEConfig sets bucket encryption config on given bucket
+func (a GatewayUnsupported) SetBucketSSEConfig(ctx context.Context, bucket string, config *bucketsse.BucketSSEConfig) error {
+	return NotImplemented{}
+}
+
+// DeleteBucketSSEConfig deletes bucket encryption config on given bucket
+func (a GatewayUnsupported) DeleteBucketSSEConfig(ctx context.Context, bucket string) error {
+	return NotImplemented{}
+}
+
 // ReloadFormat - Not implemented stub.
 func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error {
 	return NotImplemented{}
@@ -141,13 +166,8 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 	return nil, NotImplemented{}
 }
 
-// ListObjectsHeal - Not implemented stub
-func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	return ListObjectsInfo{}, NotImplemented{}
-}
-
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 
@@ -156,8 +176,13 @@ func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, c
 	return result, NotImplemented{}
 }
 
+// Walk - Not implemented stub
+func (a GatewayUnsupported) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	return NotImplemented{}
+}
+
 // HealObjects - Not implemented stub
-func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
+func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) (e error) {
 	return NotImplemented{}
 }
 
@@ -173,6 +198,24 @@ func (a GatewayUnsupported) GetMetrics(ctx context.Context) (*Metrics, error) {
 	return &Metrics{}, NotImplemented{}
 }
 
+// PutObjectTag - not implemented.
+func (a GatewayUnsupported) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	logger.LogIf(ctx, NotImplemented{})
+	return NotImplemented{}
+}
+
+// GetObjectTag - not implemented.
+func (a GatewayUnsupported) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	logger.LogIf(ctx, NotImplemented{})
+	return tagging.Tagging{}, NotImplemented{}
+}
+
+// DeleteObjectTag - not implemented.
+func (a GatewayUnsupported) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	logger.LogIf(ctx, NotImplemented{})
+	return NotImplemented{}
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (a GatewayUnsupported) IsNotificationSupported() bool {
 	return false

cmd/gateway/azure/gateway-azure.go

@@ -39,11 +39,10 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
-	"github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/policy/condition"
 	sha256 "github.com/minio/sha256-simd"
 
 	minio "github.com/minio/minio/cmd"
@@ -94,8 +93,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
      {{.Prompt}} {{.HelpName}}
 `
 
@@ -146,7 +147,14 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 		return &azureObjects{}, err
 	}
 
-	httpClient := &http.Client{Transport: minio.NewCustomHTTPTransport()}
+	metrics := minio.NewMetrics()
+
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
+	}
+
+	httpClient := &http.Client{Transport: t}
 	userAgent := fmt.Sprintf("APN/1.0 MinIO/1.0 MinIO/%s", minio.Version)
 
 	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
@@ -168,6 +176,7 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 		endpoint:   endpointURL.String(),
 		httpClient: httpClient,
 		client:     client,
+		metrics:    metrics,
 	}, nil
 }
 
@@ -357,6 +366,7 @@ type azureObjects struct {
 	minio.GatewayUnsupported
 	endpoint   string
 	httpClient *http.Client
+	metrics    *minio.Metrics
 	client     azblob.ServiceURL // Azure sdk client
 }
 
@@ -460,6 +470,11 @@ func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) {
 	return
 }
 
+// GetMetrics returns this gateway's metrics
+func (a *azureObjects) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+	return a.metrics, nil
+}
+
 // Shutdown - save any gateway metadata to disk
 // if necessary and reload upon next restart.
 func (a *azureObjects) Shutdown(ctx context.Context) error {
@@ -467,7 +482,7 @@ func (a *azureObjects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo - Not relevant to Azure backend.
-func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, a.httpClient, a.endpoint)
 	return si
@@ -1180,7 +1195,7 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	objMetadata["md5sum"] = cmd.ComputeCompleteMultipartMD5(uploadedParts)
+	objMetadata["md5sum"] = minio.ComputeCompleteMultipartMD5(uploadedParts)
 
 	_, err = objBlob.CommitBlockList(ctx, allBlocks, objProperties, objMetadata, azblob.BlobAccessConditions{})
 	if err != nil {

cmd/gateway/b2/gateway-b2.go

@@ -33,9 +33,9 @@ import (
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/policy/condition"
 	h2 "github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
 
 	minio "github.com/minio/minio/cmd"
 )
@@ -69,8 +69,11 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+
      {{.Prompt}} {{.HelpName}}
 `
 	minio.RegisterGatewayCommand(cli.Command{
@@ -127,6 +130,7 @@ type b2Objects struct {
 	b2Client   *b2.B2
 	httpClient *http.Client
 	ctx        context.Context
+	buckets    []*b2.Bucket
 }
 
 // Convert B2 errors to minio object layer errors.
@@ -217,7 +221,7 @@ func (l *b2Objects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo is not relevant to B2 backend.
-func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *b2Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1")
 	return si
@@ -257,11 +261,14 @@ func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, e
 			return nil, rerr
 		}
 	}
-	bktList, lerr := l.b2Client.ListBuckets(l.ctx)
-	if lerr != nil {
-		return l.listBuckets(ctx, lerr)
+	if len(l.buckets) == 0 {
+		bktList, lerr := l.b2Client.ListBuckets(l.ctx)
+		if lerr != nil {
+			return l.listBuckets(ctx, lerr)
+		}
+		l.buckets = bktList
 	}
-	return bktList, nil
+	return l.buckets, nil
 }
 
 // Bucket - is a helper which provides a *Bucket instance
@@ -316,6 +323,20 @@ func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error {
 		return err
 	}
 	err = bkt.DeleteBucket(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		err = bkt.DeleteBucket(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket)
+	}
+
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err, bucket)
 }
@@ -328,8 +349,18 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 	}
 	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
-		logger.LogIf(ctx, lerr)
-		return loi, b2ToObjectError(lerr, bucket)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return loi, err
+		}
+
+		files, next, lerr = bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+		if lerr != nil {
+			logger.LogIf(ctx, lerr)
+			return loi, b2ToObjectError(lerr, bucket)
+		}
 	}
 	loi.IsTruncated = next != ""
 	loi.NextMarker = next
@@ -370,8 +401,18 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	}
 	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
-		logger.LogIf(ctx, lerr)
-		return loi, b2ToObjectError(lerr, bucket)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return loi, err
+		}
+
+		files, next, lerr = bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+		if lerr != nil {
+			logger.LogIf(ctx, lerr)
+			return loi, b2ToObjectError(lerr, bucket)
+		}
 	}
 	loi.IsTruncated = next != ""
 	loi.ContinuationToken = continuationToken
@@ -433,8 +474,18 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string,
 	}
 	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		reader, err = bkt.DownloadFileByName(l.ctx, object, startOffset, length)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return b2ToObjectError(err, bucket, object)
+		}
 	}
 	defer reader.Close()
 	_, err = io.Copy(writer, reader)
@@ -451,8 +502,18 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 
 	f, _, err := bkt.ListFileNames(l.ctx, 1, object, "", "")
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return objInfo, err
+		}
+
+		f, _, err = bkt.ListFileNames(l.ctx, 1, object, "", "")
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, b2ToObjectError(err, bucket, object)
+		}
 	}
 
 	// B2's list will return the next item in the bucket if the object doesn't
@@ -466,8 +527,18 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 
 	fi, err := bkt.File(f[0].ID, object).GetFileInfo(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return objInfo, err
+		}
+
+		fi, err = bkt.File(f[0].ID, object).GetFileInfo(l.ctx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, b2ToObjectError(err, bucket, object)
+		}
 	}
 	return minio.ObjectInfo{
 		Bucket:      bucket,
@@ -557,8 +628,18 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
 	var u *b2.URL
 	u, err = bkt.GetUploadURL(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return objInfo, err
+		}
+
+		u, err = bkt.GetUploadURL(l.ctx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, b2ToObjectError(err, bucket, object)
+		}
 	}
 
 	hr := newB2Reader(data, data.Size())
@@ -597,6 +678,19 @@ func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object stri
 	// If we hide the file we'll conform to B2's versioning policy, it also
 	// saves an additional call to check if the file exists first
 	_, err = bkt.HideFile(l.ctx, object)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		_, err = bkt.HideFile(l.ctx, object)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err, bucket, object)
 }
@@ -625,8 +719,18 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 	}
 	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return lmi, b2ToObjectError(err, bucket)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return lmi, err
+		}
+
+		largeFiles, nextMarker, err = bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return lmi, b2ToObjectError(err, bucket)
+		}
 	}
 	lmi = minio.ListMultipartsInfo{
 		MaxUploads: maxUploads,
@@ -661,8 +765,18 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 	delete(opts.UserDefined, "content-type")
 	lf, err := bkt.StartLargeFile(l.ctx, object, contentType, opts.UserDefined)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return uploadID, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return uploadID, err
+		}
+
+		lf, err = bkt.StartLargeFile(l.ctx, object, contentType, opts.UserDefined)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return uploadID, b2ToObjectError(err, bucket, object)
+		}
 	}
 
 	return lf.ID, nil
@@ -678,8 +792,18 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
 
 	fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return pi, b2ToObjectError(err, bucket, object, uploadID)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return pi, err
+		}
+
+		fc, err = bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return pi, b2ToObjectError(err, bucket, object, uploadID)
+		}
 	}
 
 	hr := newB2Reader(data, data.Size())
@@ -714,8 +838,18 @@ func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object s
 	partNumberMarker++
 	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return lpi, b2ToObjectError(err, bucket, object, uploadID)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return lpi, err
+		}
+
+		partsList, next, err = bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return lpi, b2ToObjectError(err, bucket, object, uploadID)
+		}
 	}
 	if next != 0 {
 		lpi.IsTruncated = true
@@ -738,6 +872,19 @@ func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
 		return err
 	}
 	err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object, uploadID)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err, bucket, object, uploadID)
 }
@@ -762,8 +909,17 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
-		logger.LogIf(ctx, err)
-		return oi, b2ToObjectError(err, bucket, object, uploadID)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return oi, err
+		}
+
+		if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
+			logger.LogIf(ctx, err)
+			return oi, b2ToObjectError(err, bucket, object, uploadID)
+		}
 	}
 
 	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
@@ -806,6 +962,20 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPo
 	}
 	bkt.Type = bucketTypeReadOnly
 	_, err = bkt.Update(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+		bkt.Type = bucketTypeReadOnly
+
+		_, err = bkt.Update(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err)
 }
@@ -854,6 +1024,20 @@ func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 	}
 	bkt.Type = bucketTypePrivate
 	_, err = bkt.Update(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+		bkt.Type = bucketTypePrivate
+
+		_, err = bkt.Update(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err)
 }

cmd/gateway/gcs/gateway-gcs.go

@@ -41,9 +41,9 @@ import (
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/policy/condition"
 	"github.com/minio/minio/pkg/env"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
 
 	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
@@ -124,8 +124,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
      {{.Prompt}} {{.HelpName}} mygcsprojectid
 `
 
@@ -179,6 +181,13 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 		}
 	}
 
+	metrics := minio.NewMetrics()
+
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
+	}
+
 	// Initialize a GCS client.
 	// Send user-agent in this format for Google to obtain usage insights while participating in the
 	// Google Cloud Technology Partners (https://cloud.google.com/partners/)
@@ -190,8 +199,9 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 	gcs := &gcsGateway{
 		client:    client,
 		projectID: g.projectID,
+		metrics:   metrics,
 		httpClient: &http.Client{
-			Transport: minio.NewCustomHTTPTransport(),
+			Transport: t,
 		},
 	}
 
@@ -333,6 +343,7 @@ type gcsGateway struct {
 	minio.GatewayUnsupported
 	client     *storage.Client
 	httpClient *http.Client
+	metrics    *minio.Metrics
 	projectID  string
 }
 
@@ -349,6 +360,11 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
 	return googleCreds[gcsProjectIDKey], err
 }
 
+// GetMetrics returns this gateway's metrics
+func (l *gcsGateway) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+	return l.metrics, nil
+}
+
 // Cleanup old files in minio.sys.tmp of the given bucket.
 func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) {
 	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
@@ -398,7 +414,7 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo - Not relevant to GCS backend.
-func (l *gcsGateway) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com")
 	return si

cmd/gateway/hdfs/gateway-hdfs.go

@@ -75,8 +75,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90 
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
      {{.Prompt}} {{.HelpName}} hdfs://namenode:8200
 `
 
@@ -196,14 +198,14 @@ func (g *HDFS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error
 
 // Production - hdfs gateway is production ready.
 func (g *HDFS) Production() bool {
-	return false
+	return true
 }
 
 func (n *hdfsObjects) Shutdown(ctx context.Context) error {
 	return n.clnt.Close()
 }
 
-func (n *hdfsObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
+func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
 	fsInfo, err := n.clnt.StatFs()
 	if err != nil {
 		return minio.StorageInfo{}
@@ -324,7 +326,7 @@ func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketIn
 
 func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
 		f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, prefixDir))
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -339,6 +341,9 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 			logger.LogIf(context.Background(), err)
 			return
 		}
+		if len(fis) == 0 {
+			return true, nil
+		}
 		for _, fi := range fis {
 			if fi.IsDir() {
 				entries = append(entries, fi.Name()+hdfsSeparator)
@@ -346,7 +351,7 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 				entries = append(entries, fi.Name())
 			}
 		}
-		return minio.FilterMatchingPrefix(entries, prefixEntry)
+		return false, minio.FilterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -355,8 +360,23 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 
 // ListObjects lists all blobs in HDFS bucket filtered by prefix.
 func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
+	if _, err := n.clnt.Stat(minio.PathJoin(hdfsSeparator, bucket)); err != nil {
+		return loi, hdfsToObjectErr(ctx, err, bucket)
+	}
+
 	getObjectInfo := func(ctx context.Context, bucket, entry string) (minio.ObjectInfo, error) {
-		return n.GetObjectInfo(ctx, bucket, entry, minio.ObjectOptions{})
+		fi, err := n.clnt.Stat(minio.PathJoin(hdfsSeparator, bucket, entry))
+		if err != nil {
+			return minio.ObjectInfo{}, hdfsToObjectErr(ctx, err, bucket, entry)
+		}
+		return minio.ObjectInfo{
+			Bucket:  bucket,
+			Name:    entry,
+			ModTime: fi.ModTime(),
+			Size:    fi.Size(),
+			IsDir:   fi.IsDir(),
+			AccTime: fi.(*hdfs.FileInfo).AccessTime(),
+		}, nil
 	}
 
 	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), getObjectInfo, getObjectInfo)

cmd/gateway/nas/gateway-nas.go

@@ -52,8 +52,11 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+
      {{.Prompt}} {{.HelpName}} /shared/nasvol
 `
 
@@ -106,8 +109,8 @@ func (n *nasObjects) IsListenBucketSupported() bool {
 	return false
 }
 
-func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
-	sinfo := n.ObjectLayer.StorageInfo(ctx)
+func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) minio.StorageInfo {
+	sinfo := n.ObjectLayer.StorageInfo(ctx, false)
 	sinfo.Backend.GatewayOnline = sinfo.Backend.Type == minio.BackendFS
 	sinfo.Backend.Type = minio.BackendGateway
 	return sinfo
@@ -120,6 +123,6 @@ type nasObjects struct {
 
 // IsReady returns whether the layer is ready to take requests.
 func (n *nasObjects) IsReady(ctx context.Context) bool {
-	sinfo := n.ObjectLayer.StorageInfo(ctx)
+	sinfo := n.ObjectLayer.StorageInfo(ctx, false)
 	return sinfo.Backend.Type == minio.BackendFS
 }

cmd/gateway/oss/gateway-oss.go

@@ -35,9 +35,9 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/policy/condition"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
 )
 
 const (
@@ -71,8 +71,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
      {{.Prompt}} {{.HelpName}}
 `
 
@@ -320,7 +322,7 @@ func (l *ossObjects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo is not relevant to OSS backend.
-func (l *ossObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *ossObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint)
 	return si

cmd/gateway/s3/gateway-s3-sse.go

@@ -506,7 +506,6 @@ func (l *s3EncObjects) PutObjectPart(ctx context.Context, bucket string, object
 		Number: partID,
 		ETag:   pi.ETag,
 		Size:   pi.Size,
-		Name:   strconv.Itoa(partID),
 	}
 	gwMeta.ETag = data.MD5CurrentHexString() // encrypted ETag
 	gwMeta.Stat.Size = pi.Size
@@ -680,7 +679,7 @@ func getGWContentPath(object string) string {
 }
 
 // Clean-up the stale incomplete encrypted multipart uploads. Should be run in a Go routine.
-func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/gateway/s3/gateway-s3.go

@@ -35,7 +35,7 @@ import (
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 const (
@@ -66,8 +66,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
      {{.Prompt}} {{.HelpName}}
 `
 
@@ -106,11 +108,9 @@ func (g *S3) Name() string {
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
 const (
-	letterIdxBits           = 6                    // 6 bits to represent a letter index
-	letterIdxMask           = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
-	letterIdxMax            = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
-	minioReservedBucket     = "minio"
-	minioReservedBucketPath = minio.SlashSeparator + minioReservedBucket
+	letterIdxBits = 6                    // 6 bits to represent a letter index
+	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
 )
 
 // randString generates random names and prepends them with a known prefix.
@@ -158,32 +158,6 @@ var defaultAWSCredProviders = []credentials.Provider{
 	&credentials.EnvMinio{},
 }
 
-type metricsTransport struct {
-	transport *http.Transport
-	metrics   *minio.Metrics
-}
-
-func (s metricsTransport) RoundTrip(r *http.Request) (*http.Response, error) {
-	isS3Request := func() bool {
-		return !(minio.HasPrefix(r.URL.Path, minioReservedBucketPath) ||
-			minio.HasSuffix(r.URL.Path, ".js") || strings.Contains(r.URL.Path, "favicon.ico") ||
-			strings.Contains(r.URL.Path, ".html"))
-	}
-	if isS3Request() && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
-		s.metrics.IncRequests(r.Method)
-		s.metrics.IncBytesSent(r.ContentLength)
-	}
-	// Make the request to the server.
-	resp, err := s.transport.RoundTrip(r)
-	if err != nil {
-		return nil, err
-	}
-	if isS3Request() && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
-		s.metrics.IncBytesReceived(resp.ContentLength)
-	}
-	return resp, nil
-}
-
 // newS3 - Initializes a new client by auto probing S3 server signature.
 func newS3(urlStr string) (*miniogo.Core, error) {
 	if urlStr == "" {
@@ -237,9 +211,9 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 
 	metrics := minio.NewMetrics()
 
-	t := &metricsTransport{
-		transport: minio.NewCustomHTTPTransport(),
-		metrics:   metrics,
+	t := &minio.MetricsTransport{
+		Transport: minio.NewCustomHTTPTransport(),
+		Metrics:   metrics,
 	}
 
 	// Set custom transport
@@ -300,7 +274,7 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo is not relevant to S3 backend.
-func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo) {
 	si.Backend.Type = minio.BackendGateway
 	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String())
 	return si

cmd/generic-handlers.go

@@ -353,7 +353,7 @@ func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
 // supported amz date formats.
 func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
 	for _, amzDateHeader := range amzDateHeaders {
-		amzDateStr := req.Header.Get(http.CanonicalHeaderKey(amzDateHeader))
+		amzDateStr := req.Header.Get(amzDateHeader)
 		if amzDateStr != "" {
 			return parseAmzDate(amzDateStr)
 		}
@@ -436,12 +436,16 @@ func setIgnoreResourcesHandler(h http.Handler) http.Handler {
 // Checks requests for not implemented Bucket resources
 func ignoreNotImplementedBucketResources(req *http.Request) bool {
 	for name := range req.URL.Query() {
-		// Enable GetBucketACL, GetBucketCors, GetBucketWebsite,
-		// GetBucketAcccelerate, GetBucketRequestPayment,
-		// GetBucketLogging, GetBucketLifecycle,
-		// GetBucketReplication, GetBucketTagging,
-		// GetBucketVersioning, DeleteBucketTagging,
-		// and DeleteBucketWebsite dummy calls specifically.
+		// Enable PutBucketACL, GetBucketACL, GetBucketCors,
+		// GetBucketWebsite, GetBucketAcccelerate,
+		// GetBucketRequestPayment, GetBucketLogging,
+		// GetBucketLifecycle, GetBucketReplication,
+		// GetBucketTagging, GetBucketVersioning,
+		// DeleteBucketTagging, and DeleteBucketWebsite
+		// dummy calls specifically.
+		if name == "acl" && req.Method == http.MethodPut {
+			return false
+		}
 		if ((name == "acl" ||
 			name == "cors" ||
 			name == "website" ||
@@ -457,7 +461,7 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool {
 			return false
 		}
 
-		if notimplementedBucketResourceNames[name] {
+		if notImplementedBucketResourceNames[name] {
 			return true
 		}
 	}
@@ -467,11 +471,11 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool {
 // Checks requests for not implemented Object resources
 func ignoreNotImplementedObjectResources(req *http.Request) bool {
 	for name := range req.URL.Query() {
-		// Enable GetObjectACL and GetObjectTagging dummy calls specifically.
-		if (name == "acl" || name == "tagging") && req.Method == http.MethodGet {
+		// Enable Get/PutObjectACL dummy call specifically.
+		if name == "acl" && (req.Method == http.MethodGet || req.Method == http.MethodPut) {
 			return false
 		}
-		if notimplementedObjectResourceNames[name] {
+		if notImplementedObjectResourceNames[name] {
 			return true
 		}
 	}
@@ -479,9 +483,8 @@ func ignoreNotImplementedObjectResources(req *http.Request) bool {
 }
 
 // List of not implemented bucket queries
-var notimplementedBucketResourceNames = map[string]bool{
+var notImplementedBucketResourceNames = map[string]bool{
 	"accelerate":     true,
-	"acl":            true,
 	"cors":           true,
 	"inventory":      true,
 	"logging":        true,
@@ -494,10 +497,8 @@ var notimplementedBucketResourceNames = map[string]bool{
 }
 
 // List of not implemented object queries
-var notimplementedObjectResourceNames = map[string]bool{
-	"acl":     true,
+var notImplementedObjectResourceNames = map[string]bool{
 	"restore": true,
-	"tagging": true,
 	"torrent": true,
 }
 
@@ -537,9 +538,8 @@ func setHTTPStatsHandler(h http.Handler) http.Handler {
 func (h httpStatsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	isS3Request := !strings.HasPrefix(r.URL.Path, minioReservedBucketPath)
 	// record s3 connection stats.
-	recordRequest := &recordTrafficRequest{ReadCloser: r.Body, isS3Request: isS3Request}
-	r.Body = recordRequest
-	recordResponse := &recordTrafficResponse{w, isS3Request}
+	r.Body = &recordTrafficRequest{ReadCloser: r.Body, isS3Request: isS3Request}
+	recordResponse := &recordTrafficResponse{ResponseWriter: w, isS3Request: isS3Request}
 	// Execute the request
 	h.handler.ServeHTTP(recordResponse, r)
 }
@@ -632,11 +632,11 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		switch r.Method {
 		case http.MethodPut:
 			if getRequestAuthType(r) == authTypeJWT {
-				bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/upload"))
+				bucket, _ = path2BucketObjectWithBasePath(minioReservedBucketPath+"/upload", r.URL.Path)
 			}
 		case http.MethodGet:
 			if t := r.URL.Query().Get("token"); t != "" {
-				bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/download"))
+				bucket, _ = path2BucketObjectWithBasePath(minioReservedBucketPath+"/download", r.URL.Path)
 			} else if getRequestAuthType(r) != authTypeJWT && !strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
 				bucket, _ = request2BucketObjectName(r)
 			}
@@ -688,7 +688,7 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	// requests have target bucket and object in URI and source details are in
 	// header fields
 	if r.Method == http.MethodPut && r.Header.Get(xhttp.AmzCopySource) != "" {
-		bucket, object = urlPath2BucketObjectName(r.Header.Get(xhttp.AmzCopySource))
+		bucket, object = path2BucketObject(r.Header.Get(xhttp.AmzCopySource))
 		if bucket == "" || object == "" {
 			f.handler.ServeHTTP(w, r)
 			return

cmd/global-heal.go

@@ -29,7 +29,6 @@ import (
 const (
 	bgHealingUUID = "0000-0000-0000-0000"
 	leaderTick    = time.Hour
-	healTick      = time.Hour
 	healInterval  = 30 * 24 * time.Hour
 )
 
@@ -40,7 +39,7 @@ var leaderLockTimeout = newDynamicTimeout(time.Minute, time.Minute)
 func newBgHealSequence(numDisks int) *healSequence {
 
 	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
@@ -75,6 +74,7 @@ func getLocalBackgroundHealStatus() madmin.BgHealState {
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.scannedItemsCount,
 		LastHealActivity:  bgSeq.lastHealActivity,
+		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
 	}
 }
 
@@ -112,6 +112,19 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
 	return nil
 }
 
+// Returns the duration to the next background healing round
+func durationToNextHealRound(lastHeal time.Time) time.Duration {
+	if lastHeal.IsZero() {
+		lastHeal = globalBootTime
+	}
+
+	d := lastHeal.Add(healInterval).Sub(UTCNow())
+	if d < 0 {
+		return time.Second
+	}
+	return d
+}
+
 // Healing leader will take the charge of healing all erasure sets
 func execLeaderTasks(z *xlZones) {
 	ctx := context.Background()
@@ -132,10 +145,7 @@ func execLeaderTasks(z *xlZones) {
 
 	lastScanTime := time.Now() // So that we don't heal immediately, but after one month.
 	for {
-		if time.Since(lastScanTime) < healInterval {
-			time.Sleep(healTick)
-			continue
-		}
+		time.Sleep(durationToNextHealRound(lastScanTime))
 		for _, zone := range z.zones {
 			// Heal set by set
 			for i, set := range zone.sets {

cmd/globals.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/pubsub"
@@ -94,6 +95,9 @@ const (
 
 	// Limit of location constraint XML for unauthenticted PUT bucket operations.
 	maxLocationConstraintSize = 3 * humanize.MiByte
+
+	// Maximum size of default bucket encryption configuration allowed
+	maxBucketSSEConfigSize = 1 * humanize.MiByte
 )
 
 var globalCLIContext = struct {
@@ -140,10 +144,14 @@ var (
 
 	globalNotificationSys  *NotificationSys
 	globalConfigTargetList *event.TargetList
-	globalPolicySys        *PolicySys
-	globalIAMSys           *IAMSys
+	// globalEnvTargetList has list of targets configured via env.
+	globalEnvTargetList *event.TargetList
 
-	globalLifecycleSys *LifecycleSys
+	globalPolicySys *PolicySys
+	globalIAMSys    *IAMSys
+
+	globalLifecycleSys       *LifecycleSys
+	globalBucketSSEConfigSys *BucketSSEConfigSys
 
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -180,11 +188,14 @@ var (
 	// Global HTTP request statisitics
 	globalHTTPStats = newHTTPStats()
 
-	// Time when object layer was initialized on start up.
-	globalBootTime time.Time
+	// Time when the server is started
+	globalBootTime = UTCNow()
 
 	globalActiveCred auth.Credentials
 
+	// Hold the old server credentials passed by the environment
+	globalOldCred auth.Credentials
+
 	// Indicates if config is to be encrypted
 	globalConfigEncrypted bool
 
@@ -201,7 +212,7 @@ var (
 	// Is worm enabled
 	globalWORMEnabled bool
 
-	globalBucketObjectLockConfig = newBucketObjectLockConfig()
+	globalBucketObjectLockConfig = objectlock.NewBucketObjectLockConfig()
 
 	// Disk cache drives
 	globalCacheConfig cache.Config

cmd/handler-utils.go

@@ -32,10 +32,16 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
 )
 
+const (
+	copyDirective    = "COPY"
+	replaceDirective = "REPLACE"
+)
+
 // Parses location constraint from the incoming reader.
 func parseLocationConstraint(r *http.Request) (location string, s3Error APIErrorCode) {
 	// If the request has no body with content-length set to 0,
@@ -69,47 +75,27 @@ var supportedHeaders = []string{
 	"content-encoding",
 	"content-disposition",
 	xhttp.AmzStorageClass,
+	xhttp.AmzObjectTagging,
 	"expires",
 	// Add more supported headers here.
 }
 
-// isMetadataDirectiveValid - check if metadata-directive is valid.
-func isMetadataDirectiveValid(h http.Header) bool {
-	_, ok := h[http.CanonicalHeaderKey(xhttp.AmzMetadataDirective)]
-	if ok {
-		// Check atleast set metadata-directive is valid.
-		return (isMetadataCopy(h) || isMetadataReplace(h))
-	}
-	// By default if x-amz-metadata-directive is not we
+// isDirectiveValid - check if tagging-directive is valid.
+func isDirectiveValid(v string) bool {
+	// Check if set metadata-directive is valid.
+	return isDirectiveCopy(v) || isDirectiveReplace(v)
+}
+
+// Check if the directive COPY is requested.
+func isDirectiveCopy(value string) bool {
+	// By default if directive is not set we
 	// treat it as 'COPY' this function returns true.
-	return true
+	return value == copyDirective || value == ""
 }
 
-// Check if the metadata COPY is requested.
-func isMetadataCopy(h http.Header) bool {
-	return h.Get(xhttp.AmzMetadataDirective) == "COPY"
-}
-
-// Check if the metadata REPLACE is requested.
-func isMetadataReplace(h http.Header) bool {
-	return h.Get(xhttp.AmzMetadataDirective) == "REPLACE"
-}
-
-// Splits an incoming path into bucket and object components.
-func path2BucketAndObject(path string) (bucket, object string) {
-	// Skip the first element if it is '/', split the rest.
-	path = strings.TrimPrefix(path, SlashSeparator)
-	pathComponents := strings.SplitN(path, SlashSeparator, 2)
-
-	// Save the bucket and object extracted from path.
-	switch len(pathComponents) {
-	case 1:
-		bucket = pathComponents[0]
-	case 2:
-		bucket = pathComponents[0]
-		object = pathComponents[1]
-	}
-	return bucket, object
+// Check if the directive REPLACE is requested.
+func isDirectiveReplace(value string) bool {
+	return value == replaceDirective
 }
 
 // userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys.
@@ -174,6 +160,26 @@ func extractMetadataFromMap(ctx context.Context, v map[string][]string, m map[st
 	return nil
 }
 
+// extractTags extracts tag key and value from given http header. It then
+// - Parses the input format X-Amz-Tagging:"Key1=Value1&Key2=Value2" into a map[string]string
+// with entries in the format X-Amg-Tag-Key1:Value1, X-Amz-Tag-Key2:Value2
+// - Validates the tags
+// - Returns the Tag in original string format "Key1=Value1&Key2=Value2"
+func extractTags(ctx context.Context, tags string) (string, error) {
+	// Check if the metadata has tagging related header
+	if tags != "" {
+		tagging, err := tagging.FromString(tags)
+		if err != nil {
+			return "", err
+		}
+		if err := tagging.Validate(); err != nil {
+			return "", err
+		}
+		return tagging.String(), nil
+	}
+	return "", nil
+}
+
 // The Query string for the redirect URL the client is
 // redirected on successful upload.
 func getRedirectPostRawQuery(objInfo ObjectInfo) string {
@@ -195,7 +201,9 @@ func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
 		if owner {
 			return globalActiveCred
 		}
-		cred, _ = globalIAMSys.GetUser(claims.AccessKey())
+		if claims != nil {
+			cred, _ = globalIAMSys.GetUser(claims.AccessKey)
+		}
 	}
 	return cred
 }
@@ -354,14 +362,16 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 
 		isS3Request := !strings.HasPrefix(r.URL.Path, minioReservedBucketPath)
-		apiStatsWriter := &recordAPIStats{w, UTCNow(), false, 0, isS3Request}
 
 		// Time start before the call is about to start.
 		tBefore := UTCNow()
 
+		apiStatsWriter := &recordAPIStats{ResponseWriter: w, TTFB: tBefore, isS3Request: isS3Request}
+
 		if isS3Request {
 			globalHTTPStats.currentS3Requests.Inc(api)
 		}
+
 		// Execute the request
 		f.ServeHTTP(apiStatsWriter, r)
 

cmd/healthcheck-handler.go

@@ -60,7 +60,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if !globalIsXL && !globalIsDistXL {
-		s := objLayer.StorageInfo(ctx)
+		s := objLayer.StorageInfo(ctx, false)
 		if s.Backend.Type == BackendGateway {
 			if !s.Backend.GatewayOnline {
 				writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)

cmd/http-stats.go

@@ -95,36 +95,32 @@ func newConnStats() *ConnStats {
 // HTTPAPIStats holds statistics information about
 // a given API in the requests.
 type HTTPAPIStats struct {
-	APIStats map[string]int
+	apiStats map[string]int
 	sync.RWMutex
 }
 
 // Inc increments the api stats counter.
 func (stats *HTTPAPIStats) Inc(api string) {
-	stats.Lock()
-	defer stats.Unlock()
 	if stats == nil {
 		return
 	}
-	if stats.APIStats == nil {
-		stats.APIStats = make(map[string]int)
+	stats.Lock()
+	defer stats.Unlock()
+	if stats.apiStats == nil {
+		stats.apiStats = make(map[string]int)
 	}
-	if _, ok := stats.APIStats[api]; ok {
-		stats.APIStats[api]++
-		return
-	}
-	stats.APIStats[api] = 1
+	stats.apiStats[api]++
 }
 
 // Dec increments the api stats counter.
 func (stats *HTTPAPIStats) Dec(api string) {
-	stats.Lock()
-	defer stats.Unlock()
 	if stats == nil {
 		return
 	}
-	if val, ok := stats.APIStats[api]; ok && val > 0 {
-		stats.APIStats[api]--
+	stats.Lock()
+	defer stats.Unlock()
+	if val, ok := stats.apiStats[api]; ok && val > 0 {
+		stats.apiStats[api]--
 	}
 }
 
@@ -132,7 +128,11 @@ func (stats *HTTPAPIStats) Dec(api string) {
 func (stats *HTTPAPIStats) Load() map[string]int {
 	stats.Lock()
 	defer stats.Unlock()
-	return stats.APIStats
+	var apiStats = make(map[string]int, len(stats.apiStats))
+	for k, v := range stats.apiStats {
+		apiStats[k] = v
+	}
+	return apiStats
 }
 
 // HTTPStats holds statistics information about

cmd/http-tracer.go

@@ -83,14 +83,15 @@ func getOpName(name string) (op string) {
 	op = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
 	op = strings.TrimSuffix(op, "Handler-fm")
 	op = strings.Replace(op, "objectAPIHandlers", "s3", 1)
-	op = strings.Replace(op, "webAPIHandlers", "s3", 1)
+	op = strings.Replace(op, "webAPIHandlers", "webui", 1)
 	op = strings.Replace(op, "adminAPIHandlers", "admin", 1)
 	op = strings.Replace(op, "(*storageRESTServer)", "internal", 1)
 	op = strings.Replace(op, "(*peerRESTServer)", "internal", 1)
 	op = strings.Replace(op, "(*lockRESTServer)", "internal", 1)
-	op = strings.Replace(op, "stsAPIHandlers", "sts", 1)
+	op = strings.Replace(op, "(*stsAPIHandlers)", "sts", 1)
 	op = strings.Replace(op, "LivenessCheckHandler", "healthcheck", 1)
 	op = strings.Replace(op, "ReadinessCheckHandler", "healthcheck", 1)
+	op = strings.Replace(op, "-fm", "", 1)
 	return op
 }
 
@@ -121,6 +122,10 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 		t.NodeName = host
 	}
 
+	rw := logger.NewResponseWriter(w)
+	rw.LogBody = logBody
+	f(rw, r)
+
 	rq := trace.RequestInfo{
 		Time:     time.Now().UTC(),
 		Method:   r.Method,
@@ -130,10 +135,6 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 		Headers:  reqHeaders,
 		Body:     reqBodyRecorder.Data(),
 	}
-	rw := logger.NewResponseWriter(w)
-	rw.LogBody = logBody
-	f(rw, r)
-
 	rs := trace.ResponseInfo{
 		Time:       time.Now().UTC(),
 		Headers:    rw.Header().Clone(),

cmd/http-traffic-recorder.go

@@ -41,23 +41,13 @@ func (r *recordTrafficRequest) Read(p []byte) (n int, err error) {
 // Records the outgoing bytes through the responseWriter.
 type recordTrafficResponse struct {
 	// wrapper for underlying http.ResponseWriter.
-	writer      http.ResponseWriter
+	http.ResponseWriter
 	isS3Request bool
 }
 
-// Calls the underlying WriteHeader.
-func (r *recordTrafficResponse) WriteHeader(i int) {
-	r.writer.WriteHeader(i)
-}
-
-// Calls the underlying Header.
-func (r *recordTrafficResponse) Header() http.Header {
-	return r.writer.Header()
-}
-
 // Records the output bytes
 func (r *recordTrafficResponse) Write(p []byte) (n int, err error) {
-	n, err = r.writer.Write(p)
+	n, err = r.ResponseWriter.Write(p)
 	globalConnStats.incOutputBytes(n)
 	// Check if it is s3 request
 	if r.isS3Request {
@@ -68,13 +58,12 @@ func (r *recordTrafficResponse) Write(p []byte) (n int, err error) {
 
 // Calls the underlying Flush.
 func (r *recordTrafficResponse) Flush() {
-	r.writer.(http.Flusher).Flush()
+	r.ResponseWriter.(http.Flusher).Flush()
 }
 
 // Records the outgoing bytes through the responseWriter.
 type recordAPIStats struct {
-	// wrapper for underlying http.ResponseWriter.
-	writer         http.ResponseWriter
+	http.ResponseWriter
 	TTFB           time.Time // TimeToFirstByte.
 	firstByteRead  bool
 	respStatusCode int
@@ -84,12 +73,7 @@ type recordAPIStats struct {
 // Calls the underlying WriteHeader.
 func (r *recordAPIStats) WriteHeader(i int) {
 	r.respStatusCode = i
-	r.writer.WriteHeader(i)
-}
-
-// Calls the underlying Header.
-func (r *recordAPIStats) Header() http.Header {
-	return r.writer.Header()
+	r.ResponseWriter.WriteHeader(i)
 }
 
 // Records the TTFB on the first byte write.
@@ -98,10 +82,10 @@ func (r *recordAPIStats) Write(p []byte) (n int, err error) {
 		r.TTFB = UTCNow()
 		r.firstByteRead = true
 	}
-	return r.writer.Write(p)
+	return r.ResponseWriter.Write(p)
 }
 
 // Calls the underlying Flush.
 func (r *recordAPIStats) Flush() {
-	r.writer.(http.Flusher).Flush()
+	r.ResponseWriter.(http.Flusher).Flush()
 }

cmd/http/headers.go

@@ -56,6 +56,11 @@ const (
 	// S3 storage class
 	AmzStorageClass = "x-amz-storage-class"
 
+	// S3 object tagging
+	AmzObjectTagging = "X-Amz-Tagging"
+	AmzTagCount      = "X-Amz-Tag-Count"
+	AmzTagDirective  = "X-Amz-Tagging-Directive"
+
 	// S3 extensions
 	AmzCopySourceIfModifiedSince   = "x-amz-copy-source-if-modified-since"
 	AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since"
@@ -72,6 +77,12 @@ const (
 	AmzObjectLockLegalHold        = "X-Amz-Object-Lock-Legal-Hold"
 	AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention"
 
+	// Multipart parts count
+	AmzMpPartsCount = "x-amz-mp-parts-count"
+
+	// Dummy putBucketACL
+	AmzACL = "x-amz-acl"
+
 	// Signature V4 related contants.
 	AmzContentSha256        = "X-Amz-Content-Sha256"
 	AmzDate                 = "X-Amz-Date"

cmd/http/server.go

@@ -87,15 +87,19 @@ func (srv *Server) Start() (err error) {
 	// Wrap given handler to do additional
 	// * return 503 (service unavailable) if the server in shutdown.
 	wrappedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		atomic.AddInt32(&srv.requestCount, 1)
-		defer atomic.AddInt32(&srv.requestCount, -1)
-
-		// If server is in shutdown, return 503 (service unavailable)
+		// If server is in shutdown.
 		if atomic.LoadUint32(&srv.inShutdown) != 0 {
-			w.WriteHeader(http.StatusServiceUnavailable)
+			// To indicate disable keep-alives
+			w.Header().Set("Connection", "close")
+			w.WriteHeader(http.StatusForbidden)
+			w.Write([]byte(http.ErrServerClosed.Error()))
+			w.(http.Flusher).Flush()
 			return
 		}
 
+		atomic.AddInt32(&srv.requestCount, 1)
+		defer atomic.AddInt32(&srv.requestCount, -1)
+
 		// Handle request using passed handler.
 		handler.ServeHTTP(w, r)
 	})
@@ -117,13 +121,13 @@ func (srv *Server) Shutdown() error {
 	srv.listenerMutex.Lock()
 	if srv.listener == nil {
 		srv.listenerMutex.Unlock()
-		return errors.New("server not initialized")
+		return http.ErrServerClosed
 	}
 	srv.listenerMutex.Unlock()
 
 	if atomic.AddUint32(&srv.inShutdown, 1) > 1 {
 		// shutdown in progress
-		return errors.New("http server already in shutdown")
+		return http.ErrServerClosed
 	}
 
 	// Close underneath HTTP listener.