non-empty dirs should not be listed as objects (#9129)

This PR fixes a regression introduced in a1c7c9ea73

.jshintrc

@@ -1,4 +0,0 @@
-{
-  "asi": true,
-  "esnext": true   
-}

.mention-bot

@@ -1,4 +0,0 @@
-{
-    "numFilesToCheck": 10,
-    "requiredOrgs": ["minio"]
-}

.travis.yml

@@ -32,8 +32,7 @@ matrix:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
-        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
-        - make verifiers
+        - make test-race
         - make crosscompile
         - make verify
         - cd browser && npm install && npm run test && cd ..

Dockerfile

@@ -5,23 +5,24 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 ENV GOPATH /go
 ENV CGO_ENABLED 0
 ENV GO111MODULE on
-ENV GOPROXY https://proxy.golang.org
 
 RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.9
+FROM alpine:3.10
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
+COPY CREDITS /third_party/
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \

Dockerfile.arm.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm32v7/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.arm64.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm64v8/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.dev

@@ -1,14 +1,16 @@
-FROM alpine:3.9
+FROM alpine:3.10
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.release

@@ -8,16 +8,18 @@ RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio
 
-FROM alpine:3.9
+FROM alpine:3.10
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.simpleci

@@ -34,11 +34,13 @@ USER ci
 RUN make
 RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
 RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
-RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
-RUN make verifiers
+RUN make test-race
 RUN make crosscompile
 RUN make verify
 
+## -- add healing tests
+RUN make verify-healing
+
 #-------------------------------------------------------------
 # Stage 2: Test Frontend
 #-------------------------------------------------------------

Makefile

@@ -64,15 +64,21 @@ test: verifiers build
 	@echo "Running unit tests"
 	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
+test-race: verifiers build
+	@echo "Running unit tests under -race"
+	@(env bash $(PWD)/buildscripts/race.sh)
+
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
-coverage: build
-	@echo "Running all coverage for minio"
-	@(env bash $(PWD)/buildscripts/go-coverage.sh)
+# Verify healing of disks with minio binary
+verify-healing:
+	@echo "Verify healing build with race"
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
 # Builds minio locally.
 build: checks

NOTICE

@@ -1,4 +1,4 @@
-MinIO Cloud Storage, (C) 2014,2015 MinIO, Inc.
+MinIO Cloud Storage, (C) 2014-2020 MinIO, Inc.
 
 This product includes software developed at MinIO, Inc.
 (https://min.io/).
@@ -6,4 +6,4 @@ This product includes software developed at MinIO, Inc.
 The MinIO project contains unmodified/modified subcomponents too with
 separate copyright notices and license terms. Your use of the source
 code for these subcomponents is subject to the terms and conditions
-of the following licenses.
+of Apache License Version 2.0

README.md

@@ -21,7 +21,7 @@ docker run -p 9000:9000 minio/minio:edge server /data
 > NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
-### Homebrew
+### Homebrew (recommended)
 Install minio packages using [Homebrew](http://brew.sh/)
 ```sh
 brew install minio/stable/minio
@@ -74,7 +74,7 @@ minio.exe server D:\Photos
 
 ## FreeBSD
 ### Port
-Install minio packages using [pkg](https://github.com/freebsd/pkg)
+Install minio packages using [pkg](https://github.com/freebsd/pkg), MinIO doesn't officially build FreeBSD binaries but is maintained by FreeBSD upstream [here](https://www.freshports.org/www/minio).
 
 ```sh
 pkg install minio

SECURITY.md

@@ -8,12 +8,12 @@ Whenever there is a security update you just need to upgrade to the latest versi
 ## Reporting a Vulnerability
 
 All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories)
-should be reported by email to security@min.io. Your email will be acknowledged within 48 hours, 
-and you'll receive a more detailed response to your email within 72 hours indicating the next steps 
-in handling your report. 
+should be reported by email to security@min.io. Your email will be acknowledged within 48 hours,
+and you'll receive a more detailed response to your email within 72 hours indicating the next steps
+in handling your report.
 
-Please, provide a detailed explanation of the issue. In particular, outline the type of the security 
-issue (DoS, authentication bypass, information disclouse, ...) and the assumptions you're making (e.g. do 
+Please, provide a detailed explanation of the issue. In particular, outline the type of the security
+issue (DoS, authentication bypass, information disclose, ...) and the assumptions you're making (e.g. do
 you need access credentials for a successful exploit).
 
 If you have not received a reply to your email within 48 hours or you have not heard from the security team
@@ -28,14 +28,14 @@ MinIO uses the following disclosure process:
 
 1. Once the security report is received one member of the security team tries to verify and reproduce
    the issue and determines the impact it has.
-2. A member of the security team will respond and either confrim or reject the security report.
-   If the report is rejected the repsonse explains why.
+2. A member of the security team will respond and either confirm or reject the security report.
+   If the report is rejected the response explains why.
 3. Code is audited to find any potential similar problems.
-4. Fixes are prepared for the latest release. 
+4. Fixes are prepared for the latest release.
 5. On the date that the fixes are applied a security advisory will be published on https://blog.min.io.
    Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing
    the security issue. By default MinIO will **not** publish this information to protect your privacy.
-   
+
 This process can take some time, especially when coordination is required with maintainers of other projects.
-Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we 
-follow the process described above to ensure that disclosures are handled consistently. 
+Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we
+follow the process described above to ensure that disclosures are handled consistently.

browser/app/js/browser/Login.js

@@ -148,7 +148,8 @@ export class Login extends React.Component {
                   <OpenIDLoginButton
                     className="btn openid-btn"
                     clientId={this.state.clientId}
-                    authorizationEndpoint={this.state.discoveryDoc.authorization_endpoint}
+                    authEp={this.state.discoveryDoc.authorization_endpoint}
+                    authScopes={this.state.discoveryDoc.scopes_supported}
                   >
                     Log in with OpenID
                   </OpenIDLoginButton>

browser/app/js/browser/OpenIDLogin.js

@@ -66,6 +66,7 @@ export class OpenIDLogin extends React.Component {
 
       const authURL = buildOpenIDAuthURL(
         this.state.discoveryDoc.authorization_endpoint,
+        this.state.discoveryDoc.scopes_supported,
         redirectURI,
         this.state.clientID,
         nonce

browser/app/js/browser/OpenIDLoginButton.js

@@ -27,7 +27,7 @@ export class OpenIDLoginButton extends React.Component {
 
   handleClick(event) {
     event.stopPropagation()
-    const { authorizationEndpoint, clientId } = this.props
+    const { authEp, authScopes, clientId } = this.props
 
     let redirectURI = window.location.href.split("#")[0]
     if (redirectURI.endsWith('/')) {
@@ -40,7 +40,7 @@ export class OpenIDLoginButton extends React.Component {
     const nonce = getRandomString(16)
     storage.setItem(OPEN_ID_NONCE_KEY, nonce)
 
-    const authURL = buildOpenIDAuthURL(authorizationEndpoint, redirectURI, clientId, nonce)
+    const authURL = buildOpenIDAuthURL(authEp, authScopes, redirectURI, clientId, nonce)
     window.location = authURL
   }
 

browser/app/js/browser/utils.js

@@ -16,14 +16,13 @@
 
 export const OPEN_ID_NONCE_KEY = 'openIDKey'
 
-export const buildOpenIDAuthURL = (authorizationEndpoint, redirectURI, clientID, nonce) => {
+export const buildOpenIDAuthURL = (authEp, authScopes, redirectURI, clientID, nonce) => {
   const params = new URLSearchParams()
   params.set("response_type", "id_token")
-  params.set("scope", "openid")
+  params.set("scope", authScopes.join(" "))
   params.set("client_id", clientID)
   params.set("redirect_uri", redirectURI)
   params.set("nonce", nonce)
 
-  return `${authorizationEndpoint}?${params.toString()}`
+  return `${authEp}?${params.toString()}`
 }
-

buildscripts/benchcmp.sh

@@ -1,67 +0,0 @@
-#!/bin/bash          
-
-# usage: ./benchcmp.sh <commit-sha1> <commit-sha2>
-# Exit on any non zero return value on execution of a command. 
-set -e 
-
-# path of benchcmp. 
-benchcmp=${GOPATH}/bin/benchcmp
-
-# function which runs the benchmark comparison. 
-RunBenchCmp () {
-	# Path for storing output of benchmark at commit 1. 
-	commit1Bench=/tmp/minio-$1.bench
-	# Path for storing output of benchmark at commit 2. 
-	commit2Bench=/tmp/minio-$2.bench
-	# switch to commit $1. 
-	git checkout $1
-	# Check if the benchmark results for given commit 1 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit1Bench  ]]
-	then
-
-		echo "Running benchmarks at $1"
-		go test -run=NONE -bench=. | tee $commit1Bench 
-	fi
-	# get back to the commit from which it was started. 
-	git checkout -
-	echo "Checking into commit $2"
-	# switch to commit $2 
-	git checkout $2
-	# Check if the benchmark results for given commit 2 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit2Bench  ]]
-	then
-	# Running benchmarks at $2.
-		echo "Running benchmarks at $2"
-		go test -run=NONE -bench=. | tee $commit2Bench 
-	fi
-	
-	# get back to the commit from which it was started. 
-	git checkout - 
-	# Comparing the benchmarks.
-	echo "Running benchmark comparison between $1 and $2 ..."
-	$benchcmp $commit1Bench $commit2Bench 
-	echo "Done."
-}
-
-# check if 2 commit SHA's of snapshots of code for which benchmp has to be done is provided. 
-if [ ! $# -eq 2 ]
-then
-	# exit if commit SHA's are not provided. 
-	echo $#
-	echo "Need Commit SHA's of 2 snapshots to be supplied to run benchmark comparison."
-	exit 1
-fi
-
-# check if benchcmp exists. 
-if [[ -x "$benchcmp" ]]
-then
-	RunBenchCmp $1 $2
-else
-	# install benchcmp if doesnt't exist. 
-	echo "fetching Benchcmp..."
-	go get -u golang.org/x/tools/cmd/benchcmp
-	echo "Done."
-	RunBenchCmp $1 $2
-fi

buildscripts/cross-compile.sh

@@ -1,14 +1,15 @@
 #!/bin/bash
 
+set -e
 # Enable tracing if set.
-[ -n "$BASH_XTRACEFD" ] && set -ex
+[ -n "$BASH_XTRACEFD" ] && set -x
 
 function _init() {
     ## All binaries are static make sure to disable CGO.
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
 }
 
 function _build() {

buildscripts/gateway-tests.sh

@@ -46,10 +46,7 @@ function main()
     gw_pid="$(start_minio_gateway_s3)"
 
     SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
-                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
-                   aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
-                   healthcheck mc minio-dotnet minio-go minio-java minio-py \
-                   s3cmd security
+                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh
     rv=$?
 
     kill "$sr_pid"

buildscripts/go-coverage.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/race.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+for d in $(go list ./... | grep -v browser); do
+    CGO_ENABLED=1 go test -v -race --timeout 20m "$d"
+done

buildscripts/verify-healing.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+#
+# MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+function start_minio_3_node() {
+    declare -a minio_pids
+    export MINIO_ACCESS_KEY=minio
+    export MINIO_SECRET_KEY=minio123
+
+    for i in $(seq 1 3); do
+        ARGS+=("http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[8000+$i]${WORK_DIR}/$i/6/")
+    done
+
+    "${MINIO[@]}" --address ":8001" ${ARGS[@]} > "${WORK_DIR}/dist-minio-8001.log" 2>&1 &
+    minio_pids[0]=$!
+
+    "${MINIO[@]}" --address ":8002" ${ARGS[@]} > "${WORK_DIR}/dist-minio-8002.log" 2>&1 &
+    minio_pids[1]=$!
+
+    "${MINIO[@]}" --address ":8003" ${ARGS[@]} > "${WORK_DIR}/dist-minio-8003.log" 2>&1 &
+    minio_pids[2]=$!
+
+    sleep "$1"
+    echo "${minio_pids[@]}"
+}
+
+
+function check_online() {
+    for i in $(seq 1 3); do
+        if grep -q 'Server switching to safe mode' ${WORK_DIR}/dist-minio-$[8000+$i].log; then
+            echo "1"
+        fi
+    done
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+function __init__()
+{
+    echo "Initializing environment"
+    mkdir -p "$WORK_DIR"
+    mkdir -p "$MINIO_CONFIG_DIR"
+
+    ## version is purposefully set to '3' for minio to migrate configuration file
+    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+}
+
+function perform_test() {
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        if ! kill "$pid"; then
+            for i in $(seq 1 3); do
+                echo "server$i log:"
+                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
+            done
+            echo "FAILED"
+            purge "$WORK_DIR"
+            exit 1
+        fi
+        # forcibly killing, to proceed further properly.
+        kill -9 "$pid"
+    done
+
+    echo "Testing Distributed Erasure setup healing of drives"
+    echo "Remove the contents of the disks belonging to '${1}' erasure set"
+
+    rm -rf ${WORK_DIR}/${1}/*/
+
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        if ! kill "$pid"; then
+            for i in $(seq 1 3); do
+                echo "server$i log:"
+                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
+            done
+            echo "FAILED"
+            purge "$WORK_DIR"
+            exit 1
+        fi
+        # forcibly killing, to proceed further properly.
+        # if the previous kill is taking time.
+        kill -9 "$pid"
+    done
+
+    rv=$(check_online)
+    if [ "$rv" == "1" ]; then
+        for pid in "${minio_pids[@]}"; do
+            kill -9 "$pid"
+        done
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+}
+
+function main()
+{
+    perform_test "2"
+    perform_test "1"
+    perform_test "3"
+}
+
+( __init__ "$@" && main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

cmd/acl-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,11 +18,14 @@ package cmd
 
 import (
 	"encoding/xml"
+	"io"
 	"net/http"
+	"net/url"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Data types used for returning dummy access control
@@ -50,6 +53,71 @@ type accessControlPolicy struct {
 	} `xml:"AccessControlList"`
 }
 
+// PutBucketACLHandler - PUT Bucket ACL
+// -----------------
+// This operation uses the ACL subresource
+// to set ACL for a bucket, this is a dummy call
+// only responds success if the ACL is private.
+func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketACL")
+
+	defer logger.AuditLog(w, r, "PutBucketACL", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow putBucketACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Before proceeding validate if bucket exists.
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	aclHeader := r.Header.Get(xhttp.AmzACL)
+	if aclHeader == "" {
+		acl := &accessControlPolicy{}
+		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
+			if err == io.EOF {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
+					r.URL, guessIsBrowserReq(r))
+				return
+			}
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if len(acl.AccessControlList.Grants) == 0 {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if aclHeader != "" && aclHeader != "private" {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetBucketACLHandler - GET Bucket ACL
 // -----------------
 // This operation uses the ACL
@@ -99,6 +167,71 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	w.(http.Flusher).Flush()
 }
 
+// PutObjectACLHandler - PUT Object ACL
+// -----------------
+// This operation uses the ACL subresource
+// to set ACL for a bucket, this is a dummy call
+// only responds success if the ACL is private.
+func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutObjectACL")
+
+	defer logger.AuditLog(w, r, "PutObjectACL", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object, err := url.PathUnescape(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow putObjectACL if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Before proceeding validate if object exists.
+	_, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	aclHeader := r.Header.Get(xhttp.AmzACL)
+	if aclHeader == "" {
+		acl := &accessControlPolicy{}
+		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if len(acl.AccessControlList.Grants) == 0 {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+
+		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
+	if aclHeader != "" && aclHeader != "private" {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetObjectACLHandler - GET Object ACL
 // -----------------
 // This operation uses the ACL
@@ -110,7 +243,11 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	object := vars["object"]
+	object, err := url.PathUnescape(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -126,7 +263,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if object exists.
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	_, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/admin-handlers-users.go

@@ -484,7 +484,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 

cmd/admin-handlers.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"crypto/subtle"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -140,9 +141,9 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		if runtime.GOOS == "windows" {
-			u.Path = path.Dir(u.Path) + "minio.exe"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio.exe"
 		} else {
-			u.Path = path.Dir(u.Path) + "minio"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio"
 		}
 
 		updateURL = u.String()
@@ -270,12 +271,12 @@ type ServerInfo struct {
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
@@ -295,26 +296,87 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	dataUsageInfoJSON, err := json.Marshal(dataUsageInfo)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
+func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AccountingUsageInfo")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AccountingUsageInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var accountingUsageInfo = make(map[string]madmin.BucketAccountingUsage)
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		// writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	users, err := globalIAMSys.ListUsers()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Load the latest calculated data usage
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	// Calculate for each bucket, which users are allowed to access to it
+	for _, bucket := range buckets {
+		bucketUsageInfo := madmin.BucketAccountingUsage{}
+
+		// Fetch the data usage of the current bucket
+		if !dataUsageInfo.LastUpdate.IsZero() && dataUsageInfo.BucketsSizes != nil {
+			bucketUsageInfo.Size = dataUsageInfo.BucketsSizes[bucket.Name]
+		}
+
+		for user := range users {
+			rd, wr, custom := globalIAMSys.GetAccountAccess(user, bucket.Name)
+			if rd || wr || custom {
+				bucketUsageInfo.AccessList = append(bucketUsageInfo.AccessList, madmin.AccountAccess{
+					AccountName: user,
+					Read:        rd,
+					Write:       wr,
+					Custom:      custom,
+				})
+			}
+		}
+
+		accountingUsageInfo[bucket.Name] = bucketUsageInfo
+	}
+
+	usageInfoJSON, err := json.Marshal(accountingUsageInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, usageInfoJSON)
+}
+
 // ServerCPULoadInfo holds informantion about cpu utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
@@ -349,7 +411,7 @@ type ServerNetReadPerfInfo struct {
 func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PerfInfo")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.PerfInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -520,7 +582,7 @@ type PeerLocks struct {
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -565,36 +627,50 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	vars := mux.Vars(r)
-	profiler := vars["profilerType"]
-
+	profiles := strings.Split(vars["profilerType"], ",")
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Start profiling on remote servers.
-	hostErrs := globalNotificationSys.StartProfiling(profiler)
+	globalProfilerMu.Lock()
+	defer globalProfilerMu.Unlock()
 
-	// Start profiling locally as well.
-	{
-		if globalProfiler != nil {
-			globalProfiler.Stop()
+	if globalProfiler == nil {
+		globalProfiler = make(map[string]minioProfiler, 10)
+	}
+
+	// Stop profiler of all types if already running
+	for k, v := range globalProfiler {
+		for _, p := range profiles {
+			if p == k {
+				v.Stop()
+				delete(globalProfiler, k)
+			}
 		}
-		prof, err := startProfiler(profiler, "")
+	}
+
+	// Start profiling on remote servers.
+	var hostErrs []NotificationPeerErr
+	for _, profiler := range profiles {
+		hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(profiler)...)
+
+		// Start profiling locally as well.
+		prof, err := startProfiler(profiler)
 		if err != nil {
 			hostErrs = append(hostErrs, NotificationPeerErr{
 				Host: *thisAddr,
 				Err:  err,
 			})
 		} else {
-			globalProfiler = prof
+			globalProfiler[profiler] = prof
 			hostErrs = append(hostErrs, NotificationPeerErr{
 				Host: *thisAddr,
 			})
@@ -620,7 +696,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
+	writeSuccessResponseJSON(w, startProfilingResultInBytes)
 }
 
 // dummyFileInfo represents a dummy representation of a profile data file
@@ -647,7 +723,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -821,7 +897,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// find number of disks in the setup
-	info := objectAPI.StorageInfo(ctx)
+	info := objectAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
 
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
@@ -906,13 +982,24 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{}
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
 	for _, state := range bgHealStates {
 		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
 			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
 		}
-
 	}
 
 	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
@@ -979,6 +1066,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 
 	var apiErr APIError
 	switch e := err.(type) {
+	case iampolicy.Error:
+		apiErr = APIError{
+			Code:           "XMinioMalformedIAMPolicy",
+			Description:    e.Error(),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
 	case config.Error:
 		apiErr = APIError{
 			Code:           "XMinioConfigError",
@@ -1031,7 +1124,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ListServerInfoAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1085,7 +1178,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1156,7 +1249,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatusHandler")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1188,20 +1281,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// 2. Check whether we can update / re-wrap the sealed key.
-	sealedKey, err = GlobalKMS.UpdateKey(keyID, sealedKey, kmsContext)
-	if err != nil {
-		response.UpdateErr = err.Error()
-		resp, err := json.Marshal(response)
-		if err != nil {
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
-			return
-		}
-		writeSuccessResponseJSON(w, resp)
-		return
-	}
-
-	// 3. Verify that we can indeed decrypt the (encrypted) key
+	// 2. Verify that we can indeed decrypt the (encrypted) key
 	decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
 	if err != nil {
 		response.DecryptionErr = err.Error()
@@ -1214,7 +1294,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// 4. Compare generated key with decrypted key
+	// 3. Compare generated key with decrypted key
 	if subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1 {
 		response.DecryptionErr = "The generated and the decrypted data key do not match"
 		resp, err := json.Marshal(response)
@@ -1241,7 +1321,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HardwareInfo")
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerHardwareInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1296,7 +1376,7 @@ func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *ht
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, "")
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1329,10 +1409,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		} else if ldapConn == nil {
 			ldap.Status = "Not Configured"
 		} else {
+			// Close ldap connection to avoid leaks.
+			ldapConn.Close()
 			ldap.Status = "online"
 		}
-		// Close ldap connection to avoid leaks.
-		defer ldapConn.Close()
 	}
 
 	log, audit := fetchLoggerInfo(cfg)
@@ -1341,7 +1421,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	notifyTarget := fetchLambdaInfo(cfg)
 
 	// Fetching the Storage information
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	var OnDisks int
 	var OffDisks int
@@ -1450,8 +1530,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 }
 
 func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
+
+	// Fetch the targets
+	targetList, err := notify.RegisterNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport(), nil, true)
+	if err != nil {
+		return nil
+	}
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
-	targetList, _ := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport())
 
 	for targetID, target := range targetList.TargetMap() {
 		targetIDStatus := make(map[string]madmin.Status)
@@ -1464,6 +1549,8 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 		list := lambdaMap[targetID.Name]
 		list = append(list, targetIDStatus)
 		lambdaMap[targetID.Name] = list
+		// Close any leaking connections
+		_ = target.Close()
 	}
 
 	notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
@@ -1492,36 +1579,29 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 		return vault
 	}
 
-	if err := checkConnection(kmsInfo.Endpoint); err != nil {
+	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
 
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"
 
-		kmsContext := crypto.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
+		kmsContext := crypto.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
 		// 1. Generate a new key using the KMS.
 		key, sealedKey, err := GlobalKMS.GenerateKey(keyID, kmsContext)
 		if err != nil {
-			vault.Encrypt = "Encryption failed"
+			vault.Encrypt = fmt.Sprintf("Encryption failed: %v", err)
 		} else {
 			vault.Encrypt = "Ok"
 		}
 
-		// 2. Check whether we can update / re-wrap the sealed key.
-		sealedKey, err = GlobalKMS.UpdateKey(keyID, sealedKey, kmsContext)
-		if err != nil {
-			vault.Update = "Re-wrap failed:"
-		} else {
-			vault.Update = "Ok"
-		}
-
-		// 3. Verify that we can indeed decrypt the (encrypted) key
-		decryptedKey, decryptErr := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
-
-		// 4. Compare generated key with decrypted key
-		if subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1 || decryptErr != nil {
-			vault.Decrypt = "Re-wrap failed:"
-		} else {
+		// 2. Verify that we can indeed decrypt the (encrypted) key
+		decryptedKey, err := GlobalKMS.UnsealKey(keyID, sealedKey, kmsContext)
+		switch {
+		case err != nil:
+			vault.Decrypt = fmt.Sprintf("Decryption failed: %v", err)
+		case subtle.ConstantTimeCompare(key[:], decryptedKey[:]) != 1:
+			vault.Decrypt = "Decryption failed: decrypted key does not match generated key"
+		default:
 			vault.Decrypt = "Ok"
 		}
 	}
@@ -1536,7 +1616,7 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 	var auditlogger []madmin.Audit
 	for log, l := range loggerCfg.HTTP {
 		if l.Enabled {
-			err := checkConnection(l.Endpoint)
+			err := checkConnection(l.Endpoint, 15*time.Second)
 			if err == nil {
 				mapLog := make(map[string]madmin.Status)
 				mapLog[log] = madmin.Status{Status: "Online"}
@@ -1551,7 +1631,7 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 
 	for audit, l := range loggerCfg.Audit {
 		if l.Enabled {
-			err := checkConnection(l.Endpoint)
+			err := checkConnection(l.Endpoint, 15*time.Second)
 			if err == nil {
 				mapAudit := make(map[string]madmin.Status)
 				mapAudit[audit] = madmin.Status{Status: "Online"}
@@ -1567,12 +1647,19 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 }
 
 // checkConnection - ping an endpoint , return err in case of no connection
-func checkConnection(endpointStr string) error {
+func checkConnection(endpointStr string, timeout time.Duration) error {
 	u, pErr := xnet.ParseURL(endpointStr)
 	if pErr != nil {
 		return pErr
 	}
-	if dErr := u.DialHTTP(); dErr != nil {
+
+	tr := newCustomHTTPTransport(
+		&tls.Config{RootCAs: globalRootCAs},
+		timeout,
+		0, /* Default value */
+	)()
+
+	if dErr := u.DialHTTP(tr); dErr != nil {
 		if urlErr, ok := dErr.(*url.Error); ok {
 			// To treat "connection refused" errors as un reachable endpoint.
 			if target.IsConnRefusedErr(urlErr.Err) {

cmd/admin-handlers_test.go

@@ -47,6 +47,10 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// reset global variables to start afresh.
 	resetTestGlobals()
 
+	// Set globalIsXL to indicate that the setup uses an erasure
+	// code backend.
+	globalIsXL = true
+
 	// Initializing objectLayer for HealFormatHandler.
 	objLayer, xlDirs, xlErr := initTestXLObjLayer()
 	if xlErr != nil {
@@ -63,15 +67,6 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 
 	globalEndpoints = mustGetZoneEndpoints(xlDirs...)
 
-	// Set globalIsXL to indicate that the setup uses an erasure
-	// code backend.
-	globalIsXL = true
-
-	// Init global heal state
-	if globalIsXL {
-		globalAllHealState = initHealState()
-	}
-
 	globalConfigSys = NewConfigSys()
 
 	globalIAMSys = NewIAMSys()
@@ -114,7 +109,7 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 		return nil, nil, err
 	}
 	endpoints := mustGetNewEndpoints(xlDirs...)
-	format, err := waitForFormatXL(true, endpoints, 1, 16, "")
+	format, err := waitForFormatXL(true, endpoints, 1, 1, 16, "")
 	if err != nil {
 		removeRoots(xlDirs)
 		return nil, nil, err
@@ -268,6 +263,7 @@ func TestAdminServerInfo(t *testing.T) {
 	if err != nil {
 		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
+
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.

cmd/admin-heal-ops.go

@@ -575,46 +575,38 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	h.lastHealActivity = UTCNow()
-
-	// Start healing the config prefix.
-	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
+	bucketsOnly := true // heal buckets only, not objects.
+	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)
 	}
 
-	// Start healing the bucket config prefix.
-	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
+	for {
+		select {
+		case path := <-h.sourceCh:
+			var itemType madmin.HealItemType
+			switch {
+			case path == nopHeal:
+				continue
+			case path == SlashSeparator:
+				itemType = madmin.HealItemMetadata
+			case !strings.Contains(path, SlashSeparator):
+				itemType = madmin.HealItemBucket
+			default:
+				itemType = madmin.HealItemObject
+			}
 
-	// Start healing the background ops prefix.
-	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
+			if err := h.queueHealTask(path, itemType); err != nil {
+				logger.LogIf(h.ctx, err)
+			}
 
-	for path := range h.sourceCh {
-
-		var itemType madmin.HealItemType
-		switch {
-		case path == nopHeal:
-			continue
-		case path == SlashSeparator:
-			itemType = madmin.HealItemMetadata
-		case !strings.Contains(path, SlashSeparator):
-			itemType = madmin.HealItemBucket
-		default:
-			itemType = madmin.HealItemObject
+			h.scannedItemsCount++
+			h.lastHealActivity = UTCNow()
+		case <-h.traverseAndHealDoneCh:
+			return nil
+		case <-GlobalServiceDoneCh:
+			return nil
 		}
-
-		if err := h.queueHealTask(path, itemType); err != nil {
-			logger.LogIf(h.ctx, err)
-		}
-
-		h.scannedItemsCount++
-		h.lastHealActivity = UTCNow()
 	}
-
-	return nil
 }
 
 func (h *healSequence) healFromSourceCh() {
@@ -622,7 +614,7 @@ func (h *healSequence) healFromSourceCh() {
 	close(h.traverseAndHealDoneCh)
 }
 
-func (h *healSequence) healItems() error {
+func (h *healSequence) healItems(bucketsOnly bool) error {
 	// Start with format healing
 	if err := h.healDiskFormat(); err != nil {
 		return err
@@ -644,7 +636,7 @@ func (h *healSequence) healItems() error {
 	}
 
 	// Heal buckets and objects
-	return h.healBuckets()
+	return h.healBuckets(bucketsOnly)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -655,7 +647,8 @@ func (h *healSequence) healItems() error {
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
 func (h *healSequence) traverseAndHeal() {
-	if err := h.healItems(); err != nil {
+	bucketsOnly := false // Heals buckets and objects also.
+	if err := h.healItems(bucketsOnly); err != nil {
 		if h.isQuitting() {
 			err = errHealStopSignalled
 		}
@@ -711,14 +704,14 @@ func (h *healSequence) healDiskFormat() error {
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets() error {
+func (h *healSequence) healBuckets(bucketsOnly bool) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(h.bucket)
+		return h.healBucket(h.bucket, bucketsOnly)
 	}
 
 	// Get current object layer instance.
@@ -733,7 +726,7 @@ func (h *healSequence) healBuckets() error {
 	}
 
 	for _, bucket := range buckets {
-		if err = h.healBucket(bucket.Name); err != nil {
+		if err = h.healBucket(bucket.Name, bucketsOnly); err != nil {
 			return err
 		}
 	}
@@ -742,7 +735,7 @@ func (h *healSequence) healBuckets() error {
 }
 
 // healBucket - traverses and heals given bucket
-func (h *healSequence) healBucket(bucket string) error {
+func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
@@ -753,6 +746,10 @@ func (h *healSequence) healBucket(bucket string) error {
 		return err
 	}
 
+	if bucketsOnly {
+		return nil
+	}
+
 	if !h.settings.Recursive {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
@@ -776,15 +773,15 @@ func (h *healSequence) healBucket(bucket string) error {
 
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object string) error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	return h.queueHealTask(pathJoin(bucket, object), madmin.HealItemObject)
 }

cmd/admin-router.go

@@ -56,6 +56,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	// DataUsageInfo operations
 	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
+	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
+
 	if globalIsDistXL || globalIsXL {
 		/// Heal operations
 

cmd/admin-server-info.go

@@ -20,9 +20,8 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"
@@ -190,51 +189,49 @@ func getLocalNetworkInfo(endpointZones EndpointZones, r *http.Request) madmin.Se
 // getLocalServerProperty - returns ServerDrivesPerfInfo for only the
 // local endpoints from given list of endpoints
 func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
-	var di madmin.Disk
 	var disks []madmin.Disk
 	addr := r.Host
 	if globalIsDistXL {
 		addr = GetLocalPeer(endpointZones)
 	}
 	network := make(map[string]string)
-	hosts := set.NewStringSet()
 	for _, ep := range endpointZones {
 		for _, endpoint := range ep.Endpoints {
-
-			url := strings.Replace(endpoint.URL.String(), endpoint.Path, "", -1)
-			if url == "" {
-				url = r.Host
+			nodeName := endpoint.Host
+			if nodeName == "" {
+				nodeName = r.Host
 			}
-			hosts.Add(url)
-
-			// Only proceed for local endpoints
 			if endpoint.IsLocal {
-				url = fetchAddress(url)
-				network[url] = "online"
-				if _, err := os.Stat(endpoint.Path); err != nil {
-					continue
+				// Only proceed for local endpoints
+				network[nodeName] = "online"
+				var di = madmin.Disk{
+					DrivePath: endpoint.Path,
+				}
+				diInfo, err := disk.GetInfo(endpoint.Path)
+				if err != nil {
+					if os.IsNotExist(err) || isSysErrPathNotFound(err) {
+						di.State = madmin.DriveStateMissing
+					} else {
+						di.State = madmin.DriveStateCorrupt
+					}
+				} else {
+					di.State = madmin.DriveStateOk
+					di.DrivePath = endpoint.Path
+					di.TotalSpace = diInfo.Total
+					di.UsedSpace = diInfo.Total - diInfo.Free
+					di.Utilization = float64((diInfo.Total - diInfo.Free) / diInfo.Total * 100)
 				}
-
-				diInfo, _ := disk.GetInfo(endpoint.Path)
-				di.State = "ok"
-				di.DrivePath = endpoint.Path
-				di.TotalSpace = diInfo.Total
-				di.UsedSpace = diInfo.Total - diInfo.Free
-				di.Utilization = float64((diInfo.Total - diInfo.Free) / diInfo.Total * 100)
 				disks = append(disks, di)
-			}
-		}
-	}
-
-	for host := range hosts {
-		_, present := network[host]
-		if !present {
-			err := checkConnection(host)
-			host = fetchAddress(host)
-			if err != nil {
-				network[host] = "offline"
 			} else {
-				network[host] = "online"
+				_, present := network[nodeName]
+				if !present {
+					err := IsServerResolvable(endpoint)
+					if err == nil {
+						network[nodeName] = "online"
+					} else {
+						network[nodeName] = "offline"
+					}
+				}
 			}
 		}
 	}
@@ -249,13 +246,3 @@ func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin
 		Disks:    disks,
 	}
 }
-
-// Replaces http and https from address
-func fetchAddress(address string) string {
-	if strings.Contains(address, "http://") {
-		address = strings.Replace(address, "http://", "", -1)
-	} else if strings.Contains(address, "https://") {
-		address = strings.Replace(address, "https://", "", -1)
-	}
-	return address
-}

cmd/api-errors.go

@@ -21,6 +21,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
@@ -32,6 +33,10 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -90,9 +95,12 @@ const (
 	ErrMissingContentLength
 	ErrMissingContentMD5
 	ErrMissingRequestBodyError
+	ErrMissingSecurityHeader
 	ErrNoSuchBucket
 	ErrNoSuchBucketPolicy
 	ErrNoSuchBucketLifecycle
+	ErrNoSuchLifecycleConfiguration
+	ErrNoSuchBucketSSEConfig
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -142,11 +150,14 @@ const (
 	ErrBadRequest
 	ErrKeyTooLongError
 	ErrInvalidBucketObjectLockConfiguration
+	ErrObjectLockConfigurationNotAllowed
+	ErrNoSuchObjectLockConfiguration
 	ErrObjectLocked
 	ErrInvalidRetentionDate
 	ErrPastObjectLockRetainDate
 	ErrUnknownWORMModeDirective
 	ErrObjectLockInvalidHeaders
+	ErrInvalidTagDirective
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -462,6 +473,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Missing required header for this request: Content-Md5.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingSecurityHeader: {
+		Code:           "MissingSecurityHeader",
+		Description:    "Your request was missing a required header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrMissingRequestBodyError: {
 		Code:           "MissingRequestBodyError",
 		Description:    "Request body is empty.",
@@ -482,6 +498,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket lifecycle configuration does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchLifecycleConfiguration: {
+		Code:           "NoSuchLifecycleConfiguration",
+		Description:    "The lifecycle configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchBucketSSEConfig: {
+		Code:           "ServerSideEncryptionConfigurationNotFoundError",
+		Description:    "The server side encryption configuration was not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchKey: {
 		Code:           "NoSuchKey",
 		Description:    "The specified key does not exist.",
@@ -731,6 +757,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Bucket is missing ObjectLockConfiguration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectLockConfigurationNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrNoSuchObjectLockConfiguration: {
+		Code:           "NoSuchObjectLockConfiguration",
+		Description:    "The specified object does not have a ObjectLock configuration",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrObjectLocked: {
 		Code:           "InvalidRequest",
 		Description:    "Object is WORM protected and cannot be overwritten",
@@ -822,6 +858,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidTagDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown tag directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidEncryptionMethod: {
 		Code:           "InvalidRequest",
 		Description:    "The encryption method specified is not supported",
@@ -1600,18 +1641,20 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case crypto.ErrKMSAuthLogin:
 		apiErr = ErrKMSAuthFailure
-	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
+	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
 		apiErr = ErrSlowDown
-	case errInvalidRetentionDate:
+	case objectlock.ErrInvalidRetentionDate:
 		apiErr = ErrInvalidRetentionDate
-	case errPastObjectLockRetainDate:
+	case objectlock.ErrPastObjectLockRetainDate:
 		apiErr = ErrPastObjectLockRetainDate
-	case errUnknownWORMModeDirective:
+	case objectlock.ErrUnknownWORMModeDirective:
 		apiErr = ErrUnknownWORMModeDirective
-	case errObjectLockInvalidHeaders:
+	case objectlock.ErrObjectLockInvalidHeaders:
 		apiErr = ErrObjectLockInvalidHeaders
+	case objectlock.ErrMalformedXML:
+		apiErr = ErrMalformedXML
 	}
 
 	// Compression errors
@@ -1701,7 +1744,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case BucketPolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
 	case BucketLifecycleNotFound:
-		apiErr = ErrNoSuchBucketLifecycle
+		apiErr = ErrNoSuchLifecycleConfiguration
+	case BucketSSEConfigNotFound:
+		apiErr = ErrNoSuchBucketSSEConfig
 	case *event.ErrInvalidEventName:
 		apiErr = ErrEventNotification
 	case *event.ErrInvalidARN:
@@ -1724,10 +1769,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrOverlappingFilterNotification
 	case *event.ErrUnsupportedConfiguration:
 		apiErr = ErrUnsupportedNotification
+	case OperationTimedOut:
+		apiErr = ErrOperationTimedOut
 	case BackendDown:
 		apiErr = ErrBackendDown
-	case crypto.Error:
-		apiErr = ErrObjectTampered
 	case ObjectNameTooLong:
 		apiErr = ErrKeyTooLongError
 	default:
@@ -1772,9 +1817,41 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case *xml.SyntaxError:
+			apiErr = APIError{
+				Code: "MalformedXML",
+				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
+					e.Error()),
+				HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
+			}
+		case url.EscapeError:
+			apiErr = APIError{
+				Code: "XMinioInvalidObjectName",
+				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
+					e.Error()),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case lifecycle.Error:
+			apiErr = APIError{
+				Code:           "InvalidRequest",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case tagging.Error:
+			apiErr = APIError{
+				Code:           e.Code(),
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case policy.Error:
+			apiErr = APIError{
+				Code:           "MalformedPolicy",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case crypto.Error:
 			apiErr = APIError{
-				Code:           "XKMSInternalError",
+				Code:           "XMinIOEncryptionError",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}

cmd/api-headers.go

@@ -22,7 +22,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -80,6 +82,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
+	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
+		w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts)))
+	}
+
 	if objInfo.ContentType != "" {
 		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}
@@ -91,6 +97,17 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if !objInfo.Expires.IsZero() {
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
+	if globalCacheConfig.Enabled {
+		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
+		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
+	}
+
+	// Set tag count if object has tags
+	tags, _ := url.ParseQuery(objInfo.UserTags)
+	tagCount := len(tags)
+	if tagCount != 0 {
+		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	}
 
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {

cmd/api-router.go

@@ -105,26 +105,33 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler))).Queries("acl", "")
-		// GetObjectTagging - this is a dummy call.
+		// PutObjectACL - this is a dummy call.
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler))).Queries("acl", "")
+		// GetObjectTagging
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler))).Queries("tagging", "")
+		// PutObjectTagging
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler))).Queries("tagging", "")
+		// DeleteObjectTagging
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler))).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceHdrs(api.GetObjectRetentionHandler))).Queries("retention", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler))).Queries("retention", "")
+		// GetObjectLegalHold
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
 		// GetObject
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)))
 		// CopyObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)))
 		// PutObjectRetention
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceHdrs(api.PutObjectRetentionHandler))).Queries("retention", "")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler))).Queries("retention", "")
+		// PutObjectLegalHold
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
+
 		// PutObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)))
 		// DeleteObject
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)))
-		// PutObjectLegalHold
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
-		// GetObjectLegalHold
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
 
 		/// Bucket operations
 		// GetBucketLocation
@@ -133,10 +140,14 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler))).Queries("policy", "")
 		// GetBucketLifecycle
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler))).Queries("lifecycle", "")
+		// GetBucketEncryption
+		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler))).Queries("encryption", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler))).Queries("acl", "")
+		// PutBucketACL -- this is a dummy call.
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
@@ -178,6 +189,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler)))
 		// PutBucketLifecycle
 		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler))).Queries("lifecycle", "")
+		// PutBucketEncryption
+		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler))).Queries("encryption", "")
+
 		// PutBucketPolicy
 		bucket.Methods(http.MethodPut).HandlerFunc(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler))).Queries("policy", "")
 
@@ -199,6 +213,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler))).Queries("policy", "")
 		// DeleteBucketLifecycle
 		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler))).Queries("lifecycle", "")
+		// DeleteBucketEncryption
+		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler))).Queries("encryption", "")
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler)))
 	}

cmd/auth-handler.go

@@ -28,13 +28,13 @@ import (
 	"net/http"
 	"strings"
 
-	jwtgo "github.com/dgrijalva/jwt-go"
 	xhttp "github.com/minio/minio/cmd/http"
+	xjwt "github.com/minio/minio/cmd/jwt"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // Verify if request has JWT.
@@ -179,12 +179,14 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 
 // Fetch claims in the security token returned by the client.
 func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
-	claims := make(map[string]interface{})
+	claims := xjwt.NewMapClaims()
+
 	token := getSessionToken(r)
 	if token == "" {
-		return claims, nil
+		return claims.Map(), nil
 	}
-	stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
+
+	stsTokenCallback := func(claims *xjwt.MapClaims) ([]byte, error) {
 		// JWT token for x-amz-security-token is signed with admin
 		// secret key, temporary credentials become invalid if
 		// server admin credentials change. This is done to ensure
@@ -195,66 +197,42 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 		// on the client side and is treated like an opaque value.
 		return []byte(globalActiveCred.SecretKey), nil
 	}
-	p := &jwtgo.Parser{
-		ValidMethods: []string{
-			jwtgo.SigningMethodHS256.Alg(),
-			jwtgo.SigningMethodHS512.Alg(),
-		},
-	}
-	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
-	if err != nil {
-		return nil, err
-	}
-	if !jtoken.Valid {
+
+	if err := xjwt.ParseWithClaims(token, claims, stsTokenCallback); err != nil {
 		return nil, errAuthentication
 	}
-	v, ok := claims["accessKey"]
-	if !ok {
-		return nil, errInvalidAccessKeyID
-	}
-	if _, ok = v.(string); !ok {
-		return nil, errInvalidAccessKeyID
-	}
 
 	if globalPolicyOPA == nil {
-		// If OPA is not set and if ldap claim key is set,
-		// allow the claim.
-		if _, ok := claims[ldapUser]; ok {
-			return claims, nil
+		// If OPA is not set and if ldap claim key is set, allow the claim.
+		if _, ok := claims.Lookup(ldapUser); ok {
+			return claims.Map(), nil
 		}
 
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.
-		p, pok := claims[iamPolicyName()]
+		_, pok := claims.Lookup(iamPolicyClaimName())
 		if !pok {
 			return nil, errAuthentication
 		}
-		if _, pok = p.(string); !pok {
-			return nil, errAuthentication
-		}
-		sp, spok := claims[iampolicy.SessionPolicyName]
-		// Sub policy is optional, if not set return success.
+
+		sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
 		if !spok {
-			return claims, nil
-		}
-		// Sub policy is set but its not a string, reject such requests
-		spStr, spok := sp.(string)
-		if !spok {
-			return nil, errAuthentication
+			return claims.Map(), nil
 		}
 		// Looks like subpolicy is set and is a string, if set then its
 		// base64 encoded, decode it. Decoding fails reject such requests.
-		spBytes, err := base64.StdEncoding.DecodeString(spStr)
+		spBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
 			logger.LogIf(context.Background(), err, logger.Application)
 			return nil, errAuthentication
 		}
-		claims[iampolicy.SessionPolicyName] = string(spBytes)
+		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
-	return claims, nil
+
+	return claims.Map(), nil
 }
 
 // Fetch claims in the security token returned by the client and validate the token.
@@ -402,12 +380,11 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 		err                       error
 		contentMD5, contentSHA256 []byte
 	)
+
 	// Extract 'Content-Md5' if present.
-	if _, ok := r.Header[xhttp.ContentMD5]; ok {
-		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get(xhttp.ContentMD5))
-		if err != nil || len(contentMD5) == 0 {
-			return ErrInvalidDigest
-		}
+	contentMD5, err = checkValidMD5(r.Header)
+	if err != nil {
+		return ErrInvalidDigest
 	}
 
 	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)

cmd/background-heal-ops.go

@@ -52,6 +52,19 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
+func waitForLowHTTPReq(tolerance int32) {
+	if httpServer := newHTTPServerFn(); httpServer != nil {
+		// Wait at max 10 minute for an inprogress request before proceeding to heal
+		waitCount := 600
+		// Any requests in progress, delay the heal.
+		for (httpServer.GetRequestCount() >= tolerance) &&
+			waitCount > 0 {
+			waitCount--
+			time.Sleep(1 * time.Second)
+		}
+	}
+}
+
 // Wait for heal requests and process them
 func (h *healRoutine) run() {
 	ctx := context.Background()
@@ -61,20 +74,13 @@ func (h *healRoutine) run() {
 			if !ok {
 				break
 			}
-			if httpServer := newHTTPServerFn(); httpServer != nil {
-				// Wait at max 10 minute for an inprogress request before proceeding to heal
-				waitCount := 600
-				// Any requests in progress, delay the heal.
-				for (httpServer.GetRequestCount() >= int32(globalEndpoints.Nodes())) &&
-					waitCount > 0 {
-					waitCount--
-					time.Sleep(1 * time.Second)
-				}
-			}
+
+			// Wait and proceed if there are active requests
+			waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
 
 			var res madmin.HealResultItem
 			var err error
-			bucket, object := urlPath2BucketObjectName(task.path)
+			bucket, object := path2BucketObject(task.path)
 			switch {
 			case bucket == "" && object == "":
 				res, err = bgHealDiskFormat(ctx, task.opts)
@@ -121,7 +127,7 @@ func startBackgroundHealing() {
 
 	// Launch the background healer sequence to track
 	// background healing operations
-	info := objAPI.StorageInfo(ctx)
+	info := objAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
 	nh := newBgHealSequence(numDisks)
 	globalBackgroundHealState.LaunchNewHealSequence(nh)

cmd/background-newdisks-heal-ops.go

@@ -62,10 +62,11 @@ func monitorLocalDisksAndHeal() {
 		time.Sleep(time.Second)
 	}
 
-	// Perform automatic disk healing when a new one is inserted
+	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		time.Sleep(defaultMonitorNewDiskInterval)
 
+		// Attempt a heal as the server starts-up first.
 		localDisksInZoneHeal := make([]Endpoints, len(z.zones))
 		for i, ep := range globalEndpoints {
 			localDisksToHeal := Endpoints{}
@@ -88,6 +89,7 @@ func monitorLocalDisksAndHeal() {
 
 		// Reformat disks
 		bgSeq.sourceCh <- SlashSeparator
+
 		// Ensure that reformatting disks is finished
 		bgSeq.sourceCh <- nopHeal
 

cmd/bootstrap-peer-server.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"

cmd/bucket-encryption-handlers.go

@@ -0,0 +1,181 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/policy"
+)
+
+const (
+	// Bucket Encryption configuration file name.
+	bucketSSEConfig = "bucket-encryption.xml"
+)
+
+// PutBucketEncryptionHandler - Stores given bucket encryption configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketEncryption.html
+func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketEncryption")
+
+	defer logger.AuditLog(w, r, "PutBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// PutBucketEncyrption API requires Content-Md5
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Parse bucket encryption xml
+	encConfig, err := validateBucketSSEConfig(io.LimitReader(r.Body, maxBucketSSEConfigSize))
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Return error if KMS is not initialized
+	if GlobalKMS == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Store the bucket encryption configuration in the object layer
+	if err = objAPI.SetBucketSSEConfig(ctx, bucket, encConfig); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Update the in-memory bucket encryption cache
+	globalBucketSSEConfigSys.Set(bucket, *encConfig)
+
+	// Update peer MinIO servers of the updated bucket encryption config
+	globalNotificationSys.SetBucketSSEConfig(ctx, bucket, encConfig)
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketEncryptionHandler - Returns bucket policy configuration
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketEncryption.html
+func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketEncryption")
+
+	defer logger.AuditLog(w, r, "GetBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists
+	var err error
+	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Fetch bucket encryption configuration from object layer
+	var encConfig *bucketsse.BucketSSEConfig
+	if encConfig, err = objAPI.GetBucketSSEConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	var encConfigData []byte
+	if encConfigData, err = xml.Marshal(encConfig); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write bucket encryption configuration to client
+	writeSuccessResponseXML(w, encConfigData)
+}
+
+// DeleteBucketEncryptionHandler - Removes bucket encryption configuration
+func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketEncryption")
+
+	defer logger.AuditLog(w, r, "DeleteBucketEncryption", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists
+	var err error
+	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Delete bucket encryption config from object layer
+	if err = objAPI.DeleteBucketSSEConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Remove entry from the in-memory bucket encryption cache
+	globalBucketSSEConfigSys.Remove(bucket)
+	// Update peer MinIO servers of the updated bucket encryption config
+	globalNotificationSys.RemoveBucketSSEConfig(ctx, bucket)
+
+	writeSuccessNoContent(w)
+}

cmd/bucket-encryption.go

@@ -0,0 +1,169 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/xml"
+	"errors"
+	"io"
+	"path"
+	"sync"
+
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+)
+
+// BucketSSEConfigSys - in-memory cache of bucket encryption config
+type BucketSSEConfigSys struct {
+	sync.RWMutex
+	bucketSSEConfigMap map[string]bucketsse.BucketSSEConfig
+}
+
+// NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache
+func NewBucketSSEConfigSys() *BucketSSEConfigSys {
+	return &BucketSSEConfigSys{
+		bucketSSEConfigMap: make(map[string]bucketsse.BucketSSEConfig),
+	}
+}
+
+// load - Loads the bucket encryption configuration for the given list of buckets
+func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
+	for _, bucket := range buckets {
+		config, err := objAPI.GetBucketSSEConfig(context.Background(), bucket.Name)
+		if err != nil {
+			if _, ok := err.(BucketSSEConfigNotFound); ok {
+				sys.Remove(bucket.Name)
+			}
+			continue
+		}
+		sys.Set(bucket.Name, *config)
+	}
+
+	return nil
+}
+
+// Init - Initializes in-memory bucket encryption config cache for the given list of buckets
+func (sys *BucketSSEConfigSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// We don't cache bucket encryption config in gateway mode, nothing to do.
+	if globalIsGateway {
+		return nil
+	}
+
+	// Load bucket encryption config cache once during boot.
+	return sys.load(buckets, objAPI)
+}
+
+// Get - gets bucket encryption config for the given bucket.
+func (sys *BucketSSEConfigSys) Get(bucket string) (config bucketsse.BucketSSEConfig, ok bool) {
+	// We don't cache bucket encryption config in gateway mode.
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return
+		}
+
+		cfg, err := objAPI.GetBucketSSEConfig(context.Background(), bucket)
+		if err != nil {
+			return
+		}
+		return *cfg, true
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+	config, ok = sys.bucketSSEConfigMap[bucket]
+	return
+}
+
+// Set - sets bucket encryption config to given bucket name.
+func (sys *BucketSSEConfigSys) Set(bucket string, config bucketsse.BucketSSEConfig) {
+	// We don't cache bucket encryption config in gateway mode.
+	if globalIsGateway {
+		return
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+	sys.bucketSSEConfigMap[bucket] = config
+}
+
+// Remove - removes bucket encryption config for given bucket.
+func (sys *BucketSSEConfigSys) Remove(bucket string) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.bucketSSEConfigMap, bucket)
+}
+
+// saveBucketSSEConfig - save bucket encryption config for given bucket.
+func saveBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string, config *bucketsse.BucketSSEConfig) error {
+	data, err := xml.Marshal(config)
+	if err != nil {
+		return err
+	}
+
+	// Path to store bucket encryption config for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+	return saveConfig(ctx, objAPI, configFile, data)
+}
+
+// getBucketSSEConfig - get bucket encryption config for given bucket.
+func getBucketSSEConfig(objAPI ObjectLayer, bucket string) (*bucketsse.BucketSSEConfig, error) {
+	// Path to bucket-encryption.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+	configData, err := readConfig(context.Background(), objAPI, configFile)
+	if err != nil {
+		if err == errConfigNotFound {
+			err = BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+
+	return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
+}
+
+// removeBucketSSEConfig - removes bucket encryption config for given bucket.
+func removeBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
+	// Path to bucket-encryption.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
+
+	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return err
+	}
+	return nil
+}
+
+// validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.
+func validateBucketSSEConfig(r io.Reader) (*bucketsse.BucketSSEConfig, error) {
+	encConfig, err := bucketsse.ParseBucketSSEConfig(r)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(encConfig.Rules) == 1 && encConfig.Rules[0].DefaultEncryptionAction.Algorithm == bucketsse.AES256 {
+		return encConfig, nil
+	}
+	return nil, errors.New("Unsupported bucket encryption configuration")
+}

cmd/bucket-encryption_test.go

@@ -0,0 +1,70 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+)
+
+func TestValidateBucketSSEConfig(t *testing.T) {
+	testCases := []struct {
+		inputXML    string
+		expectedErr error
+		shouldPass  bool
+	}{
+		// MinIO supported XML
+		{
+			inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Rule>
+			<ApplyServerSideEncryptionByDefault>
+			<SSEAlgorithm>AES256</SSEAlgorithm>
+			</ApplyServerSideEncryptionByDefault>
+			</Rule>
+			</ServerSideEncryptionConfiguration>`,
+			expectedErr: nil,
+			shouldPass:  true,
+		},
+		// Unsupported XML
+		{
+			inputXML: `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+			<Rule>
+			<ApplyServerSideEncryptionByDefault>
+                        <SSEAlgorithm>aws:kms</SSEAlgorithm>
+                        <KMSMasterKeyID>arn:aws:kms:us-east-1:1234/5678example</KMSMasterKeyID>
+			</ApplyServerSideEncryptionByDefault>
+			</Rule>
+			</ServerSideEncryptionConfiguration>`,
+			expectedErr: errors.New("Unsupported bucket encryption configuration"),
+			shouldPass:  false,
+		},
+	}
+
+	for i, tc := range testCases {
+		_, err := validateBucketSSEConfig(bytes.NewReader([]byte(tc.inputXML)))
+		if tc.shouldPass && err != nil {
+			t.Fatalf("Test case %d: Expected to succeed but got %s", i+1, err)
+		}
+
+		if !tc.shouldPass {
+			if err == nil || err != nil && err.Error() != tc.expectedErr.Error() {
+				t.Fatalf("Test case %d: Expected %s but got %s", i+1, tc.expectedErr, err)
+			}
+		}
+	}
+}

cmd/bucket-handlers-listobjects.go

@@ -24,7 +24,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode

cmd/bucket-handlers.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -36,11 +35,12 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -74,40 +74,45 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		return
 	}
 
-	bucketSet := set.NewStringSet()
+	bucketsSet := set.NewStringSet()
+	bucketsToBeUpdated := set.NewStringSet()
+	bucketsInConflict := set.NewStringSet()
+	for _, bucket := range buckets {
+		bucketsSet.Add(bucket.Name)
+		r, ok := dnsBuckets[bucket.Name]
+		if !ok {
+			bucketsToBeUpdated.Add(bucket.Name)
+			continue
+		}
+		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+			if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// No difference in terms of domainIPs and nothing
+				// has changed so we don't change anything on the etcd.
+				continue
+			}
+			// if domain IPs intersect then it won't be an empty set.
+			// such an intersection means that bucket exists on etcd.
+			// but if we do see a difference with local domain IPs with
+			// hostSlice from etcd then we should update with newer
+			// domainIPs, we proceed to do that here.
+			bucketsToBeUpdated.Add(bucket.Name)
+			continue
+		}
+		// No IPs seem to intersect, this means that bucket exists but has
+		// different IP addresses perhaps from a different deployment.
+		// bucket names are globally unique in federation at a given
+		// path prefix, name collision is not allowed. We simply log
+		// an error and continue.
+		bucketsInConflict.Add(bucket.Name)
+	}
 
-	// Add buckets that are not registered with the DNS
+	// Add/update buckets that are not registered with the DNS
 	g := errgroup.WithNErrs(len(buckets))
-	for index := range buckets {
-		bucketSet.Add(buckets[index].Name)
+	bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice()
+	for index := range bucketsToBeUpdatedSlice {
 		index := index
 		g.Go(func() error {
-			r, gerr := globalDNSConfig.Get(buckets[index].Name)
-			if gerr != nil {
-				if gerr == dns.ErrNoEntriesFound {
-					return globalDNSConfig.Put(buckets[index].Name)
-				}
-				return gerr
-			}
-			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-					// No difference in terms of domainIPs and nothing
-					// has changed so we don't change anything on the etcd.
-					return nil
-				}
-				// if domain IPs intersect then it won't be an empty set.
-				// such an intersection means that bucket exists on etcd.
-				// but if we do see a difference with local domain IPs with
-				// hostSlice from etcd then we should update with newer
-				// domainIPs, we proceed to do that here.
-				return globalDNSConfig.Put(buckets[index].Name)
-			}
-			// No IPs seem to intersect, this means that bucket exists but has
-			// different IP addresses perhaps from a different deployment.
-			// bucket names are globally unique in federation at a given
-			// path prefix, name collision is not allowed. We simply log
-			// an error and continue.
-			return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", buckets[index].Name, globalDomainIPs.ToSlice())
+			return globalDNSConfig.Put(bucketsToBeUpdatedSlice[index])
 		}, index)
 	}
 
@@ -117,36 +122,26 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		}
 	}
 
-	g = errgroup.WithNErrs(len(dnsBuckets))
-	// Remove buckets that are in DNS for this server, but aren't local
-	for index := range dnsBuckets {
-		index := index
-		g.Go(func() error {
-			// This is a local bucket that exists, so we can continue
-			if bucketSet.Contains(dnsBuckets[index].Key) {
-				return nil
-			}
-
-			// This is not for our server, so we can continue
-			hostPort := net.JoinHostPort(dnsBuckets[index].Host, string(dnsBuckets[index].Port))
-			if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
-				return nil
-			}
-
-			// We go to here, so we know the bucket no longer exists,
-			// but is registered in DNS to this server
-			if err := globalDNSConfig.DeleteRecord(dnsBuckets[index]); err != nil {
-				return fmt.Errorf("Failed to remove DNS entry for %s due to %w",
-					dnsBuckets[index].Key, err)
-			}
-
-			return nil
-		}, index)
+	for _, bucket := range bucketsInConflict.ToSlice() {
+		logger.LogIf(context.Background(), fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
-	for _, err := range g.Wait() {
-		if err != nil {
-			logger.LogIf(context.Background(), err)
+	// Remove buckets that are in DNS for this server, but aren't local
+	for bucket, records := range dnsBuckets {
+		if bucketsSet.Contains(bucket) {
+			continue
+		}
+
+		if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(records)...)).IsEmpty() {
+			// This is not for our server, so we can continue
+			continue
+		}
+
+		// We go to here, so we know the bucket no longer exists,
+		// but is registered in DNS to this server
+		if err = globalDNSConfig.Delete(bucket); err != nil {
+			logger.LogIf(context.Background(), fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+				bucket, err))
 		}
 	}
 }
@@ -278,22 +273,17 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 	// If etcd, dns federation configured list buckets from etcd.
 	var bucketsInfo []BucketInfo
-	if globalDNSConfig != nil {
+	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		bucketSet := set.NewStringSet()
-		for _, dnsRecord := range dnsBuckets {
-			if bucketSet.Contains(dnsRecord.Key) {
-				continue
-			}
+		for _, dnsRecords := range dnsBuckets {
 			bucketsInfo = append(bucketsInfo, BucketInfo{
-				Name:    dnsRecord.Key,
-				Created: dnsRecord.CreationDate,
+				Name:    dnsRecords[0].Key,
+				Created: dnsRecords[0].CreationDate,
 			})
-			bucketSet.Add(dnsRecord.Key)
 		}
 	} else {
 		// Invoke the list buckets.
@@ -352,6 +342,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
@@ -359,34 +356,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Content-Md5 is requied should be set
-	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Allocate incoming content length bytes.
-	var deleteXMLBytes []byte
-	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
-	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
-		deleteXMLBytes = make([]byte, maxBodySize)
-	} else {
-		deleteXMLBytes = make([]byte, r.ContentLength)
-	}
-
-	// Read incoming body XML bytes.
-	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
+	// The max. XML contains 100000 object names (each at most 1024 bytes long) + XML overhead
+	const maxBodySize = 2 * 100000 * 1024
 
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
-	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
+	if err := xmlDecoder(r.Body, deleteObjects, maxBodySize); err != nil {
 		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -533,7 +510,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	if globalDNSConfig != nil {
-		if _, err := globalDNSConfig.Get(bucket); err != nil {
+		sr, err := globalDNSConfig.Get(bucket)
+		if err != nil {
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
@@ -566,7 +544,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			return
 
 		}
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBucketAlreadyOwnedByYou), r.URL, guessIsBrowserReq(r))
+		apiErr := ErrBucketAlreadyExists
+		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+			apiErr = ErrBucketAlreadyOwnedByYou
+		}
+		// No IPs seem to intersect, this means that bucket exists but has
+		// different IP addresses perhaps from a different deployment.
+		// bucket names are globally unique in federation at a given
+		// path prefix, name collision is not allowed. Return appropriate error.
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -577,14 +563,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if objectLockEnabled {
+	if objectLockEnabled && !globalIsGateway {
 		configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
 		if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		globalBucketObjectLockConfig.Set(bucket, Retention{})
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, Retention{})
+		globalBucketObjectLockConfig.Set(bucket, objectlock.Retention{})
+		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, objectlock.Retention{})
 	}
 
 	// Make sure to add Location information here only for bucket
@@ -758,8 +744,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	pReader := NewPutObjReader(rawReader, nil, nil)
 	var objectEncryptionKey []byte
 
+	// Check if bucket encryption is enabled
+	_, encEnabled := globalBucketSSEConfigSys.Get(bucket)
 	// This request header needs to be set prior to setting ObjectOptions
-	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+	if (globalAutoEncryption || encEnabled) && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
 	// get gateway encryption options
@@ -1005,7 +993,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	config, err := parseObjectLockConfig(r.Body)
+	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		apiErr.Description = err.Error()
@@ -1017,7 +1005,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	if err != nil {
 		aerr := toAPIError(ctx, err)
 		if err == errConfigNotFound {
-			aerr = errorCodes.ToAPIErr(ErrMethodNotAllowed)
+			aerr = errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed)
 		}
 		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
 		return
@@ -1099,7 +1087,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 			return
 		}
 
-		if configData, err = xml.Marshal(newObjectLockConfig()); err != nil {
+		if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}

cmd/bucket-lifecycle-handler.go

@@ -22,9 +22,10 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/lifecycle"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 const (
@@ -48,6 +49,12 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	// PutBucketLifecycle always needs a Content-Md5
+	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -59,15 +66,9 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// PutBucketLifecycle always needs a Content-Md5
-	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
 	bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/bucket-notification-handlers.go

@@ -30,8 +30,8 @@ import (
 	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -84,8 +84,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 		config.SetRegion(globalServerRegion)
+		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 		notificationBytes, err := xml.Marshal(config)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/bucket-policy-handlers.go

@@ -24,7 +24,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 const (
@@ -77,7 +77,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/bucket-policy-handlers_test.go

@@ -29,8 +29,8 @@ import (
 	"testing"
 
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/policy"
-	"github.com/minio/minio/pkg/policy/condition"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/policy/condition"
 )
 
 func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {

cmd/common-main.go

@@ -17,7 +17,9 @@
 package cmd
 
 import (
+	"context"
 	"crypto/x509"
+	"encoding/gob"
 	"errors"
 	"net"
 	"path/filepath"
@@ -34,6 +36,17 @@ import (
 	"github.com/minio/minio/pkg/env"
 )
 
+func init() {
+	logger.Init(GOPATH, GOROOT)
+	logger.RegisterError(config.FmtError)
+
+	// Initialize globalConsoleSys system
+	globalConsoleSys = NewConsoleLogger(context.Background())
+	logger.AddTarget(globalConsoleSys)
+
+	gob.Register(StorageErr(""))
+}
+
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 	if (globalAutoEncryption || GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
 		logger.Fatal(errInvalidArgument,

cmd/config-current.go

@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 func initHelp() {
@@ -315,6 +316,13 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
+	// Bucket federation is 'true' only when IAM assets are not namespaced
+	// per tenant and all tenants interested in globally available users
+	// if namespace was requested such as specifying etcdPathPrefix then
+	// we assume that users are interested in global bucket support
+	// but not federation.
+	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
+
 	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 			dns.DomainNames(globalDomainNames),
@@ -462,6 +470,20 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 
 	envHelp := config.HelpKVS{}
 	if envOnly {
+		// Only for multiple targets, make sure
+		// to list the ENV, for regular k/v EnableKey is
+		// implicit, for ENVs we cannot make it implicit.
+		if subSysHelp.MultipleTargets {
+			envK := config.EnvPrefix + strings.Join([]string{
+				strings.ToTitle(subSys), strings.ToTitle(madmin.EnableKey),
+			}, config.EnvWordDelimiter)
+			envHelp = append(envHelp, config.HelpKV{
+				Key:         envK,
+				Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
+				Optional:    false,
+				Type:        "on|off",
+			})
+		}
 		for _, hkv := range h {
 			envK := config.EnvPrefix + strings.Join([]string{
 				strings.ToTitle(subSys), strings.ToTitle(hkv.Key),

cmd/config-encrypted.go

@@ -84,7 +84,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 			return nil
 		}
 		if !globalActiveCred.IsValid() {
-			return errInvalidArgument
+			return config.ErrMissingCredentialsBackendEncrypted(nil)
 		}
 	}
 
@@ -267,7 +267,12 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 
 		if !utf8.Valid(data) {
-			return errors.New("config data not in plain-text form")
+			_, err = decryptData(data, globalActiveCred)
+			if err == nil {
+				// Config is already encrypted with right keys
+				continue
+			}
+			return errors.New("config data not in plain-text form or encrypted")
 		}
 
 		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -279,9 +284,11 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 			return err
 		}
 	}
-	if encrypted && globalActiveCred.IsValid() {
+
+	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
+
 	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
 }
 
@@ -309,7 +316,8 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 
 	var marker string
 	for {
-		res, err := objAPI.ListObjects(context.Background(), minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
+		res, err := objAPI.ListObjects(context.Background(), minioMetaBucket,
+			minioConfigPrefix, marker, "", maxObjectList)
 		if err != nil {
 			return err
 		}
@@ -339,7 +347,12 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 			}
 
 			if !utf8.Valid(data) {
-				return errors.New("config data not in plain-text form")
+				_, err = decryptData(data, globalActiveCred)
+				if err == nil {
+					// Config is already encrypted with right keys
+					continue
+				}
+				return errors.New("config data not in plain-text form or encrypted")
 			}
 
 			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -359,7 +372,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		marker = res.NextMarker
 	}
 
-	if encrypted && globalActiveCred.IsValid() {
+	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
 

cmd/config.go

@@ -20,14 +20,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 	"path"
 	"sort"
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -170,6 +169,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	}
 
 	var config = config.New()
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(configData, &config); err != nil {
 		return nil, err
 	}
@@ -210,33 +210,7 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
-	// Initializing configuration needs a retry mechanism for
-	// the following reasons:
-	//  - Read quorum is lost just after the initialization
-	//    of the object layer.
-	//  - Write quorum not met when upgrading configuration
-	//    version is needed.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case <-retryTimerCh:
-			if err := initConfig(objAPI); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for configuration to be initialized..")
-					continue
-				}
-				return err
-			}
-			return nil
-		case <-globalOSSignalCh:
-			return fmt.Errorf("Initializing config sub-system gracefully stopped")
-		}
-	}
+	return initConfig(objAPI)
 }
 
 // NewConfigSys - creates new config system object.

cmd/config/cache/config.go

@@ -28,12 +28,15 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled bool     `json:"-"`
-	Drives  []string `json:"drives"`
-	Expiry  int      `json:"expiry"`
-	MaxUse  int      `json:"maxuse"`
-	Quota   int      `json:"quota"`
-	Exclude []string `json:"exclude"`
+	Enabled       bool     `json:"-"`
+	Drives        []string `json:"drives"`
+	Expiry        int      `json:"expiry"`
+	MaxUse        int      `json:"maxuse"`
+	Quota         int      `json:"quota"`
+	Exclude       []string `json:"exclude"`
+	After         int      `json:"after"`
+	WatermarkLow  int      `json:"watermark_low"`
+	WatermarkHigh int      `json:"watermark_high"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -60,7 +63,18 @@ func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
 	if _cfg.Quota < 0 {
 		return errors.New("config quota value should not be null or negative")
 	}
-
+	if _cfg.After < 0 {
+		return errors.New("cache after value should not be less than 0")
+	}
+	if _cfg.WatermarkLow < 0 || _cfg.WatermarkLow > 100 {
+		return errors.New("config low watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkHigh < 0 || _cfg.WatermarkHigh > 100 {
+		return errors.New("config high watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkLow > 0 && (_cfg.WatermarkLow >= _cfg.WatermarkHigh) {
+		return errors.New("config low watermark value should be less than high watermark")
+	}
 	return nil
 }
 

cmd/config/cache/help.go

@@ -50,5 +50,23 @@ var (
 			Optional:    true,
 			Type:        "sentence",
 		},
+		config.HelpKV{
+			Key:         After,
+			Description: `minimum accesses before caching an object`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         WatermarkLow,
+			Description: `% of cache use at which to stop cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         WatermarkHigh,
+			Description: `% of cache use at which to start cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -26,21 +26,31 @@ import (
 
 // Cache ENVs
 const (
-	Drives  = "drives"
-	Exclude = "exclude"
-	Expiry  = "expiry"
-	MaxUse  = "maxuse"
-	Quota   = "quota"
+	Drives        = "drives"
+	Exclude       = "exclude"
+	Expiry        = "expiry"
+	MaxUse        = "maxuse"
+	Quota         = "quota"
+	After         = "after"
+	WatermarkLow  = "watermark_low"
+	WatermarkHigh = "watermark_high"
+
+	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry        = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse        = "MINIO_CACHE_MAXUSE"
+	EnvCacheQuota         = "MINIO_CACHE_QUOTA"
+	EnvCacheAfter         = "MINIO_CACHE_AFTER"
+	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
+	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 
-	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
-	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
-	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
-	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
-	EnvCacheQuota               = "MINIO_CACHE_QUOTA"
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
-	DefaultExpiry = "90"
-	DefaultQuota  = "80"
+	DefaultExpiry        = "90"
+	DefaultQuota         = "80"
+	DefaultAfter         = "0"
+	DefaultWaterMarkLow  = "70"
+	DefaultWaterMarkHigh = "80"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -62,6 +72,18 @@ var (
 			Key:   Quota,
 			Value: DefaultQuota,
 		},
+		config.KV{
+			Key:   After,
+			Value: DefaultAfter,
+		},
+		config.KV{
+			Key:   WatermarkLow,
+			Value: DefaultWaterMarkLow,
+		},
+		config.KV{
+			Key:   WatermarkHigh,
+			Value: DefaultWaterMarkHigh,
+		},
 	}
 )
 
@@ -79,7 +101,6 @@ func Enabled(kvs config.KVS) bool {
 // variables and merge them with provided CacheConfiguration.
 func LookupConfig(kvs config.KVS) (Config, error) {
 	cfg := Config{}
-
 	if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
@@ -121,9 +142,7 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 			return cfg, config.ErrInvalidCacheQuota(err)
 		}
 		cfg.Quota = cfg.MaxUse
-	}
-
-	if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
+	} else if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
 		cfg.Quota, err = strconv.Atoi(quotaStr)
 		if err != nil {
 			return cfg, config.ErrInvalidCacheQuota(err)
@@ -136,5 +155,45 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		cfg.MaxUse = cfg.Quota
 	}
 
+	if afterStr := env.Get(EnvCacheAfter, kvs.Get(After)); afterStr != "" {
+		cfg.After, err = strconv.Atoi(afterStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheAfter(err)
+		}
+		// after should be a valid value >= 0.
+		if cfg.After < 0 {
+			err := errors.New("cache after value cannot be less than 0")
+			return cfg, config.ErrInvalidCacheAfter(err)
+		}
+	}
+
+	if lowWMStr := env.Get(EnvCacheWatermarkLow, kvs.Get(WatermarkLow)); lowWMStr != "" {
+		cfg.WatermarkLow, err = strconv.Atoi(lowWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+		// WatermarkLow should be a valid percentage.
+		if cfg.WatermarkLow < 0 || cfg.WatermarkLow > 100 {
+			err := errors.New("config min watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+	}
+
+	if highWMStr := env.Get(EnvCacheWatermarkHigh, kvs.Get(WatermarkHigh)); highWMStr != "" {
+		cfg.WatermarkHigh, err = strconv.Atoi(highWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+
+		// MaxWatermark should be a valid percentage.
+		if cfg.WatermarkHigh < 0 || cfg.WatermarkHigh > 100 {
+			err := errors.New("config high watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+	}
+	if cfg.WatermarkLow > cfg.WatermarkHigh {
+		err := errors.New("config high watermark value should be greater than low watermark value")
+		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+	}
 	return cfg, nil
 }

cmd/config/config.go

@@ -24,7 +24,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
@@ -621,6 +621,12 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 	currKVS, ok := c[subSys][tgt]
 	if !ok {
 		currKVS = defaultKVS[subSys]
+	} else {
+		for _, kv := range defaultKVS[subSys] {
+			if _, ok = currKVS.Lookup(kv.Key); !ok {
+				currKVS.Set(kv.Key, kv.Value)
+			}
+		}
 	}
 
 	for _, kv := range kvs {

cmd/config/errors-utils.go

@@ -20,7 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net"
 	"syscall"
 
 	"github.com/minio/minio/pkg/color"
@@ -36,6 +36,16 @@ type Err struct {
 	hint   string
 }
 
+// Clone returns a new Err struct with the same information
+func (u Err) Clone() Err {
+	return Err{
+		msg:    u.msg,
+		detail: u.detail,
+		action: u.action,
+		hint:   u.hint,
+	}
+}
+
 // Return the error message
 func (u Err) Error() string {
 	if u.detail == "" {
@@ -49,12 +59,16 @@ func (u Err) Error() string {
 
 // Msg - Replace the current error's message
 func (u Err) Msg(m string, args ...interface{}) Err {
-	return Err{
-		msg:    fmt.Sprintf(m, args...),
-		detail: u.detail,
-		action: u.action,
-		hint:   u.hint,
-	}
+	e := u.Clone()
+	e.msg = fmt.Sprintf(m, args...)
+	return e
+}
+
+// Hint - Replace the current error's message
+func (u Err) Hint(m string, args ...interface{}) Err {
+	e := u.Clone()
+	e.hint = fmt.Sprintf(m, args...)
+	return e
 }
 
 // ErrFn function wrapper
@@ -92,9 +106,11 @@ func ErrorToErr(err error) Err {
 	// Show a generic message for known golang errors
 	if errors.Is(err, syscall.EADDRINUSE) {
 		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
-	} else if errors.Is(err, syscall.EACCES) {
-		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
-	} else if os.IsPermission(err) {
+	} else if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) {
+		switch err.(type) {
+		case *net.OpError:
+			return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+		}
 		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
 	} else if errors.Is(err, io.ErrUnexpectedEOF) {
 		return ErrUnexpectedDataContent(err)

cmd/config/errors.go

@@ -66,6 +66,24 @@ var (
 		"MINIO_CACHE_QUOTA: Valid cache quota value must be between 0-100",
 	)
 
+	ErrInvalidCacheAfter = newErrFn(
+		"Invalid cache after value",
+		"Please check the passed value",
+		"MINIO_CACHE_AFTER: Valid cache after value must be 0 or greater",
+	)
+
+	ErrInvalidCacheWatermarkLow = newErrFn(
+		"Invalid cache low watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_LOW: Valid cache low watermark value must be between 0-100",
+	)
+
+	ErrInvalidCacheWatermarkHigh = newErrFn(
+		"Invalid cache high watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_HIGH: Valid cache high watermark value must be between 0-100",
+	)
+
 	ErrInvalidCacheEncryptionKey = newErrFn(
 		"Invalid cache encryption master key value",
 		"Please check the passed value",

cmd/config/etcd/dns/etcd_dns.go

@@ -38,18 +38,18 @@ var ErrNoEntriesFound = errors.New("No entries found for this key")
 const etcdPathSeparator = "/"
 
 // create a new coredns service record for the bucket.
-func newCoreDNSMsg(ip string, port string, ttl uint32) ([]byte, error) {
+func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, error) {
 	return json.Marshal(&SrvRecord{
 		Host:         ip,
 		Port:         json.Number(port),
 		TTL:          ttl,
-		CreationDate: time.Now().UTC(),
+		CreationDate: t,
 	})
 }
 
 // List - Retrieves list of DNS entries for the domain.
-func (c *CoreDNS) List() ([]SrvRecord, error) {
-	var srvRecords []SrvRecord
+func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
+	var srvRecords = map[string][]SrvRecord{}
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
 		records, err := c.list(key)
@@ -60,7 +60,11 @@ func (c *CoreDNS) List() ([]SrvRecord, error) {
 			if record.Key == "" {
 				continue
 			}
-			srvRecords = append(srvRecords, record)
+			if _, ok := srvRecords[record.Key]; ok {
+				srvRecords[record.Key] = append(srvRecords[record.Key], record)
+			} else {
+				srvRecords[record.Key] = []SrvRecord{record}
+			}
 		}
 	}
 	return srvRecords, nil
@@ -117,9 +121,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		if err != nil {
 			return nil, err
 		}
-		if r.Count == 0 {
-			return nil, ErrNoEntriesFound
-		}
 	}
 
 	var srvRecords []SrvRecord
@@ -143,9 +144,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		srvRecords = append(srvRecords, srvRecord)
 
 	}
-	if len(srvRecords) == 0 {
-		return nil, ErrNoEntriesFound
-	}
 	sort.Slice(srvRecords, func(i int, j int) bool {
 		return srvRecords[i].Key < srvRecords[j].Key
 	})
@@ -156,8 +154,9 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 func (c *CoreDNS) Put(bucket string) error {
 	c.Delete(bucket) // delete any existing entries.
 
+	t := time.Now().UTC()
 	for ip := range c.domainIPs {
-		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL)
+		bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL, t)
 		if err != nil {
 			return err
 		}

cmd/config/identity/openid/help.go

@@ -32,9 +32,15 @@ var (
 			Type:        "string",
 			Optional:    true,
 		},
+		config.HelpKV{
+			Key:         ClaimName,
+			Description: `JWT canned policy claim name, defaults to "policy"`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         ClaimPrefix,
-			Description: `JWT claim namespace prefix e.g. "customer1"`,
+			Description: `JWT claim namespace prefix e.g. "customer1/"`,
 			Optional:    true,
 			Type:        "string",
 		},

cmd/config/identity/openid/jwt.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
@@ -41,6 +42,7 @@ type Config struct {
 	} `json:"jwks"`
 	URL          *xnet.URL `json:"url,omitempty"`
 	ClaimPrefix  string    `json:"claimPrefix,omitempty"`
+	ClaimName    string    `json:"claimName,omitempty"`
 	DiscoveryDoc DiscoveryDoc
 	ClientID     string
 	publicKeys   map[string]crypto.PublicKey
@@ -209,12 +211,14 @@ func (p *JWT) ID() ID {
 const (
 	JwksURL     = "jwks_url"
 	ConfigURL   = "config_url"
+	ClaimName   = "claim_name"
 	ClaimPrefix = "claim_prefix"
 	ClientID    = "client_id"
 
 	EnvIdentityOpenIDClientID    = "MINIO_IDENTITY_OPENID_CLIENT_ID"
 	EnvIdentityOpenIDJWKSURL     = "MINIO_IDENTITY_OPENID_JWKS_URL"
 	EnvIdentityOpenIDURL         = "MINIO_IDENTITY_OPENID_CONFIG_URL"
+	EnvIdentityOpenIDClaimName   = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
 	EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
 )
 
@@ -247,6 +251,7 @@ func parseDiscoveryDoc(u *xnet.URL, transport *http.Transport, closeRespFn func(
 	}
 	resp, err := clnt.Do(req)
 	if err != nil {
+		clnt.CloseIdleConnections()
 		return d, err
 	}
 	defer closeRespFn(resp.Body)
@@ -271,6 +276,10 @@ var (
 			Key:   ClientID,
 			Value: "",
 		},
+		config.KV{
+			Key:   ClaimName,
+			Value: iampolicy.PolicyName,
+		},
 		config.KV{
 			Key:   ClaimPrefix,
 			Value: "",
@@ -299,6 +308,7 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	}
 
 	c = Config{
+		ClaimName:   env.Get(EnvIdentityOpenIDClaimName, kvs.Get(ClaimName)),
 		ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kvs.Get(ClaimPrefix)),
 		publicKeys:  make(map[string]crypto.PublicKey),
 		ClientID:    env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
@@ -317,6 +327,11 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 			return c, err
 		}
 	}
+
+	if c.ClaimName == "" {
+		c.ClaimName = iampolicy.PolicyName
+	}
+
 	if jwksURL == "" {
 		// Fallback to discovery document jwksURL
 		jwksURL = c.DiscoveryDoc.JwksURI
@@ -330,9 +345,11 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	if err != nil {
 		return c, err
 	}
+
 	if err = c.PopulatePublicKey(); err != nil {
 		return c, err
 	}
+
 	return c, nil
 }
 

cmd/config/notify/help.go

@@ -213,6 +213,12 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         target.KafkaVersion,
+			Description: "specify the version of the Kafka cluster",
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/notify/parse.go

@@ -42,7 +42,14 @@ const (
 func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport,
 	targetIDs []event.TargetID) error {
 	test := true
-	_, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test)
+	targets, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test)
+	if err == nil {
+		// Close all targets since we are only testing connections.
+		for _, t := range targets.TargetMap() {
+			_ = t.Close()
+		}
+	}
+
 	return err
 }
 
@@ -58,9 +65,19 @@ func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool) (*event.TargetList, error) {
+func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool) (_ *event.TargetList, err error) {
 	targetList := event.NewTargetList()
-	if err := checkValidNotificationKeys(cfg); err != nil {
+
+	defer func() {
+		// Automatically close all connections to targets when an error occur
+		if err != nil {
+			for _, t := range targetList.TargetMap() {
+				_ = t.Close()
+			}
+		}
+	}()
+
+	if err = checkValidNotificationKeys(cfg); err != nil {
 		return nil, err
 	}
 
@@ -118,23 +135,20 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range esTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 
@@ -142,9 +156,6 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range kafkaTargets {
@@ -152,16 +163,13 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 			continue
 		}
 		args.TLS.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range mqttTargets {
@@ -169,113 +177,91 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 			continue
 		}
 		args.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range mysqlTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range natsTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range nsqTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range postgresTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range redisTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-		}
 	}
 
 	for id, args := range webhookTargets {
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport)
+		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport, test)
 		if err != nil {
 			return nil, err
 		}
 		if err := targetList.Add(newTarget); err != nil {
 			return nil, err
 		}
-		if test {
-			newTarget.Close()
-			continue
-		}
 	}
 
 	if test {
@@ -398,6 +384,10 @@ var (
 			Key:   target.KafkaQueueDir,
 			Value: "",
 		},
+		config.KV{
+			Key:   target.KafkaVersion,
+			Value: "",
+		},
 	}
 )
 
@@ -465,12 +455,18 @@ func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs
 			queueDirEnv = queueDirEnv + config.Default + k
 		}
 
+		versionEnv := target.EnvKafkaVersion
+		if k != config.Default {
+			versionEnv = versionEnv + config.Default + k
+		}
+
 		kafkaArgs := target.KafkaArgs{
 			Enable:     enabled,
 			Brokers:    brokers,
 			Topic:      env.Get(topicEnv, kv.Get(target.KafkaTopic)),
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.KafkaQueueDir)),
 			QueueLimit: queueLimit,
+			Version:    env.Get(versionEnv, kv.Get(target.KafkaVersion)),
 		}
 
 		tlsEnableEnv := target.EnvKafkaTLS

cmd/config/policy/opa/config.go

@@ -223,5 +223,6 @@ func (o *Opa) IsAllowed(args iampolicy.Args) (bool, error) {
 		}
 		return resultAllow.Result.Allow, nil
 	}
+
 	return result.Result, nil
 }

cmd/crypto/config.go

@@ -16,7 +16,6 @@ package crypto
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
 	"reflect"
 	"strconv"
@@ -342,7 +341,7 @@ func LookupVaultConfig(kvs config.KVS) (VaultConfig, error) {
 	if keyVersion := env.Get(EnvKMSVaultKeyVersion, kvs.Get(KMSVaultKeyVersion)); keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
-			return vcfg, fmt.Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
+			return vcfg, Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
 		}
 	}
 

cmd/crypto/error.go

@@ -14,60 +14,79 @@
 
 package crypto
 
-import "errors"
+import (
+	"fmt"
+)
 
 // Error is the generic type for any error happening during decrypting
 // an object. It indicates that the object itself or its metadata was
 // modified accidentally or maliciously.
-type Error string
+type Error struct {
+	err error
+}
 
-func (e Error) Error() string { return string(e) }
+// Errorf - formats according to a format specifier and returns
+// the string as a value that satisfies error of type crypto.Error
+func Errorf(format string, a ...interface{}) error {
+	return Error{err: fmt.Errorf(format, a...)}
+}
+
+// Unwrap the internal error.
+func (e Error) Unwrap() error { return e.err }
+
+// Error 'error' compatible method.
+func (e Error) Error() string {
+	if e.err == nil {
+		return "crypto: cause <nil>"
+	}
+	return e.err.Error()
+}
 
 var (
 	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
 	// is not supported.
-	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+	ErrInvalidEncryptionMethod = Errorf("The encryption method is not supported")
 
 	// ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
 	// is not supported.
-	ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
+	ErrInvalidCustomerAlgorithm = Errorf("The SSE-C algorithm is not supported")
 
 	// ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
-	ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
+	ErrMissingCustomerKey = Errorf("The SSE-C request is missing the customer key")
 
 	// ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
 	// MD5 checksum.
-	ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
+	ErrMissingCustomerKeyMD5 = Errorf("The SSE-C request is missing the customer key MD5")
 
 	// ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
 	// base64-encoded string or not 256 bits long.
-	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
+	ErrInvalidCustomerKey = Errorf("The SSE-C client key is invalid")
 
 	// ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key)
 	// does not match the secret key used during encrypting the object.
-	ErrSecretKeyMismatch = errors.New("The secret key does not match the secret key used during upload")
+	ErrSecretKeyMismatch = Errorf("The secret key does not match the secret key used during upload")
 
 	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
 	// computed MD5 sum. This means that the client provided either the wrong key for
 	// a certain MD5 checksum or the wrong MD5 for a certain key.
-	ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
+	ErrCustomerKeyMD5Mismatch = Errorf("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
 	// ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
 	// The client needs to remove the SSE-S3 header or the SSE-C headers
-	ErrIncompatibleEncryptionMethod = errors.New("Server side encryption specified with both SSE-C and SSE-S3 headers")
+	ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers")
 )
 
-const (
-	errMissingInternalIV            Error = "The object metadata is missing the internal encryption IV"
-	errMissingInternalSealAlgorithm Error = "The object metadata is missing the internal seal algorithm"
+var (
+	errMissingInternalIV            = Errorf("The object metadata is missing the internal encryption IV")
+	errMissingInternalSealAlgorithm = Errorf("The object metadata is missing the internal seal algorithm")
 
-	errInvalidInternalIV            Error = "The internal encryption IV is malformed"
-	errInvalidInternalSealAlgorithm Error = "The internal seal algorithm is invalid and not supported"
+	errInvalidInternalIV            = Errorf("The internal encryption IV is malformed")
+	errInvalidInternalSealAlgorithm = Errorf("The internal seal algorithm is invalid and not supported")
 
-	errMissingUpdatedKey Error = "The key update returned no error but also no sealed key"
+	errMissingUpdatedKey = Errorf("The key update returned no error but also no sealed key")
 )
 
 var (
 	// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
 	// to produce enough random data. This is fatal error and should cause a panic.
-	errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
+	errOutOfEntropy = Errorf("Unable to read enough randomness from the system")
 )

cmd/crypto/kes.go

@@ -19,7 +19,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -76,13 +75,13 @@ type KesConfig struct {
 func (k KesConfig) Verify() (err error) {
 	switch {
 	case k.Endpoint == "":
-		err = errors.New("crypto: missing kes endpoint")
+		err = Errorf("crypto: missing kes endpoint")
 	case k.CertFile == "":
-		err = errors.New("crypto: missing cert file")
+		err = Errorf("crypto: missing cert file")
 	case k.KeyFile == "":
-		err = errors.New("crypto: missing key file")
+		err = Errorf("crypto: missing key file")
 	case k.DefaultKeyID == "":
-		err = errors.New("crypto: missing default key id")
+		err = Errorf("crypto: missing default key id")
 	}
 	return err
 }
@@ -153,7 +152,7 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 		return key, nil, err
 	}
 	if len(plainKey) != len(key) {
-		return key, nil, errors.New("crypto: received invalid plaintext key size from KMS")
+		return key, nil, Errorf("crypto: received invalid plaintext key size from KMS")
 	}
 	copy(key[:], plainKey)
 	return key, sealedKey, nil
@@ -176,7 +175,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 		return key, err
 	}
 	if len(plainKey) != len(key) {
-		return key, errors.New("crypto: received invalid plaintext key size from KMS")
+		return key, Errorf("crypto: received invalid plaintext key size from KMS")
 	}
 	copy(key[:], plainKey)
 	return key, nil
@@ -301,7 +300,7 @@ func (c *kesClient) parseErrorResponse(resp *http.Response) error {
 	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
 		return err
 	}
-	return fmt.Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
+	return Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
 }
 
 // loadCACertificates returns a new CertPool
@@ -334,7 +333,7 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 		if os.IsNotExist(err) || os.IsPermission(err) {
 			return rootCAs, nil
 		}
-		return nil, fmt.Errorf("crypto: cannot open '%s': %v", path, err)
+		return nil, Errorf("crypto: cannot open '%s': %v", path, err)
 	}
 
 	// If path is a file, parse as PEM-encoded certifcate
@@ -346,7 +345,7 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 			return nil, err
 		}
 		if !rootCAs.AppendCertsFromPEM(cert) {
-			return nil, fmt.Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
+			return nil, Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
 		}
 		return rootCAs, nil
 	}

cmd/crypto/key.go

@@ -21,7 +21,6 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io"
 	"path"
 
@@ -108,7 +107,7 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 	)
 	switch sealedKey.Algorithm {
 	default:
-		return Error(fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm))
+		return Errorf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)
 	case SealAlgorithm:
 		mac := hmac.New(sha256.New, extKey[:])
 		mac.Write(sealedKey.IV[:])

cmd/crypto/legacy.go

@@ -17,7 +17,6 @@
 package crypto
 
 import (
-	"fmt"
 	"reflect"
 	"strconv"
 
@@ -167,7 +166,7 @@ func lookupConfigLegacy(kvs config.KVS) (VaultConfig, error) {
 	if keyVersion := env.Get(EnvLegacyVaultKeyVersion, ""); keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
-			return vcfg, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
+			return vcfg, Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
 				EnvLegacyVaultKeyVersion, keyVersion)
 		}
 	}

cmd/crypto/metadata.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/base64"
 	"errors"
-	"fmt"
 
 	"github.com/minio/minio/cmd/logger"
 )
@@ -126,7 +125,7 @@ func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 // is nil.
 func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
 	}
 
 	// There are two possibilites:
@@ -172,7 +171,7 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	b64SealedKey, ok := metadata[S3SealedKey]
 	if !ok {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed key for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
 	}
 
 	// There are two possibilites:
@@ -182,10 +181,10 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	keyID, idPresent := metadata[S3KMSKeyID]
 	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
 	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal KMS key-ID for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
 	}
 	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -198,12 +197,12 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Error("The internal sealed key for SSE-S3 is invalid")
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
 		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
 		if err != nil {
-			return keyID, kmsKey, sealedKey, Error("The internal sealed KMS data key for SSE-S3 is invalid")
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
 		}
 	}
 
@@ -217,7 +216,7 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 // It allocates a new metadata map if metadata is nil.
 func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
 	}
 
 	if metadata == nil {
@@ -244,7 +243,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	b64SealedKey, ok := metadata[SSECSealedKey]
 	if !ok {
-		return sealedKey, Error("The object metadata is missing the internal sealed key for SSE-C")
+		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -257,7 +256,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Error("The internal sealed key for SSE-C is invalid")
+		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
 	}
 
 	sealedKey.Algorithm = algorithm

cmd/crypto/metadata_test.go

@@ -125,15 +125,15 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
-		ExpectedErr: Error("The object metadata is missing the internal KMS key-ID for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
@@ -150,7 +150,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
-		ExpectedErr: Error("The internal sealed key for SSE-S3 is invalid"),
+		ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
 			S3KMSKeyID: "", S3KMSSealedKey: "",
@@ -158,7 +158,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
-		ExpectedErr: Error("The internal sealed KMS data key for SSE-S3 is invalid"),
+		ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
 			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
@@ -188,9 +188,17 @@ var s3ParseMetadataTests = []struct {
 func TestS3ParseMetadata(t *testing.T) {
 	for i, test := range s3ParseMetadataTests {
 		keyID, dataKey, sealedKey, err := S3.ParseMetadata(test.Metadata)
-		if err != test.ExpectedErr {
+		if err != nil && test.ExpectedErr == nil {
 			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
 		}
+		if err == nil && test.ExpectedErr != nil {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if err != nil && test.ExpectedErr != nil {
+			if err.Error() != test.ExpectedErr.Error() {
+				t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+			}
+		}
 		if !bytes.Equal(dataKey, test.DataKey) {
 			t.Errorf("Test %d: got data key '%v' - want data key '%v'", i, dataKey, test.DataKey)
 		}
@@ -218,7 +226,7 @@ var ssecParseMetadataTests = []struct {
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
 	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-C"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
@@ -233,7 +241,7 @@ var ssecParseMetadataTests = []struct {
 		SealedKey: SealedKey{},
 	}, // 4
 	{
-		ExpectedErr: Error("The internal sealed key for SSE-C is invalid"),
+		ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
 		},
@@ -267,9 +275,17 @@ func TestCreateMultipartMetadata(t *testing.T) {
 func TestSSECParseMetadata(t *testing.T) {
 	for i, test := range ssecParseMetadataTests {
 		sealedKey, err := SSEC.ParseMetadata(test.Metadata)
-		if err != test.ExpectedErr {
+		if err != nil && test.ExpectedErr == nil {
 			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
 		}
+		if err == nil && test.ExpectedErr != nil {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if err != nil && test.ExpectedErr != nil {
+			if err.Error() != test.ExpectedErr.Error() {
+				t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+			}
+		}
 		if sealedKey.Algorithm != test.SealedKey.Algorithm {
 			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
 		}

cmd/crypto/parse.go

@@ -16,7 +16,6 @@ package crypto
 
 import (
 	"encoding/hex"
-	"fmt"
 	"strings"
 )
 
@@ -25,18 +24,18 @@ import (
 func ParseMasterKey(envArg string) (KMS, error) {
 	values := strings.SplitN(envArg, ":", 2)
 	if len(values) != 2 {
-		return nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
+		return nil, Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
 	}
 	var (
 		keyID  = values[0]
 		hexKey = values[1]
 	)
 	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
-		return nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
+		return nil, Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
 	}
 	var masterKey [32]byte
 	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
-		return nil, err
+		return nil, Errorf("Invalid KMS master key: %v", err)
 	}
 	return NewMasterKey(keyID, masterKey), nil
 }

cmd/crypto/vault.go

@@ -17,7 +17,6 @@ package crypto
 import (
 	"bytes"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -27,7 +26,7 @@ import (
 
 var (
 	//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
-	ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
+	ErrKMSAuthLogin = Errorf("Vault service did not return auth info")
 )
 
 // VaultKey represents vault encryption key-ring.
@@ -75,17 +74,17 @@ var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements
 func (v *VaultConfig) Verify() (err error) {
 	switch {
 	case v.Endpoint == "":
-		err = errors.New("crypto: missing hashicorp vault endpoint")
+		err = Errorf("crypto: missing hashicorp vault endpoint")
 	case strings.ToLower(v.Auth.Type) != "approle":
-		err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
+		err = Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
 	case v.Auth.AppRole.ID == "":
-		err = errors.New("crypto: missing hashicorp vault AppRole ID")
+		err = Errorf("crypto: missing hashicorp vault AppRole ID")
 	case v.Auth.AppRole.Secret == "":
-		err = errors.New("crypto: missing hashicorp vault AppSecret ID")
+		err = Errorf("crypto: missing hashicorp vault AppSecret ID")
 	case v.Key.Name == "":
-		err = errors.New("crypto: missing hashicorp vault key name")
+		err = Errorf("crypto: missing hashicorp vault key name")
 	case v.Key.Version < 0:
-		err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
+		err = Errorf("crypto: invalid hashicorp vault key version: The key version must not be negative")
 	}
 	return
 }
@@ -107,7 +106,7 @@ func NewVault(config VaultConfig) (KMS, error) {
 	}
 	client, err := vault.NewClient(&vaultCfg)
 	if err != nil {
-		return nil, err
+		return nil, Errorf("crypto: client error %w", err)
 	}
 	if config.Namespace != "" {
 		client.SetNamespace(config.Namespace)
@@ -167,6 +166,7 @@ func (v *vaultService) authenticate() (err error) {
 	var secret *vault.Secret
 	secret, err = v.client.Logical().Write("auth/approle/login", payload)
 	if err != nil {
+		err = Errorf("crypto: client error %w", err)
 		return
 	}
 	if secret == nil {
@@ -217,14 +217,23 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
-		return key, sealedKey, err
+		return key, sealedKey, Errorf("crypto: client error %w", err)
 	}
-	sealKey := s.Data["ciphertext"].(string)
-	plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
+	sealKey, ok := s.Data["ciphertext"].(string)
+	if !ok {
+		return key, sealedKey, Errorf("crypto: incorrect 'ciphertext' key type %v", s.Data["ciphertext"])
+	}
+
+	plainKeyB64, ok := s.Data["plaintext"].(string)
+	if !ok {
+		return key, sealedKey, Errorf("crypto: incorrect 'plaintext' key type %v", s.Data["plaintext"])
+	}
+
+	plainKey, err := base64.StdEncoding.DecodeString(plainKeyB64)
 	if err != nil {
-		return key, sealedKey, err
+		return key, sealedKey, Errorf("crypto: invalid base64 key %w", err)
 	}
-	copy(key[:], []byte(plainKey))
+	copy(key[:], plainKey)
 	return key, []byte(sealKey), nil
 }
 
@@ -243,16 +252,22 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 		"ciphertext": string(sealedKey),
 		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
 	}
+
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
 	if err != nil {
-		return key, err
+		return key, Errorf("crypto: client error %w", err)
 	}
-	base64Key := s.Data["plaintext"].(string)
+
+	base64Key, ok := s.Data["plaintext"].(string)
+	if !ok {
+		return key, Errorf("crypto: incorrect 'plaintext' key type %v", s.Data["plaintext"])
+	}
+
 	plainKey, err := base64.StdEncoding.DecodeString(base64Key)
 	if err != nil {
-		return key, err
+		return key, Errorf("crypto: invalid base64 key %w", err)
 	}
-	copy(key[:], []byte(plainKey))
+	copy(key[:], plainKey)
 	return key, nil
 }
 
@@ -273,7 +288,7 @@ func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (r
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {
-		return nil, err
+		return nil, Errorf("crypto: client error %w", err)
 	}
 	ciphertext, ok := s.Data["ciphertext"]
 	if !ok {

cmd/daily-lifecycle-ops.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/lifecycle"
 )
 
 const (
@@ -104,12 +104,12 @@ func startDailyLifecycle() {
 	}
 }
 
-var lifecycleTimeout = newDynamicTimeout(60*time.Second, time.Second)
+var lifecycleLockTimeout = newDynamicTimeout(60*time.Second, time.Second)
 
 func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 	// Lock to avoid concurrent lifecycle ops from other nodes
 	sweepLock := objAPI.NewNSLock(ctx, "system", "daily-lifecycle-ops")
-	if err := sweepLock.GetLock(lifecycleTimeout); err != nil {
+	if err := sweepLock.GetLock(lifecycleLockTimeout); err != nil {
 		return err
 	}
 	defer sweepLock.Unlock()
@@ -129,29 +129,39 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 		// Calculate the common prefix of all lifecycle rules
 		var prefixes []string
 		for _, rule := range l.Rules {
-			prefixes = append(prefixes, rule.Filter.Prefix)
+			prefixes = append(prefixes, rule.Prefix())
 		}
 		commonPrefix := lcp(prefixes)
 
-		// List all objects and calculate lifecycle action based on object name & object modtime
-		marker := ""
+		// Allocate new results channel to receive ObjectInfo.
+		objInfoCh := make(chan ObjectInfo)
+
+		// Walk through all objects
+		if err := objAPI.Walk(ctx, bucket.Name, commonPrefix, objInfoCh); err != nil {
+			return err
+		}
+
 		for {
-			res, err := objAPI.ListObjects(ctx, bucket.Name, commonPrefix, marker, "", maxObjectList)
-			if err != nil {
-				continue
-			}
 			var objects []string
-			for _, obj := range res.Objects {
+			for obj := range objInfoCh {
+				if len(objects) == maxObjectList {
+					// Reached maximum delete requests, attempt a delete for now.
+					break
+				}
+
 				// Find the action that need to be executed
-				action := l.ComputeAction(obj.Name, obj.ModTime)
-				switch action {
-				case lifecycle.DeleteAction:
+				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
 					objects = append(objects, obj.Name)
-				default:
-					// Do nothing, for now.
 				}
 			}
 
+			// Nothing to do.
+			if len(objects) == 0 {
+				break
+			}
+
+			waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
+
 			// Deletes a list of objects.
 			deleteErrs, err := objAPI.DeleteObjects(ctx, bucket.Name, objects)
 			if err != nil {
@@ -173,12 +183,6 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 					})
 				}
 			}
-
-			if !res.IsTruncated {
-				// We are done here, proceed to next bucket.
-				break
-			}
-			marker = res.NextMarker
 		}
 	}
 

cmd/data-usage.go

@@ -20,18 +20,28 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"os"
+	"path/filepath"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
 )
 
 const (
 	dataUsageObjName       = "data-usage"
 	dataUsageCrawlInterval = 12 * time.Hour
+	dataUsageCrawlConf     = "MINIO_DISK_USAGE_CRAWL"
 )
 
 func initDataUsageStats() {
+	dataUsageEnabled, err := config.ParseBool(env.Get(dataUsageCrawlConf, config.EnableOn))
+	if err == nil && !dataUsageEnabled {
+		return
+	}
 	go runDataUsageInfoUpdateRoutine()
 }
 
@@ -47,63 +57,60 @@ func runDataUsageInfoUpdateRoutine() {
 		break
 	}
 
-	ctx := context.Background()
-
-	switch v := objAPI.(type) {
-	case *xlZones:
-		runDataUsageInfoForXLZones(ctx, v, GlobalServiceDoneCh)
-	case *FSObjects:
-		runDataUsageInfoForFS(ctx, v, GlobalServiceDoneCh)
-	default:
-		return
-	}
+	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
 }
 
-func runDataUsageInfoForFS(ctx context.Context, fsObj *FSObjects, endCh <-chan struct{}) {
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
-	for {
-		// Get data usage info of the FS Object
-		usageInfo := fsObj.crawlAndGetDataUsageInfo(ctx, endCh)
-		// Save the data usage in the disk
-		err := storeDataUsageInBackend(ctx, fsObj, usageInfo)
-		if err != nil {
-			logger.LogIf(ctx, err)
-		}
-		select {
-		case <-endCh:
-			return
-		// Wait until the next crawl interval
-		case <-t.C:
-		}
+// timeToNextCrawl returns the duration until next crawl should occur
+// this is validated by verifying the LastUpdate time.
+func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
+	if err != nil {
+		// Upon an error wait for like 10
+		// seconds to start the crawler.
+		return 10 * time.Second
 	}
+	// File indeed doesn't exist when LastUpdate is zero
+	// so we have never crawled, start crawl right away.
+	if dataUsageInfo.LastUpdate.IsZero() {
+		return 1 * time.Second
+	}
+	timeSinceLastUpdate := UTCNow().Sub(dataUsageInfo.LastUpdate)
+	if timeSinceLastUpdate > dataUsageCrawlInterval {
+		// Waited long enough start crawl in a 1 second
+		return 1 * time.Second
+	}
+	// No crawling needed, ask the routine to wait until
+	// the daily interval 12hrs - delta between last update
+	// with current time.
+	return dataUsageCrawlInterval - timeSinceLastUpdate
 }
 
-func runDataUsageInfoForXLZones(ctx context.Context, z *xlZones, endCh <-chan struct{}) {
-	locker := z.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
+var dataUsageLockTimeout = lifecycleLockTimeout
+
+func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
+	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
 	for {
-		err := locker.GetLock(newDynamicTimeout(time.Millisecond, time.Millisecond))
+		err := locker.GetLock(dataUsageLockTimeout)
 		if err != nil {
 			time.Sleep(5 * time.Minute)
 			continue
 		}
-		// Break without locking
+		// Break without unlocking, this node will acquire
+		// data usage calculator role for its lifetime.
 		break
 	}
 
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
 	for {
-		usageInfo := z.crawlAndGetDataUsage(ctx, endCh)
-		err := storeDataUsageInBackend(ctx, z, usageInfo)
-		if err != nil {
-			logger.LogIf(ctx, err)
-		}
+		wait := timeToCrawl(ctx, objAPI)
 		select {
 		case <-endCh:
 			locker.Unlock()
 			return
-		case <-t.C:
+		case <-time.NewTimer(wait).C:
+			// Crawl only when no previous crawl has occurred,
+			// or its been too long since last crawl.
+			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
+			logger.LogIf(ctx, err)
 		}
 	}
 }
@@ -129,10 +136,14 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
 	if err != nil {
-		return DataUsageInfo{}, nil
+		if isErrObjectNotFound(err) {
+			return DataUsageInfo{}, nil
+		}
+		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
 	}
 
 	var dataUsageInfo DataUsageInfo
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
 	if err != nil {
 		return DataUsageInfo{}, err
@@ -140,3 +151,61 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	return dataUsageInfo, nil
 }
+
+// Item represents each file while walking.
+type Item struct {
+	Path string
+	Typ  os.FileMode
+}
+
+type getSizeFn func(item Item) (int64, error)
+
+func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO func(), getSize getSizeFn) DataUsageInfo {
+	var dataUsageInfo = DataUsageInfo{
+		BucketsSizes:          make(map[string]uint64),
+		ObjectsSizesHistogram: make(map[string]uint64),
+	}
+
+	fastWalk(basePath, 1, doneCh, func(path string, typ os.FileMode) error {
+		// Wait for I/O to go down.
+		waitForLowActiveIO()
+
+		bucket, entry := path2BucketObjectWithBasePath(basePath, path)
+		if bucket == "" {
+			return nil
+		}
+
+		if isReservedOrInvalidBucket(bucket, false) {
+			return filepath.SkipDir
+		}
+
+		if entry == "" && typ&os.ModeDir != 0 {
+			dataUsageInfo.BucketsCount++
+			dataUsageInfo.BucketsSizes[bucket] = 0
+			return nil
+		}
+
+		if typ&os.ModeDir != 0 {
+			return nil
+		}
+
+		t := time.Now()
+		size, err := getSize(Item{path, typ})
+		// Use the response time of the getSize call to guess system load.
+		// Sleep equivalent time.
+		if d := time.Since(t); d > 100*time.Microsecond {
+			time.Sleep(d)
+		}
+		if err != nil {
+			return errSkipFile
+		}
+
+		dataUsageInfo.ObjectsCount++
+		dataUsageInfo.ObjectsTotalSize += uint64(size)
+		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
+		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
+		return nil
+	})
+
+	return dataUsageInfo
+}

cmd/disk-cache-backend.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,7 +27,7 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"reflect"
+	"strings"
 	"sync"
 	"time"
 
@@ -37,7 +37,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/sio"
-	"github.com/ncw/directio"
+	"go.uber.org/atomic"
 )
 
 const (
@@ -45,7 +45,7 @@ const (
 	cacheMetaJSONFile = "cache.json"
 	cacheDataFile     = "part.1"
 	cacheMetaVersion  = "1.0.0"
-
+	cacheExpiryDays   = time.Duration(90 * time.Hour * 24) // defaults to 90 days
 	// SSECacheEncrypted is the metadata key indicating that the object
 	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
 	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
@@ -68,6 +68,8 @@ type cacheMeta struct {
 	Meta map[string]string `json:"meta,omitempty"`
 	// Ranges maps cached range to associated filename.
 	Ranges map[string]string `json:"ranges,omitempty"`
+	// Hits is a counter on the number of times this object has been accessed so far.
+	Hits int `json:"hits,omitempty"`
 }
 
 // RangeInfo has the range, file and range length information for a cached range.
@@ -89,8 +91,10 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 	}
 
 	o = ObjectInfo{
-		Bucket: bucket,
-		Name:   object,
+		Bucket:            bucket,
+		Name:              object,
+		CacheStatus:       CacheHit,
+		CacheLookupStatus: CacheHit,
 	}
 
 	// We set file info only if its valid.
@@ -123,44 +127,54 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 type diskCache struct {
 	dir      string // caching directory
 	quotaPct int    // max usage in %
-	expiry   int    // cache expiry in days
 	// mark false if drive is offline
 	online bool
 	// mutex to protect updates to online variable
-	onlineMutex *sync.RWMutex
-	// purge() listens on this channel to start the cache-purge process
-	purgeChan chan struct{}
-	pool      sync.Pool
+	onlineMutex   *sync.RWMutex
+	pool          sync.Pool
+	after         int // minimum accesses before an object is cached.
+	lowWatermark  int
+	highWatermark int
+	gcCounter     atomic.Uint64
+	// nsMutex namespace lock
+	nsMutex *nsLockMap
+	// Object functions pointing to the corresponding functions of backend implementation.
+	NewNSLockFn func(ctx context.Context, cachePath string) RWLocker
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(dir string, expiry int, quotaPct int) (*diskCache, error) {
+func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
-		dir:         dir,
-		expiry:      expiry,
-		quotaPct:    quotaPct,
-		purgeChan:   make(chan struct{}),
-		online:      true,
-		onlineMutex: &sync.RWMutex{},
+		dir:           dir,
+		quotaPct:      quotaPct,
+		after:         after,
+		lowWatermark:  lowWatermark,
+		highWatermark: highWatermark,
+		online:        true,
+		onlineMutex:   &sync.RWMutex{},
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := directio.AlignedBlock(int(cacheBlkSize))
+				b := disk.AlignedBlock(int(cacheBlkSize))
 				return &b
 			},
 		},
+		nsMutex: newNSLock(false),
+	}
+	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
+		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
 	}
 	return &cache, nil
 }
 
-// Returns if the disk usage is low.
-// Disk usage is low if usage is < 80% of cacheMaxDiskUsagePct
-// Ex. for a 100GB disk, if maxUsage is configured as 70% then cacheMaxDiskUsagePct is 70G
-// hence disk usage is low if the disk usage is less than 56G (because 80% of 70G is 56G)
+// diskUsageLow() returns true if disk usage falls below the low watermark w.r.t configured cache quota.
+// Ex. for a 100GB disk, if quota is configured as 70%  and watermark_low = 80% and
+// watermark_high = 90% then garbage collection starts when 63% of disk is used and
+// stops when disk usage drops to 56%
 func (c *diskCache) diskUsageLow() bool {
-	minUsage := c.quotaPct * 80 / 100
+	gcStopPct := c.quotaPct * c.lowWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
@@ -169,21 +183,22 @@ func (c *diskCache) diskUsageLow() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < minUsage
+	return int(usedPercent) < gcStopPct
 }
 
-// Return if the disk usage is high.
-// Disk usage is high if disk used is > cacheMaxDiskUsagePct
+// Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
+// gc starts if high water mark reached.
 func (c *diskCache) diskUsageHigh() bool {
+	gcTriggerPct := c.quotaPct * c.highWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
 		logger.LogIf(ctx, err)
-		return true
+		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) > c.quotaPct
+	return int(usedPercent) >= gcTriggerPct
 }
 
 // Returns if size space can be allocated without exceeding
@@ -200,12 +215,42 @@ func (c *diskCache) diskAvailable(size int64) bool {
 	return int(usedPercent) < c.quotaPct
 }
 
+// toClear returns how many bytes should be cleared to reach the low watermark quota.
+// returns 0 if below quota.
+func (c *diskCache) toClear() uint64 {
+	di, err := disk.GetInfo(c.dir)
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		return 0
+	}
+	return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark))
+}
+
 // Purge cache entries that were not accessed.
-func (c *diskCache) purge() {
+func (c *diskCache) purge(ctx context.Context, doneCh <-chan struct{}) {
+	if c.diskUsageLow() {
+		return
+	}
+	toFree := c.toClear()
+	if toFree == 0 {
+		return
+	}
+	// expiry for cleaning up old cache.json files that
+	// need to be cleaned up.
+	expiry := UTCNow().Add(-cacheExpiryDays)
+	// defaulting max hits count to 100
+	scorer, err := newFileScorer(int64(toFree), time.Now().Unix(), 100)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+
 	// this function returns FileInfo for cached range files and cache data file.
 	fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
 		fm := make(map[string]os.FileInfo)
-		fname := pathJoin(pathPrefix, cacheDataFile)
+		fname := pathJoin(pathPrefix, dataFile)
 		if fi, err := os.Stat(fname); err == nil {
 			fm[fname] = fi
 		}
@@ -218,65 +263,73 @@ func (c *diskCache) purge() {
 		}
 		return fm
 	}
-	ctx := context.Background()
-	for {
-		olderThan := c.expiry * 24
-		for !c.diskUsageLow() {
-			// delete unaccessed objects older than expiry duration
-			expiry := UTCNow().Add(time.Hour * time.Duration(-1*olderThan))
-			olderThan /= 2
-			if olderThan < 1 {
-				break
-			}
-			deletedCount := 0
+	objDirs, err := ioutil.ReadDir(c.dir)
+	if err != nil {
+		log.Fatal(err)
+	}
 
-			objDirs, err := ioutil.ReadDir(c.dir)
-			if err != nil {
-				log.Fatal(err)
-			}
+	for _, obj := range objDirs {
+		if obj.Name() == minioMetaBucket {
+			continue
+		}
 
-			for _, obj := range objDirs {
-				if obj.Name() == minioMetaBucket {
-					continue
-				}
-				meta, _, err := c.statCachedMeta(pathJoin(c.dir, obj.Name()))
-				if err != nil {
-					// delete any partially filled cache entry left behind.
-					removeAll(pathJoin(c.dir, obj.Name()))
-					continue
-				}
-				// stat all cached file ranges and cacheDataFile.
-				fis := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
-				objInfo := meta.ToObjectInfo("", "")
-				cc := cacheControlOpts(objInfo)
-
-				for fname, fi := range fis {
-					if atime.Get(fi).Before(expiry) ||
-						cc.isStale(objInfo.ModTime) {
-						if err = removeAll(fname); err != nil {
-							logger.LogIf(ctx, err)
-						}
-						deletedCount++
-						// break early if sufficient disk space reclaimed.
-						if !c.diskUsageLow() {
-							break
-						}
+		cacheDir := pathJoin(c.dir, obj.Name())
+		meta, _, numHits, err := c.statCachedMeta(ctx, cacheDir)
+		if err != nil {
+			// delete any partially filled cache entry left behind.
+			removeAll(cacheDir)
+			continue
+		}
+		// stat all cached file ranges and cacheDataFile.
+		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
+		objInfo := meta.ToObjectInfo("", "")
+		cc := cacheControlOpts(objInfo)
+		for fname, fi := range cachedFiles {
+			if cc != nil {
+				if cc.isStale(objInfo.ModTime) {
+					if err = removeAll(fname); err != nil {
+						logger.LogIf(ctx, err)
+					}
+					scorer.adjustSaveBytes(-fi.Size())
+					// break early if sufficient disk space reclaimed.
+					if c.diskUsageLow() {
+						return
 					}
 				}
+				continue
 			}
-			if deletedCount == 0 {
-				break
-			}
+			scorer.addFile(fname, atime.Get(fi), fi.Size(), numHits)
 		}
-		lastRunTime := time.Now()
-		for {
-			<-c.purgeChan
-			timeElapsed := time.Since(lastRunTime)
-			if timeElapsed > time.Hour {
-				break
-			}
+		// clean up stale cache.json files for objects that never got cached but access count was maintained in cache.json
+		fi, err := os.Stat(pathJoin(cacheDir, cacheMetaJSONFile))
+		if err != nil || (fi.ModTime().Before(expiry) && len(cachedFiles) == 0) {
+			removeAll(cacheDir)
+			scorer.adjustSaveBytes(-fi.Size())
+			continue
+		}
+		if c.diskUsageLow() {
+			return
 		}
 	}
+	for _, path := range scorer.fileNames() {
+		removeAll(path)
+		slashIdx := strings.LastIndex(path, SlashSeparator)
+		pathPrefix := path[0:slashIdx]
+		fname := path[slashIdx+1:]
+		if fname == cacheDataFile {
+			removeAll(pathPrefix)
+		}
+	}
+}
+
+func (c *diskCache) incGCCounter() {
+	c.gcCounter.Add(uint64(1))
+}
+func (c *diskCache) resetGCCounter() {
+	c.gcCounter.Store(uint64(0))
+}
+func (c *diskCache) gcCount() uint64 {
+	return c.gcCounter.Load()
 }
 
 // sets cache drive status
@@ -294,63 +347,67 @@ func (c *diskCache) IsOnline() bool {
 }
 
 // Stat returns ObjectInfo from disk cache
-func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectInfo, err error) {
+func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectInfo, numHits int, err error) {
+	var partial bool
+	var meta *cacheMeta
+
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	oi, err = c.statCache(cacheObjPath)
+	// Stat the file to get file size.
+	meta, partial, numHits, err = c.statCachedMeta(ctx, cacheObjPath)
 	if err != nil {
 		return
 	}
+	if partial {
+		return oi, numHits, errFileNotFound
+	}
+	oi = meta.ToObjectInfo("", "")
 	oi.Bucket = bucket
 	oi.Name = object
 
 	if err = decryptCacheObjectETag(&oi); err != nil {
-		return oi, err
+		return
 	}
 	return
 }
 
 // statCachedMeta returns metadata from cache - including ranges cached, partial to indicate
 // if partial object is cached.
-func (c *diskCache) statCachedMeta(cacheObjPath string) (meta *cacheMeta, partial bool, err error) {
-	// Stat the file to get file size.
-	metaPath := pathJoin(cacheObjPath, cacheMetaJSONFile)
-	f, err := os.Open(metaPath)
-	if err != nil {
-		return meta, partial, err
+func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
+
+	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	if err = cLock.GetRLock(globalObjectTimeout); err != nil {
+		return
 	}
-	defer f.Close()
-	meta = &cacheMeta{Version: cacheMetaVersion}
-	if err := jsonLoad(f, meta); err != nil {
-		return meta, partial, err
-	}
-	// get metadata of part.1 if full file has been cached.
-	partial = true
-	fi, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile))
-	if err == nil {
-		meta.Stat.ModTime = atime.Get(fi)
-		partial = false
-	}
-	return meta, partial, nil
+
+	defer cLock.RUnlock()
+	return c.statCache(ctx, cacheObjPath)
 }
 
 // statRange returns ObjectInfo and RangeInfo from disk cache
-func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HTTPRangeSpec) (oi ObjectInfo, rngInfo RangeInfo, err error) {
+func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HTTPRangeSpec) (oi ObjectInfo, rngInfo RangeInfo, numHits int, err error) {
 	// Stat the file to get file size.
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
+	var meta *cacheMeta
+	var partial bool
 
-	if rs == nil {
-		oi, err = c.statCache(cacheObjPath)
-		return oi, rngInfo, err
-	}
-	meta, _, err := c.statCachedMeta(cacheObjPath)
+	meta, partial, numHits, err = c.statCachedMeta(ctx, cacheObjPath)
 	if err != nil {
-		return oi, rngInfo, err
+		return
+	}
+
+	oi = meta.ToObjectInfo("", "")
+	oi.Bucket = bucket
+	oi.Name = object
+	if !partial {
+		err = decryptCacheObjectETag(&oi)
+		return
 	}
 
 	actualSize := uint64(meta.Stat.Size)
-	_, length, err := rs.GetOffsetLength(int64(actualSize))
+	var length int64
+	_, length, err = rs.GetOffsetLength(int64(actualSize))
 	if err != nil {
-		return oi, rngInfo, err
+		return
 	}
 
 	actualRngSize := uint64(length)
@@ -361,38 +418,61 @@ func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HT
 	rng := rs.String(int64(actualSize))
 	rngFile, ok := meta.Ranges[rng]
 	if !ok {
-		return oi, rngInfo, ObjectNotFound{Bucket: bucket, Object: object}
+		return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
+	}
+	if _, err = os.Stat(pathJoin(cacheObjPath, rngFile)); err != nil {
+		return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
 	}
 	rngInfo = RangeInfo{Range: rng, File: rngFile, Size: int64(actualRngSize)}
 
-	oi = meta.ToObjectInfo("", "")
-	oi.Bucket = bucket
-	oi.Name = object
-
-	if err = decryptCacheObjectETag(&oi); err != nil {
-		return oi, rngInfo, err
-	}
+	err = decryptCacheObjectETag(&oi)
 	return
 }
 
 // statCache is a convenience function for purge() to get ObjectInfo for cached object
-func (c *diskCache) statCache(cacheObjPath string) (oi ObjectInfo, e error) {
+func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
 	// Stat the file to get file size.
-	meta, partial, err := c.statCachedMeta(cacheObjPath)
+	metaPath := pathJoin(cacheObjPath, cacheMetaJSONFile)
+	f, err := os.Open(metaPath)
 	if err != nil {
-		return oi, err
+		return meta, partial, 0, err
 	}
-	if partial {
-		return oi, errFileNotFound
+	defer f.Close()
+	meta = &cacheMeta{Version: cacheMetaVersion}
+	if err := jsonLoad(f, meta); err != nil {
+		return meta, partial, 0, err
 	}
-	return meta.ToObjectInfo("", ""), nil
+	// get metadata of part.1 if full file has been cached.
+	partial = true
+	fi, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile))
+	if err == nil {
+		meta.Stat.ModTime = atime.Get(fi)
+		partial = false
+	}
+	return meta, partial, meta.Hits, nil
 }
 
 // saves object metadata to disk cache
-func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string) error {
-	fileName := getCacheSHADir(c.dir, bucket, object)
-	metaPath := pathJoin(fileName, cacheMetaJSONFile)
+// incHitsOnly is true if metadata update is incrementing only the hit counter
+func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
+	cachedPath := getCacheSHADir(c.dir, bucket, object)
+	cLock := c.NewNSLockFn(ctx, cachedPath)
+	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer cLock.Unlock()
+	return c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly)
+}
 
+// saves object metadata to disk cache
+// incHitsOnly is true if metadata update is incrementing only the hit counter
+func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
+	cachedPath := getCacheSHADir(c.dir, bucket, object)
+	metaPath := pathJoin(cachedPath, cacheMetaJSONFile)
+	// Create cache directory if needed
+	if err := os.MkdirAll(cachedPath, 0777); err != nil {
+		return err
+	}
 	f, err := os.OpenFile(metaPath, os.O_RDWR|os.O_CREATE, 0666)
 	if err != nil {
 		return err
@@ -403,6 +483,7 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	if err := jsonLoad(f, m); err != nil && err != io.EOF {
 		return err
 	}
+	// increment hits
 	if rs != nil {
 		if m.Ranges == nil {
 			m.Ranges = make(map[string]string)
@@ -411,46 +492,30 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	} else {
 		// this is necessary cleanup of range files if entire object is cached.
 		for _, f := range m.Ranges {
-			removeAll(pathJoin(fileName, f))
+			removeAll(pathJoin(cachedPath, f))
 		}
 		m.Ranges = nil
 	}
 	m.Stat.Size = actualSize
 	m.Stat.ModTime = UTCNow()
-	m.Meta = meta
+	if !incHitsOnly {
+		// reset meta
+		m.Meta = meta
+	} else {
+		if m.Meta == nil {
+			m.Meta = make(map[string]string)
+		}
+		if etag, ok := meta["etag"]; ok {
+			m.Meta["etag"] = etag
+		}
+	}
+
+	m.Hits++
+
 	m.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
-
 	return jsonSave(f, m)
 }
 
-// Backend metadata could have changed through server side copy - reset cache metadata if that is the case
-func (c *diskCache) updateMetadataIfChanged(ctx context.Context, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
-
-	bkMeta := make(map[string]string)
-	cacheMeta := make(map[string]string)
-	for k, v := range bkObjectInfo.UserDefined {
-		if HasPrefix(k, ReservedMetadataPrefix) {
-			// Do not need to send any internal metadata
-			continue
-		}
-		bkMeta[http.CanonicalHeaderKey(k)] = v
-	}
-	for k, v := range cacheObjInfo.UserDefined {
-		if HasPrefix(k, ReservedMetadataPrefix) {
-			// Do not need to send any internal metadata
-			continue
-		}
-		cacheMeta[http.CanonicalHeaderKey(k)] = v
-	}
-	if !reflect.DeepEqual(bkMeta, cacheMeta) ||
-		bkObjectInfo.ETag != cacheObjInfo.ETag ||
-		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
-		bkObjectInfo.Expires != cacheObjInfo.Expires {
-		return c.saveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "")
-	}
-	return nil
-}
-
 func getCacheSHADir(dir, bucket, object string) string {
 	return pathJoin(dir, getSHA256Hash([]byte(pathJoin(bucket, object))))
 }
@@ -546,21 +611,40 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 }
 
 // Caches the object to disk
-func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions) error {
+func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
 	if c.diskUsageHigh() {
-		select {
-		case c.purgeChan <- struct{}{}:
-		default:
-		}
+		c.incGCCounter()
+		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}
+	cachePath := getCacheSHADir(c.dir, bucket, object)
+	cLock := c.NewNSLockFn(ctx, cachePath)
+	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer cLock.Unlock()
+
+	meta, _, numHits, err := c.statCache(ctx, cachePath)
+	// Case where object not yet cached
+	if os.IsNotExist(err) && c.after >= 1 {
+		return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+	}
+	// Case where object already has a cache metadata entry but not yet cached
+	if err == nil && numHits < c.after {
+		cETag := extractETag(meta.Meta)
+		bETag := extractETag(opts.UserDefined)
+		if cETag == bETag {
+			return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+		}
+		incHitsOnly = true
+	}
+
 	if rs != nil {
 		return c.putRange(ctx, bucket, object, data, size, rs, opts)
 	}
 	if !c.diskAvailable(size) {
 		return errDiskFull
 	}
-	cachePath := getCacheSHADir(c.dir, bucket, object)
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return err
 	}
@@ -570,7 +654,6 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	}
 	var reader = data
 	var actualSize = uint64(size)
-	var err error
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
@@ -586,11 +669,12 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 		removeAll(cachePath)
 		return err
 	}
+
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
 		return IncompleteBody{}
 	}
-	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "")
+	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
 }
 
 // Caches the range to disk
@@ -636,7 +720,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 		removeAll(cachePath)
 		return IncompleteBody{}
 	}
-	return c.saveMetadata(ctx, bucket, object, metadata, int64(objSize), rs, cacheFile)
+	return c.saveMetadata(ctx, bucket, object, metadata, int64(objSize), rs, cacheFile, false)
 }
 
 // checks streaming bitrot checksum of cached object before returning data
@@ -736,13 +820,18 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 }
 
 // Get returns ObjectInfo and reader for object from disk cache
-func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	var objInfo ObjectInfo
+func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	var rngInfo RangeInfo
+	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
+		return nil, numHits, err
+	}
 
-	if objInfo, rngInfo, err = c.statRange(ctx, bucket, object, rs); err != nil {
-		return nil, toObjectErr(err, bucket, object)
+	defer cLock.RUnlock()
+	var objInfo ObjectInfo
+	var rngInfo RangeInfo
+	if objInfo, rngInfo, numHits, err = c.statRange(ctx, bucket, object, rs); err != nil {
+		return nil, numHits, toObjectErr(err, bucket, object)
 	}
 	cacheFile := cacheDataFile
 	objSize := objInfo.Size
@@ -758,12 +847,13 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	if HasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
 		// objReader.Close() is called by the caller.
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+		gr, gerr := NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+		return gr, numHits, gerr
 	}
 
 	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
 	if nErr != nil {
-		return nil, nErr
+		return nil, numHits, nErr
 	}
 	filePath := pathJoin(cacheObjPath, cacheFile)
 	pr, pw := io.Pipe()
@@ -780,7 +870,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 
 	gr, gerr := fn(pr, h, opts.CheckCopyPrecondFn, pipeCloser)
 	if gerr != nil {
-		return gr, gerr
+		return gr, numHits, gerr
 	}
 	if globalCacheKMS != nil {
 		// clean up internal SSE cache metadata
@@ -790,15 +880,24 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		// overlay Size with actual object size and not the range size
 		gr.ObjInfo.Size = objSize
 	}
-	return gr, nil
+	return gr, numHits, nil
 
 }
 
 // Deletes the cached object
-func (c *diskCache) Delete(ctx context.Context, bucket, object string) (err error) {
-	cachePath := getCacheSHADir(c.dir, bucket, object)
-	return removeAll(cachePath)
+func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
+	cLock := c.NewNSLockFn(ctx, cacheObjPath)
+	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+		return err
+	}
+	defer cLock.Unlock()
+	return removeAll(cacheObjPath)
+}
 
+// Deletes the cached object
+func (c *diskCache) Delete(ctx context.Context, bucket, object string) (err error) {
+	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
+	return c.delete(ctx, cacheObjPath)
 }
 
 // convenience function to check if object is cached on this diskCache

cmd/disk-cache-utils.go

@@ -17,8 +17,12 @@
 package cmd
 
 import (
+	"container/list"
 	"encoding/hex"
+	"errors"
+	"fmt"
 	"io"
+	"math"
 	"os"
 	"path"
 	"strconv"
@@ -28,6 +32,24 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 )
 
+// CacheStatusType - whether the request was served from cache.
+type CacheStatusType string
+
+const (
+	// CacheHit - whether object was served from cache.
+	CacheHit CacheStatusType = "HIT"
+
+	// CacheMiss - object served from backend.
+	CacheMiss CacheStatusType = "MISS"
+)
+
+func (c CacheStatusType) String() string {
+	if c != "" {
+		return string(c)
+	}
+	return string(CacheMiss)
+}
+
 type cacheControl struct {
 	expiry       time.Time
 	maxAge       int
@@ -39,13 +61,8 @@ type cacheControl struct {
 	noCache      bool
 }
 
-func (c cacheControl) isEmpty() bool {
-	return c == cacheControl{}
-
-}
-
-func (c cacheControl) isStale(modTime time.Time) bool {
-	if c.isEmpty() {
+func (c *cacheControl) isStale(modTime time.Time) bool {
+	if c == nil {
 		return false
 	}
 	// response will never be stale if only-if-cached is set
@@ -82,7 +99,8 @@ func (c cacheControl) isStale(modTime time.Time) bool {
 }
 
 // returns struct with cache-control settings from user metadata.
-func cacheControlOpts(o ObjectInfo) (c cacheControl) {
+func cacheControlOpts(o ObjectInfo) *cacheControl {
+	c := cacheControl{}
 	m := o.UserDefined
 	if o.Expires != timeSentinel {
 		c.expiry = o.Expires
@@ -96,7 +114,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 
 	}
 	if headerVal == "" {
-		return
+		return nil
 	}
 	headerVal = strings.ToLower(headerVal)
 	headerVal = strings.TrimSpace(headerVal)
@@ -128,7 +146,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			p[0] == "max-stale" {
 			i, err := strconv.Atoi(p[1])
 			if err != nil {
-				return cacheControl{}
+				return nil
 			}
 			if p[0] == "max-age" {
 				c.maxAge = i
@@ -144,7 +162,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			}
 		}
 	}
-	return c
+	return &c
 }
 
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
@@ -247,3 +265,179 @@ func decryptCacheObjectETag(info *ObjectInfo) error {
 
 	return nil
 }
+
+func isMetadataSame(m1, m2 map[string]string) bool {
+	if m1 == nil && m2 == nil {
+		return true
+	}
+	if (m1 == nil && m2 != nil) || (m2 == nil && m1 != nil) {
+		return false
+	}
+	if len(m1) != len(m2) {
+		return false
+	}
+	for k1, v1 := range m1 {
+		if v2, ok := m2[k1]; !ok || (v1 != v2) {
+			return false
+		}
+	}
+	return true
+}
+
+type fileScorer struct {
+	saveBytes int64
+	now       int64
+	maxHits   int
+	// 1/size for consistent score.
+	sizeMult float64
+
+	// queue is a linked list of files we want to delete.
+	// The list is kept sorted according to score, highest at top, lowest at bottom.
+	queue       list.List
+	queuedBytes int64
+}
+
+type queuedFile struct {
+	name  string
+	size  int64
+	score float64
+}
+
+// newFileScorer allows to collect files to save a specific number of bytes.
+// Each file is assigned a score based on its age, size and number of hits.
+// A list of files is maintained
+func newFileScorer(saveBytes int64, now int64, maxHits int) (*fileScorer, error) {
+	if saveBytes <= 0 {
+		return nil, errors.New("newFileScorer: saveBytes <= 0")
+	}
+	if now < 0 {
+		return nil, errors.New("newFileScorer: now < 0")
+	}
+	if maxHits <= 0 {
+		return nil, errors.New("newFileScorer: maxHits <= 0")
+	}
+	f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
+	f.queue.Init()
+	return &f, nil
+}
+
+func (f *fileScorer) addFile(name string, lastAccess time.Time, size int64, hits int) {
+	// Calculate how much we want to delete this object.
+	file := queuedFile{
+		name: name,
+		size: size,
+	}
+	score := float64(f.now - lastAccess.Unix())
+	// Size as fraction of how much we want to save, 0->1.
+	szWeight := math.Max(0, (math.Min(1, float64(size)*f.sizeMult)))
+	// 0 at f.maxHits, 1 at 0.
+	hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
+	file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
+	// If we still haven't saved enough, just add the file
+	if f.queuedBytes < f.saveBytes {
+		f.insertFile(file)
+		f.trimQueue()
+		return
+	}
+	// If we score less than the worst, don't insert.
+	worstE := f.queue.Back()
+	if worstE != nil && file.score < worstE.Value.(queuedFile).score {
+		return
+	}
+	f.insertFile(file)
+	f.trimQueue()
+}
+
+// adjustSaveBytes allows to adjust the number of bytes to save.
+// This can be used to adjust the count on the fly.
+// Returns true if there still is a need to delete files (saveBytes >0),
+// false if no more bytes needs to be saved.
+func (f *fileScorer) adjustSaveBytes(n int64) bool {
+	f.saveBytes += n
+	if f.saveBytes <= 0 {
+		f.queue.Init()
+		f.saveBytes = 0
+		return false
+	}
+	if n < 0 {
+		f.trimQueue()
+	}
+	return true
+}
+
+// insertFile will insert a file into the list, sorted by its score.
+func (f *fileScorer) insertFile(file queuedFile) {
+	e := f.queue.Front()
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if v.score < file.score {
+			break
+		}
+		e = e.Next()
+	}
+	f.queuedBytes += file.size
+	// We reached the end.
+	if e == nil {
+		f.queue.PushBack(file)
+		return
+	}
+	f.queue.InsertBefore(file, e)
+}
+
+// trimQueue will trim the back of queue and still keep below wantSave.
+func (f *fileScorer) trimQueue() {
+	for {
+		e := f.queue.Back()
+		if e == nil {
+			return
+		}
+		v := e.Value.(queuedFile)
+		if f.queuedBytes-v.size < f.saveBytes {
+			return
+		}
+		f.queue.Remove(e)
+		f.queuedBytes -= v.size
+	}
+}
+
+// fileNames returns all queued file names.
+func (f *fileScorer) fileNames() []string {
+	res := make([]string, 0, f.queue.Len())
+	e := f.queue.Front()
+	for e != nil {
+		res = append(res, e.Value.(queuedFile).name)
+		e = e.Next()
+	}
+	return res
+}
+
+func (f *fileScorer) reset() {
+	f.queue.Init()
+	f.queuedBytes = 0
+}
+
+func (f *fileScorer) queueString() string {
+	var res strings.Builder
+	e := f.queue.Front()
+	i := 0
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if i > 0 {
+			res.WriteByte('\n')
+		}
+		res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
+		i++
+		e = e.Next()
+	}
+	return res.String()
+}
+
+// bytesToClear() returns the number of bytes to clear to reach low watermark
+// w.r.t quota given disk total and free space, quota in % allocated to cache
+// and low watermark % w.r.t allowed quota.
+func bytesToClear(total, free int64, quotaPct, lowWatermark uint64) uint64 {
+	used := (total - free)
+	quotaAllowed := total * (int64)(quotaPct) / 100
+	lowWMUsage := (total * (int64)(lowWatermark*quotaPct) / (100 * 100))
+	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
+}

cmd/disk-cache-utils_test.go

@@ -29,16 +29,16 @@ func TestGetCacheControlOpts(t *testing.T) {
 	testCases := []struct {
 		cacheControlHeaderVal string
 		expiryHeaderVal       time.Time
-		expectedCacheControl  cacheControl
+		expectedCacheControl  *cacheControl
 		expectedErr           bool
 	}{
-		{"", timeSentinel, cacheControl{}, false},
-		{"max-age=2592000, public", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"max-age=2592000, no-store", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
-		{"must-revalidate, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", expiry, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
-		{"s-maxAge=2500, max-age=600s", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
+		{"", timeSentinel, nil, false},
+		{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
+		{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
+		{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
 	}
 
 	for _, testCase := range testCases {
@@ -49,7 +49,7 @@ func TestGetCacheControlOpts(t *testing.T) {
 				m["expires"] = testCase.expiryHeaderVal.String()
 			}
 			c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
-			if testCase.expectedErr && (c != cacheControl{}) {
+			if testCase.expectedErr && (c != nil) {
 				t.Errorf("expected err, got <nil>")
 			}
 			if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
@@ -58,3 +58,115 @@ func TestGetCacheControlOpts(t *testing.T) {
 		})
 	}
 }
+
+func TestIsMetadataSame(t *testing.T) {
+
+	testCases := []struct {
+		m1       map[string]string
+		m2       map[string]string
+		expected bool
+	}{
+		{nil, nil, true},
+		{nil, map[string]string{}, false},
+		{map[string]string{"k": "v"}, map[string]string{"k": "v"}, true},
+		{map[string]string{"k": "v"}, map[string]string{"a": "b"}, false},
+		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v1"}, false},
+		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, true},
+		{map[string]string{"K1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
+		{map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
+	}
+
+	for i, testCase := range testCases {
+		actual := isMetadataSame(testCase.m1, testCase.m2)
+		if testCase.expected != actual {
+			t.Errorf("test %d expected %v, got %v", i, testCase.expected, actual)
+		}
+	}
+}
+
+func TestNewFileScorer(t *testing.T) {
+	fs, err := newFileScorer(1000, time.Now().Unix(), 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fs.fileNames()) != 0 {
+		t.Fatal("non zero files??")
+	}
+	now := time.Now()
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	fs.addFile("older", now.Add(-time.Hour), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("less", now.Add(-time.Minute), 1000, 5)
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("small", now.Add(-time.Minute), 200, 10)
+	fs.addFile("medium", now.Add(-time.Minute), 300, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.addFile("large", now.Add(-time.Minute), 700, 10)
+	fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+
+	fs.reset()
+	fs.addFile("less", now.Add(-time.Minute), 500, 5)
+	fs.addFile("recent", now.Add(-time.Minute), 500, 10)
+	if !fs.adjustSaveBytes(-500) {
+		t.Fatal("we should still need more bytes, got false")
+	}
+	// We should only need 500 bytes now.
+	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	if fs.adjustSaveBytes(-500) {
+		t.Fatal("we shouldn't need any more bytes, got true")
+	}
+	fs, err = newFileScorer(1000, time.Now().Unix(), 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
+	// sorting should be consistent after adjusting savebytes.
+	fs.adjustSaveBytes(-800)
+	fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+}
+func TestBytesToClear(t *testing.T) {
+	testCases := []struct {
+		total        int64
+		free         int64
+		quotaPct     uint64
+		watermarkLow uint64
+		expected     uint64
+	}{
+		{1000, 800, 40, 90, 0},
+		{1000, 200, 40, 90, 400},
+		{1000, 400, 40, 90, 240},
+		{1000, 600, 40, 90, 40},
+		{1000, 600, 40, 70, 120},
+		{1000, 1000, 90, 70, 0},
+		{1000, 0, 90, 70, 370},
+	}
+	for i, tc := range testCases {
+		toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow)
+		if tc.expected != toClear {
+			t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
+		}
+	}
+}

cmd/disk-cache.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019,2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -31,13 +31,15 @@ import (
 	"github.com/djherbis/atime"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/logger"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
-	cacheBlkSize = int64(1 * 1024 * 1024)
+	cacheBlkSize    = int64(1 * 1024 * 1024)
+	cacheGCInterval = time.Minute * 30
 )
 
 // CacheStorageInfo - represents total, free capacity of
@@ -55,6 +57,7 @@ type CacheObjectLayer interface {
 	DeleteObject(ctx context.Context, bucket, object string) error
 	DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	// Storage operations.
 	StorageInfo(ctx context.Context) CacheStorageInfo
 	CacheStats() *CacheStats
@@ -66,74 +69,57 @@ type cacheObjects struct {
 	cache []*diskCache
 	// file path patterns to exclude from cache
 	exclude []string
-
+	// number of accesses after which to cache an object
+	after int
 	// if true migration is in progress from v1 to v2
 	migrating bool
 	// mutex to protect migration bool
 	migMutex sync.Mutex
 
-	// nsMutex namespace lock
-	nsMutex *nsLockMap
-
 	// Cache stats
 	cacheStats *CacheStats
 
-	// Object functions pointing to the corresponding functions of backend implementation.
-	NewNSLockFn      func(ctx context.Context, bucket, object string) RWLocker
 	GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn   func(ctx context.Context, bucket, object string) error
 	DeleteObjectsFn  func(ctx context.Context, bucket string, objects []string) ([]error, error)
 	PutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	CopyObjectFn     func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 }
 
-func (c *cacheObjects) delete(ctx context.Context, dcache *diskCache, bucket, object string) (err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
-		return err
-	}
-	defer cLock.Unlock()
-	return dcache.Delete(ctx, bucket, object)
+func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string) error {
+	metadata := make(map[string]string)
+	metadata["etag"] = eTag
+	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, nil, "", true)
 }
 
-func (c *cacheObjects) put(ctx context.Context, dcache *diskCache, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions) error {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
-		return err
-	}
-	defer cLock.Unlock()
-	return dcache.Put(ctx, bucket, object, data, size, rs, opts)
-}
+// Backend metadata could have changed through server side copy - reset cache metadata if that is the case
+func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *diskCache, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
 
-func (c *cacheObjects) get(ctx context.Context, dcache *diskCache, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
-		return nil, err
+	bkMeta := make(map[string]string)
+	cacheMeta := make(map[string]string)
+	for k, v := range bkObjectInfo.UserDefined {
+		if HasPrefix(k, ReservedMetadataPrefix) {
+			// Do not need to send any internal metadata
+			continue
+		}
+		bkMeta[http.CanonicalHeaderKey(k)] = v
+	}
+	for k, v := range cacheObjInfo.UserDefined {
+		if HasPrefix(k, ReservedMetadataPrefix) {
+			// Do not need to send any internal metadata
+			continue
+		}
+		cacheMeta[http.CanonicalHeaderKey(k)] = v
 	}
 
-	defer cLock.RUnlock()
-	return dcache.Get(ctx, bucket, object, rs, h, opts)
-}
-
-func (c *cacheObjects) stat(ctx context.Context, dcache *diskCache, bucket, object string) (oi ObjectInfo, err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
-		return oi, err
+	if !isMetadataSame(bkMeta, cacheMeta) ||
+		bkObjectInfo.ETag != cacheObjInfo.ETag ||
+		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
+		!bkObjectInfo.Expires.Equal(cacheObjInfo.Expires) {
+		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false)
 	}
-
-	defer cLock.RUnlock()
-	return dcache.Stat(ctx, bucket, object)
-}
-
-func (c *cacheObjects) statRange(ctx context.Context, dcache *diskCache, bucket, object string, rs *HTTPRangeSpec) (oi ObjectInfo, err error) {
-	cLock := c.NewNSLockFn(ctx, bucket, object)
-	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
-		return oi, err
-	}
-
-	defer cLock.RUnlock()
-	oi, _, err = dcache.statRange(ctx, bucket, object, rs)
-	return oi, err
+	return c.incHitsToMeta(ctx, dcache, bucket, object, cacheObjInfo.Size, cacheObjInfo.ETag)
 }
 
 // DeleteObject clears cache entry if backend delete operation succeeds
@@ -149,9 +135,7 @@ func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string)
 	if cerr != nil {
 		return
 	}
-	if dcache.Exists(ctx, bucket, object) {
-		c.delete(ctx, dcache, bucket, object)
-	}
+	dcache.Delete(ctx, bucket, object)
 	return
 }
 
@@ -191,7 +175,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
-	var cc cacheControl
+	var cc *cacheControl
 	var cacheObjSize int64
 	// fetch diskCache if object is currently cached or nearest available cache drive
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
@@ -199,7 +183,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 
-	cacheReader, cacheErr := c.get(ctx, dcache, bucket, object, rs, h, opts)
+	cacheReader, numCacheHits, cacheErr := dcache.Get(ctx, bucket, object, rs, h, opts)
 	if cacheErr == nil {
 		cacheObjSize = cacheReader.ObjInfo.Size
 		if rs != nil {
@@ -208,8 +192,8 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			}
 		}
 		cc = cacheControlOpts(cacheReader.ObjInfo)
-		if (!cc.isEmpty() && !cc.isStale(cacheReader.ObjInfo.ModTime)) ||
-			cc.onlyIfCached {
+		if cc != nil && (!cc.isStale(cacheReader.ObjInfo.ModTime) ||
+			cc.onlyIfCached) {
 			// This is a cache hit, mark it so
 			bytesServed := cacheReader.ObjInfo.Size
 			if rs != nil {
@@ -219,11 +203,15 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			}
 			c.cacheStats.incHit()
 			c.cacheStats.incBytesServed(bytesServed)
+			c.incHitsToMeta(ctx, dcache, bucket, object, cacheReader.ObjInfo.Size, cacheReader.ObjInfo.ETag)
 			return cacheReader, nil
 		}
-		if cc.noStore {
+		if cc != nil && cc.noStore {
 			c.cacheStats.incMiss()
-			return c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+			bReader, err := c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+			bReader.ObjInfo.CacheLookupStatus = CacheHit
+			bReader.ObjInfo.CacheStatus = CacheMiss
+			return bReader, err
 		}
 	}
 
@@ -248,32 +236,52 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		c.cacheStats.incMiss()
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
-
+	// skip cache for objects with locks
+	objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
+	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
+		c.cacheStats.incMiss()
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	}
 	if cacheErr == nil {
 		// if ETag matches for stale cache entry, serve from cache
 		if cacheReader.ObjInfo.ETag == objInfo.ETag {
 			// Update metadata in case server-side copy might have changed object metadata
-			dcache.updateMetadataIfChanged(ctx, bucket, object, objInfo, cacheReader.ObjInfo)
+			c.updateMetadataIfChanged(ctx, dcache, bucket, object, objInfo, cacheReader.ObjInfo)
 			c.incCacheStats(cacheObjSize)
 			return cacheReader, nil
 		}
 		cacheReader.Close()
 		// Object is stale, so delete from cache
-		c.delete(ctx, dcache, bucket, object)
+		dcache.Delete(ctx, bucket, object)
 	}
 
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
 	// Since we got here, we are serving the request from backend,
 	// and also adding the object to the cache.
-	if !dcache.diskUsageLow() {
-		select {
-		case dcache.purgeChan <- struct{}{}:
-		default:
-		}
+	if dcache.diskUsageHigh() {
+		dcache.incGCCounter()
+	}
+
+	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+
+	if bkErr != nil {
+		return bkReader, bkErr
+	}
+	// If object has less hits than configured cache after, just increment the hit counter
+	// but do not cache it.
+	if numCacheHits < c.after {
+		c.incHitsToMeta(ctx, dcache, bucket, object, objInfo.Size, objInfo.ETag)
+		return bkReader, bkErr
+	}
+
+	// Record if cache has a hit that was invalidated by ETag verification
+	if cacheErr == nil {
+		bkReader.ObjInfo.CacheLookupStatus = CacheHit
 	}
 	if !dcache.diskAvailable(objInfo.Size) {
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return bkReader, bkErr
 	}
 
 	if rs != nil {
@@ -284,23 +292,20 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 				return
 			}
 			defer bReader.Close()
-			oi, err := c.statRange(ctx, dcache, bucket, object, rs)
+			oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
+				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
 			}
 		}()
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
-	}
-	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
-	if bkErr != nil {
-		return nil, bkErr
+		return bkReader, bkErr
 	}
+
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	go func() {
-		putErr := c.put(ctx, dcache, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)})
+		putErr := dcache.Put(ctx, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)}, false)
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
@@ -323,12 +328,12 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	if err != nil {
 		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
-	var cc cacheControl
+	var cc *cacheControl
 	// if cache control setting is valid, avoid HEAD operation to backend
-	cachedObjInfo, cerr := c.stat(ctx, dcache, bucket, object)
+	cachedObjInfo, _, cerr := dcache.Stat(ctx, bucket, object)
 	if cerr == nil {
 		cc = cacheControlOpts(cachedObjInfo)
-		if !cc.isStale(cachedObjInfo.ModTime) {
+		if cc == nil || (cc != nil && !cc.isStale(cachedObjInfo.ModTime)) {
 			// This is a cache hit, mark it so
 			c.cacheStats.incHit()
 			return cachedObjInfo, nil
@@ -339,7 +344,7 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
-			c.delete(ctx, dcache, bucket, object)
+			dcache.Delete(ctx, bucket, object)
 			c.cacheStats.incMiss()
 			return ObjectInfo{}, err
 		}
@@ -363,11 +368,35 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	}
 	if cachedObjInfo.ETag != objInfo.ETag {
 		// Delete the cached entry if the backend object was replaced.
-		c.delete(ctx, dcache, bucket, object)
+		dcache.Delete(ctx, bucket, object)
 	}
 	return objInfo, nil
 }
 
+// CopyObject reverts to backend after evicting any stale cache entries
+func (c *cacheObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+	copyObjectFn := c.CopyObjectFn
+	if c.isCacheExclude(srcBucket, srcObject) || c.skipCache() {
+		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+	}
+	if srcBucket != dstBucket || srcObject != dstObject {
+		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+	}
+	// fetch diskCache if object is currently cached or nearest available cache drive
+	dcache, err := c.getCacheToLoc(ctx, srcBucket, srcObject)
+	if err != nil {
+		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+	}
+	// if currently cached, evict old entry and revert to backend.
+	if cachedObjInfo, _, cerr := dcache.Stat(ctx, srcBucket, srcObject); cerr == nil {
+		cc := cacheControlOpts(cachedObjInfo)
+		if cc == nil || !cc.isStale(cachedObjInfo.ModTime) {
+			dcache.Delete(ctx, srcBucket, srcObject)
+		}
+	}
+	return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
+}
+
 // StorageInfo - returns underlying storage statistics.
 func (c *cacheObjects) StorageInfo(ctx context.Context) (cInfo CacheStorageInfo) {
 	var total, free uint64
@@ -491,16 +520,10 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		if quota == 0 {
 			quota = config.Quota
 		}
-
-		cache, err := newDiskCache(dir, config.Expiry, quota)
+		cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
 		if err != nil {
 			return nil, false, err
 		}
-		// Start the purging go-routine for entries that have expired if no migration in progress
-		if !migrating {
-			go cache.purge()
-		}
-
 		caches = append(caches, cache)
 	}
 	return caches, migrating, nil
@@ -547,13 +570,12 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	}
 
 	errCnt := 0
-	for index, err := range g.Wait() {
+	for _, err := range g.Wait() {
 		if err != nil {
 			errCnt++
 			logger.LogIf(ctx, err)
 			continue
 		}
-		go c.cache[index].purge()
 	}
 
 	if errCnt > 0 {
@@ -590,8 +612,9 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	}
 
 	// skip cache for objects with locks
-	objRetention := getObjectRetentionMeta(opts.UserDefined)
-	if objRetention.Mode == Governance || objRetention.Mode == Compliance {
+	objRetention := objectlock.GetObjectRetentionMeta(opts.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(opts.UserDefined)
+	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
@@ -613,10 +636,10 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 				return
 			}
 			defer bReader.Close()
-			oi, err := c.stat(ctx, dcache, bucket, object)
+			oi, _, err := dcache.Stat(ctx, bucket, object)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
+				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
 			}
 		}()
 	}
@@ -632,13 +655,12 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 	if err != nil {
 		return nil, err
 	}
-
 	c := &cacheObjects{
 		cache:      cache,
 		exclude:    config.Exclude,
+		after:      config.After,
 		migrating:  migrateSw,
 		migMutex:   sync.Mutex{},
-		nsMutex:    newNSLock(false),
 		cacheStats: newCacheStats(),
 		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
@@ -659,13 +681,43 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
 		},
-	}
-	c.NewNSLockFn = func(ctx context.Context, bucket, object string) RWLocker {
-		return c.nsMutex.NewNSLock(ctx, nil, bucket, object)
+		CopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
+		},
 	}
 
 	if migrateSw {
 		go c.migrateCacheFromV1toV2(ctx)
 	}
+	go c.gc(ctx, GlobalServiceDoneCh)
 	return c, nil
 }
+
+func (c *cacheObjects) gc(ctx context.Context, doneCh chan struct{}) {
+	ticker := time.NewTicker(cacheGCInterval)
+
+	defer ticker.Stop()
+	for {
+		select {
+		case <-doneCh:
+			return
+		case <-ticker.C:
+			if c.migrating {
+				continue
+			}
+			var wg sync.WaitGroup
+			for _, dcache := range c.cache {
+				if dcache.gcCount() == 0 {
+					continue
+				}
+				wg.Add(1)
+				go func(d *diskCache) {
+					defer wg.Done()
+					d.resetGCCounter()
+					d.purge(ctx, doneCh)
+				}(dcache)
+			}
+			wg.Wait()
+		}
+	}
+}

cmd/disk-cache_test.go

@@ -27,15 +27,15 @@ import (
 )
 
 // Initialize cache objects.
-func initCacheObjects(disk string, cacheMaxUse int) (*diskCache, error) {
-	return newDiskCache(disk, 80, cacheMaxUse)
+func initCacheObjects(disk string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int) (*diskCache, error) {
+	return newDiskCache(disk, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) ([]*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int, t *testing.T) ([]*diskCache, error) {
 	var cb []*diskCache
 	for _, d := range drives {
-		obj, err := initCacheObjects(d, cacheMaxUse)
+		obj, err := initCacheObjects(d, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
 		if err != nil {
 			return nil, err
 		}
@@ -70,7 +70,7 @@ func TestGetCachedLoc(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, 100, t)
+		d, err := initDiskCaches(fsDirs, 100, 1, 80, 90, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -109,7 +109,7 @@ func TestGetCacheMaxUse(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, 80, t)
+		d, err := initDiskCaches(fsDirs, 80, 1, 80, 90, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -180,7 +180,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, 100, t)
+	d, err := initDiskCaches(fsDirs, 100, 0, 80, 90, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -212,11 +212,11 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta})
+	err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta}, false)
 	if err != nil {
 		t.Fatal(err)
 	}
-	cReader, err := cache.Get(ctx, bucketName, objectName, nil, http.Header{
+	cReader, _, err := cache.Get(ctx, bucketName, objectName, nil, http.Header{
 		"Content-Type": []string{"application/json"},
 	}, opts)
 	if err != nil {
@@ -258,7 +258,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, 80, t)
+	d, err := initDiskCaches(fsDirs, 90, 0, 90, 99, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -290,16 +290,16 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta}, false)
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), nil, ObjectOptions{UserDefined: httpMeta}, false)
 		if err != nil {
 			t.Fatal(err)
 		}
-		cReader, err := cache.Get(ctx, bucketName, objectName, nil, nil, opts)
+		cReader, _, err := cache.Get(ctx, bucketName, objectName, nil, nil, opts)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/dummy-handlers.go

@@ -21,27 +21,14 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
 )
 
 // Data types used for returning dummy tagging XML.
 // These variables shouldn't be used elsewhere.
 // They are only defined to be used in this file alone.
 
-type tagging struct {
-	XMLName xml.Name `xml:"Tagging"`
-	TagSet  tagSet   `xml:"TagSet"`
-}
-
-type tagSet struct {
-	Tag []tagElem `xml:"Tag"`
-}
-
-type tagElem struct {
-	Key   string `xml:"Key"`
-	Value string `xml:"Value"`
-}
-
 // GetBucketWebsite  - GET bucket website, a dummy api
 func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseHeadersOnly(w)
@@ -171,47 +158,8 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	tags := &tagging{}
-	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
-
-	if err := xml.NewEncoder(w).Encode(tags); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	w.(http.Flusher).Flush()
-}
-
-// GetObjectTaggingHandler - GET object tagging, a dummy api
-func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetObjectTagging")
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	object := vars["object"]
-
-	objAPI := api.ObjectAPI()
-	if objAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Allow getObjectTagging if policy action is set, since this is a dummy call
-	// we are simply re-purposing the bucketPolicyAction.
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Validate if object exists, before proceeding further...
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	tags := &tagging{}
-	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
+	tags := &tagging.Tagging{}
+	tags.TagSet.Tags = append(tags.TagSet.Tags, tagging.Tag{})
 
 	if err := xml.NewEncoder(w).Encode(tags); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/encryption-v1_test.go

@@ -280,15 +280,11 @@ func TestGetDecryptedRange_Issue50(t *testing.T) {
 		Parts: []ObjectPartInfo{
 			{
 				Number:     1,
-				Name:       "part.1",
-				ETag:       "etag1",
 				Size:       297580380,
 				ActualSize: 297435132,
 			},
 			{
 				Number:     2,
-				Name:       "part.2",
-				ETag:       "etag2",
 				Size:       297580380,
 				ActualSize: 297435132,
 			},

cmd/endpoint-ellipses.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -76,7 +76,8 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 	for _, totalSize := range totalSizes {
 		// Check if totalSize has minimum range upto setSize
 		if totalSize < setSizes[0] || totalSize < customSetDriveCount {
-			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
+			msg := fmt.Sprintf("Incorrect number of endpoints provided %s", args)
+			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 		}
 	}
 
@@ -107,12 +108,24 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		return ss
 	}
 
+	setCounts := possibleSetCounts(commonSize)
+	if len(setCounts) == 0 {
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
+	}
+
 	if customSetDriveCount > 0 {
-		msg := fmt.Sprintf("Invalid set drive count, leads to non-uniform distribution for the given number of disks. Possible values for custom set count are %d", possibleSetCounts(setSize))
+		msg := fmt.Sprintf("Invalid set drive count. Acceptable values for %d number drives are %d", commonSize, setCounts)
 		if customSetDriveCount > setSize {
 			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
-		if setSize%customSetDriveCount != 0 {
+		var found bool
+		for _, ss := range setCounts {
+			if ss == customSetDriveCount {
+				found = true
+			}
+		}
+		if !found {
 			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
 		setSize = customSetDriveCount
@@ -120,7 +133,8 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 
 	for i := range totalSizes {
@@ -203,18 +217,7 @@ func parseEndpointSet(customSetDriveCount uint64, args ...string) (ep endpointSe
 // specific set size.
 // For example: {1...64} is divided into 4 sets each of size 16.
 // This applies to even distributed setup syntax as well.
-func GetAllSets(args ...string) ([][]string, error) {
-	var customSetDriveCount uint64
-	if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
-		customSetDriveCount, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return nil, config.ErrInvalidErasureSetSize(err)
-		}
-		if !isValidSetSize(customSetDriveCount) {
-			return nil, config.ErrInvalidErasureSetSize(nil)
-		}
-	}
-
+func GetAllSets(customSetDriveCount uint64, args ...string) ([][]string, error) {
 	var setArgs [][]string
 	if !ellipses.HasEllipses(args...) {
 		var setIndexes [][]uint64
@@ -257,16 +260,23 @@ func GetAllSets(args ...string) ([][]string, error) {
 
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
-func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, int, SetupType, error) {
+func createServerEndpoints(serverAddr string, args ...string) (
+	endpointZones EndpointZones, setDriveCount int,
+	setupType SetupType, err error) {
+
 	if len(args) == 0 {
 		return nil, -1, -1, errInvalidArgument
 	}
 
-	var endpointZones EndpointZones
-	var setupType SetupType
-	var drivesPerSet int
+	if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
+		setDriveCount, err = strconv.Atoi(v)
+		if err != nil {
+			return nil, -1, -1, config.ErrInvalidErasureSetSize(err)
+		}
+	}
+
 	if !ellipses.HasEllipses(args...) {
-		setArgs, err := GetAllSets(args...)
+		setArgs, err := GetAllSets(uint64(setDriveCount), args...)
 		if err != nil {
 			return nil, -1, -1, err
 		}
@@ -283,43 +293,37 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in
 		return endpointZones, len(setArgs[0]), setupType, nil
 	}
 
+	var prevSetupType SetupType
 	var foundPrevLocal bool
-
-	// Verify the args setup-type appropriately.
-	{
-		setArgs, err := GetAllSets(args...)
-		if err != nil {
-			return nil, -1, -1, err
-		}
-
-		var endpoints Endpoints
-		endpoints, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
-		if err != nil {
-			return nil, -1, -1, err
-		}
-
-		foundPrevLocal = endpoints.atleastOneEndpointLocal()
-	}
-
 	for _, arg := range args {
-		setArgs, err := GetAllSets(arg)
+		setArgs, err := GetAllSets(uint64(setDriveCount), arg)
 		if err != nil {
 			return nil, -1, -1, err
 		}
-		endpointList, _, err := CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
+		var endpointList Endpoints
+		endpointList, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
 		if err != nil {
 			return nil, -1, -1, err
 		}
-		if drivesPerSet != 0 && drivesPerSet != len(setArgs[0]) {
-			return nil, -1, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", drivesPerSet, len(setArgs[0]))
+		if setDriveCount != 0 && setDriveCount != len(setArgs[0]) {
+			return nil, -1, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
 		}
-		endpointZones = append(endpointZones, ZoneEndpoints{
+		if prevSetupType != UnknownSetupType && prevSetupType != setupType {
+			return nil, -1, -1, fmt.Errorf("All zones should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
+		}
+		if err = endpointZones.Add(ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
-		})
-		drivesPerSet = len(setArgs[0])
+		}); err != nil {
+			return nil, -1, -1, err
+		}
+		foundPrevLocal = endpointList.atleastOneEndpointLocal()
+		if setDriveCount == 0 {
+			setDriveCount = len(setArgs[0])
+		}
+		prevSetupType = setupType
 	}
 
-	return endpointZones, drivesPerSet, setupType, nil
+	return endpointZones, setDriveCount, setupType, nil
 }

cmd/endpoint-ellipses_test.go

@@ -104,6 +104,36 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
 			8,
 			true,
 		},
+		{
+			[]string{"http://host{1...12}/data{1...12}"},
+			[]uint64{144},
+			[][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16}},
+			16,
+			true,
+		},
+		{
+			[]string{"http://host{0...5}/data{1...28}"},
+			[]uint64{168},
+			[][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}},
+			12,
+			true,
+		},
+		// Incorrect custom set drive count.
+		{
+			[]string{"http://host{0...5}/data{1...28}"},
+			[]uint64{168},
+			nil,
+			10,
+			false,
+		},
+		// Failure not divisible number of disks.
+		{
+			[]string{"http://host{1...11}/data{1...11}"},
+			[]uint64{121},
+			nil,
+			11,
+			false,
+		},
 		{
 			[]string{"data{1...60}"},
 			nil,

cmd/endpoint.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
@@ -53,8 +52,7 @@ const (
 // Endpoint - any type of endpoint.
 type Endpoint struct {
 	*url.URL
-	IsLocal  bool
-	SetIndex int
+	IsLocal bool
 }
 
 func (endpoint Endpoint) String() string {
@@ -194,6 +192,24 @@ type ZoneEndpoints struct {
 // EndpointZones - list of list of endpoints
 type EndpointZones []ZoneEndpoints
 
+// Add add zone endpoints
+func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
+	existSet := set.NewStringSet()
+	for _, zep := range *l {
+		for _, ep := range zep.Endpoints {
+			existSet.Add(ep.String())
+		}
+	}
+	// Validate if there are duplicate endpoints across zones
+	for _, ep := range zeps.Endpoints {
+		if existSet.Contains(ep.String()) {
+			return fmt.Errorf("duplicate endpoints found")
+		}
+	}
+	*l = append(*l, zeps)
+	return nil
+}
+
 // FirstLocal returns true if the first endpoint is local.
 func (l EndpointZones) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
@@ -442,28 +458,6 @@ func NewEndpoints(args ...string) (endpoints Endpoints, err error) {
 	return endpoints, nil
 }
 
-func checkEndpointsSubOptimal(ctx *cli.Context, setupType SetupType, endpointZones EndpointZones) (err error) {
-	// Validate sub optimal ordering only for distributed setup.
-	if setupType != DistXLSetupType {
-		return nil
-	}
-	var endpointOrder int
-	err = fmt.Errorf("Too many disk args are local, input is in sub-optimal order. Please review input args: %s", ctx.Args())
-	for _, endpoints := range endpointZones {
-		for _, endpoint := range endpoints.Endpoints {
-			if endpoint.IsLocal {
-				endpointOrder++
-			} else {
-				endpointOrder--
-			}
-			if endpointOrder >= 2 {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
 // Checks if there are any cross device mounts.
 func checkCrossDeviceMounts(endpoints Endpoints) (err error) {
 	var absPaths []string
@@ -517,9 +511,8 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		return endpoints, setupType, nil
 	}
 
-	for i, iargs := range args {
+	for _, iargs := range args {
 		// Convert args to endpoints
-		var newEndpoints Endpoints
 		eps, err := NewEndpoints(iargs...)
 		if err != nil {
 			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
@@ -530,11 +523,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
-		for _, ep := range eps {
-			ep.SetIndex = i
-			newEndpoints = append(newEndpoints, ep)
-		}
-		endpoints = append(endpoints, newEndpoints...)
+		endpoints = append(endpoints, eps...)
 	}
 
 	if len(endpoints) == 0 {
@@ -608,13 +597,14 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 
 	// All endpoints are pointing to local host
 	if len(endpoints) == localEndpointCount {
-		// If all endpoints have same port number, then this is XL setup using URL style endpoints.
+		// If all endpoints have same port number, Just treat it as distXL setup
+		// using URL style endpoints.
 		if len(localPortSet) == 1 {
 			if len(localServerHostSet) > 1 {
 				return endpoints, setupType,
 					config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
 			}
-			return endpoints, XLSetupType, nil
+			return endpoints, DistXLSetupType, nil
 		}
 
 		// Even though all endpoints are local, but those endpoints use different ports.

cmd/endpoint_test.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"flag"
 	"fmt"
 	"net"
 	"net/url"
@@ -25,47 +24,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
 )
 
-func TestSubOptimalEndpointInput(t *testing.T) {
-	args1 := []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}
-	args2 := []string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}
-
-	tests := []struct {
-		setupType SetupType
-		ctx       *cli.Context
-		endpoints EndpointZones
-		isErr     bool
-	}{
-		{
-			setupType: DistXLSetupType,
-			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
-			endpoints: mustGetZoneEndpoints(args1...),
-			isErr:     false,
-		},
-		{
-			setupType: DistXLSetupType,
-			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
-			endpoints: mustGetZoneEndpoints(args2...),
-			isErr:     false,
-		},
-	}
-	for i, test := range tests {
-		test := test
-		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
-			err := checkEndpointsSubOptimal(test.ctx, test.setupType, test.endpoints)
-			if test.isErr && err == nil {
-				t.Error("expected err but found nil")
-			}
-			if !test.isErr && err != nil {
-				t.Errorf("expected err nil but found an err %s", err)
-			}
-		})
-	}
-}
-
 func TestNewEndpoint(t *testing.T) {
 	u2, _ := url.Parse("https://example.org/path")
 	u4, _ := url.Parse("http://192.168.253.200/path")
@@ -285,7 +246,7 @@ func TestCreateEndpoints(t *testing.T) {
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d2"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d3"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d4"}, IsLocal: true},
-		}, XLSetupType, nil},
+		}, DistXLSetupType, nil},
 		// DistXL Setup with URLEndpointType having mixed naming to local host.
 		{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
 

cmd/erasure-encode_test.go

@@ -44,6 +44,10 @@ func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) e
 	return errFaultyDisk
 }
 
+func (badDisk) Hostname() string {
+	return ""
+}
+
 const oneMiByte = 1 * humanize.MiByte
 
 var erasureEncodeTests = []struct {

cmd/fastwalk.go

@@ -12,19 +12,11 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"sync"
 )
 
-// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the
-// symlink named in the call may be traversed.
-var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory")
-
-// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the
-// callback should not be called for any other files in the current directory.
-// Child directories will still be traversed.
-var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
+var errSkipFile = errors.New("fastwalk: skip this file")
 
 // Walk is a faster implementation of filepath.Walk.
 //
@@ -51,16 +43,7 @@ var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
 //   * fastWalk can follow symlinks if walkFn returns the TraverseLink
 //     sentinel error. It is the walkFn's responsibility to prevent
 //     fastWalk from going into symlink cycles.
-func fastWalk(root string, walkFn func(path string, typ os.FileMode) error) error {
-	// TODO(bradfitz): make numWorkers configurable? We used a
-	// minimum of 4 to give the kernel more info about multiple
-	// things we want, in hopes its I/O scheduling can take
-	// advantage of that. Hopefully most are in cache. Maybe 4 is
-	// even too low of a minimum. Profile more.
-	numWorkers := 4
-	if n := runtime.NumCPU(); n > numWorkers {
-		numWorkers = n
-	}
+func fastWalk(root string, nworkers int, doneCh <-chan struct{}, walkFn func(path string, typ os.FileMode) error) error {
 
 	// Make sure to wait for all workers to finish, otherwise
 	// walkFn could still be called after returning. This Wait call
@@ -70,19 +53,20 @@ func fastWalk(root string, walkFn func(path string, typ os.FileMode) error) erro
 
 	w := &walker{
 		fn:       walkFn,
-		enqueuec: make(chan walkItem, numWorkers), // buffered for performance
-		workc:    make(chan walkItem, numWorkers), // buffered for performance
+		enqueuec: make(chan walkItem, nworkers), // buffered for performance
+		workc:    make(chan walkItem, nworkers), // buffered for performance
 		donec:    make(chan struct{}),
 
 		// buffered for correctness & not leaking goroutines:
-		resc: make(chan error, numWorkers),
+		resc: make(chan error, nworkers),
 	}
 	defer close(w.donec)
 
-	for i := 0; i < numWorkers; i++ {
+	for i := 0; i < nworkers; i++ {
 		wg.Add(1)
 		go w.doWork(&wg)
 	}
+
 	todo := []walkItem{{dir: root}}
 	out := 0
 	for {
@@ -94,6 +78,8 @@ func fastWalk(root string, walkFn func(path string, typ os.FileMode) error) erro
 			workItem = todo[len(todo)-1]
 		}
 		select {
+		case <-doneCh:
+			return nil
 		case workc <- workItem:
 			todo = todo[:len(todo)-1]
 			out++
@@ -161,25 +147,32 @@ func (w *walker) enqueue(it walkItem) {
 	}
 }
 
+var stringsBuilderPool = sync.Pool{
+	New: func() interface{} {
+		return &strings.Builder{}
+	},
+}
+
 func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
-	joined := dirName + string(os.PathSeparator) + baseName
+	builder := stringsBuilderPool.Get().(*strings.Builder)
+	defer func() {
+		builder.Reset()
+		stringsBuilderPool.Put(builder)
+	}()
+
+	builder.WriteString(dirName)
+	if !strings.HasSuffix(dirName, SlashSeparator) {
+		builder.WriteString(SlashSeparator)
+	}
+	builder.WriteString(baseName)
 	if typ == os.ModeDir {
-		w.enqueue(walkItem{dir: joined})
+		w.enqueue(walkItem{dir: builder.String()})
 		return nil
 	}
 
-	err := w.fn(joined, typ)
-	if typ == os.ModeSymlink {
-		if err == ErrTraverseLink {
-			// Set callbackDone so we don't call it twice for both the
-			// symlink-as-symlink and the symlink-as-directory later:
-			w.enqueue(walkItem{dir: joined, callbackDone: true})
-			return nil
-		}
-		if err == filepath.SkipDir {
-			// Permit SkipDir on symlinks too.
-			return nil
-		}
+	err := w.fn(builder.String(), typ)
+	if err == filepath.SkipDir || err == errSkipFile {
+		return nil
 	}
 	return err
 }
@@ -189,22 +182,13 @@ func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode)
 	if err != nil {
 		return err
 	}
-	skipFiles := false
 	for _, fi := range fis {
 		var mode os.FileMode
 		if strings.HasSuffix(fi, SlashSeparator) {
 			mode |= os.ModeDir
 		}
 
-		if mode == 0 && skipFiles {
-			continue
-		}
-
-		if err := fn(dirName, fi, mode); err != nil {
-			if err == ErrSkipFiles {
-				skipFiles = true
-				continue
-			}
+		if err = fn(dirName, fi, mode); err != nil {
 			return err
 		}
 	}
@@ -214,7 +198,7 @@ func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode)
 func (w *walker) walk(root string, runUserCallback bool) error {
 	if runUserCallback {
 		err := w.fn(root, os.ModeDir)
-		if err == filepath.SkipDir {
+		if err == filepath.SkipDir || err == errSkipFile {
 			return nil
 		}
 		if err != nil {

cmd/format-disk-cache.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -29,6 +28,7 @@ import (
 	"reflect"
 	"strings"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/sio"
 )
@@ -444,6 +444,7 @@ func migrateOldCache(ctx context.Context, c *diskCache) error {
 			}
 			// marshal cache metadata after adding version and stat info
 			meta := &cacheMeta{}
+			var json = jsoniter.ConfigCompatibleWithStandardLibrary
 			if err = json.Unmarshal(metaBytes, &meta); err != nil {
 				return err
 			}

cmd/format-xl.go

@@ -19,15 +19,17 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"reflect"
-
-	"encoding/hex"
+	"sync"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	sha256 "github.com/minio/sha256-simd"
 )
@@ -424,7 +426,7 @@ func checkFormatXLValue(formatXL *formatXLV3) error {
 }
 
 // Check all format values.
-func checkFormatXLValues(formats []*formatXLV3) error {
+func checkFormatXLValues(formats []*formatXLV3, drivesPerSet int) error {
 	for i, formatXL := range formats {
 		if formatXL == nil {
 			continue
@@ -436,6 +438,9 @@ func checkFormatXLValues(formats []*formatXLV3) error {
 			return fmt.Errorf("%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)",
 				humanize.Ordinal(i+1), len(formats), humanize.Ordinal(i+1), len(formatXL.XL.Sets)*len(formatXL.XL.Sets[0]))
 		}
+		if len(formatXL.XL.Sets[0]) != drivesPerSet {
+			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatXL.XL.Sets[0]), drivesPerSet)
+		}
 	}
 	return nil
 }
@@ -619,24 +624,29 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
 	return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.XL.Sets)
 }
 
-// Initializes meta volume on all input storage disks.
-func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error {
-	// This happens for the first time, but keep this here since this
-	// is the only place where it can be made expensive optimizing all
-	// other calls. Create minio meta volume, if it doesn't exist yet.
+// Initializes meta volume only on local storage disks.
+func initXLMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*formatXLV3) error {
+
+	// Compute the local disks eligible for meta volumes (re)initialization
+	var disksToInit []StorageAPI
+	for index := range storageDisks {
+		if formats[index] == nil || storageDisks[index] == nil || storageDisks[index].Hostname() != "" {
+			// Ignore create meta volume on disks which are not found or not local.
+			continue
+		}
+		disksToInit = append(disksToInit, storageDisks[index])
+	}
 
 	// Initialize errs to collect errors inside go-routine.
-	g := errgroup.WithNErrs(len(storageDisks))
+	g := errgroup.WithNErrs(len(disksToInit))
 
 	// Initialize all disks in parallel.
-	for index := range storageDisks {
+	for index := range disksToInit {
+		// Initialize a new index variable in each loop so each
+		// goroutine will return its own instance of index variable.
 		index := index
 		g.Go(func() error {
-			if formats[index] == nil || storageDisks[index] == nil {
-				// Ignore create meta volume on disks which are not found.
-				return nil
-			}
-			return makeFormatXLMetaVolumes(storageDisks[index])
+			return makeFormatXLMetaVolumes(disksToInit[index])
 		}, index)
 	}
 
@@ -753,16 +763,42 @@ func fixFormatXLV3(storageDisks []StorageAPI, endpoints Endpoints, formats []*fo
 func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, drivesPerSet int, deploymentID string) (*formatXLV3, error) {
 	format := newFormatXLV3(setCount, drivesPerSet)
 	formats := make([]*formatXLV3, len(storageDisks))
+	wantAtMost := ecDrivesNoConfig(drivesPerSet)
 
 	for i := 0; i < setCount; i++ {
+		hostCount := make(map[string]int, drivesPerSet)
 		for j := 0; j < drivesPerSet; j++ {
+			disk := storageDisks[i*drivesPerSet+j]
 			newFormat := format.Clone()
 			newFormat.XL.This = format.XL.Sets[i][j]
 			if deploymentID != "" {
 				newFormat.ID = deploymentID
 			}
+			hostCount[disk.Hostname()]++
 			formats[i*drivesPerSet+j] = newFormat
 		}
+		if len(hostCount) > 0 {
+			var once sync.Once
+			for host, count := range hostCount {
+				if count > wantAtMost {
+					if host == "" {
+						host = "local"
+					}
+					once.Do(func() {
+						if len(hostCount) == 1 {
+							return
+						}
+						logger.Info(" * Set %v:", i+1)
+						for j := 0; j < drivesPerSet; j++ {
+							disk := storageDisks[i*drivesPerSet+j]
+							logger.Info("   - Drive: %s", disk.String())
+						}
+					})
+					logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+
+						"A host failure will result in data becoming unavailable.", host, wantAtMost)
+				}
+			}
+		}
 	}
 
 	// Save formats `format.json` across all disks.
@@ -773,16 +809,26 @@ func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, driv
 	return getFormatXLInQuorum(formats)
 }
 
+// ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
+// It will attempt to read it from env variable and fall back to drives/2.
+func ecDrivesNoConfig(drivesPerSet int) int {
+	ecDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
+	if ecDrives == 0 {
+		cfg, err := storageclass.LookupConfig(nil, drivesPerSet)
+		if err == nil {
+			ecDrives = cfg.Standard.Parity
+		}
+		if ecDrives == 0 {
+			ecDrives = drivesPerSet / 2
+		}
+	}
+	return ecDrives
+}
+
 // Make XL backend meta volumes.
 func makeFormatXLMetaVolumes(disk StorageAPI) error {
 	// Attempt to create MinIO internal buckets.
-	err := disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
-	if err != nil {
-		if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
-			return err
-		}
-	}
-	return nil
+	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
 }
 
 var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

cmd/format-xl_test.go

@@ -101,7 +101,7 @@ func TestFixFormatV3(t *testing.T) {
 		formats[j] = newFormat
 	}
 
-	if err = initFormatXLMetaVolume(storageDisks, formats); err != nil {
+	if err = initXLMetaVolumesInLocalDisks(storageDisks, formats); err != nil {
 		t.Fatal(err)
 	}
 

cmd/fs-v1-helpers.go

@@ -429,7 +429,7 @@ func fsDeleteFile(ctx context.Context, basePath, deletePath string) error {
 		return err
 	}
 
-	if err := deleteFile(basePath, deletePath); err != nil {
+	if err := deleteFile(basePath, deletePath, false); err != nil {
 		if err != errFileNotFound {
 			logger.LogIf(ctx, err)
 		}

cmd/fs-v1-metadata.go

@@ -90,6 +90,7 @@ func (c *FSChecksumInfoV1) UnmarshalJSON(data []byte) error {
 	}
 
 	var info checksuminfo
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	err := json.Unmarshal(data, &info)
 	if err != nil {
 		return err
@@ -181,9 +182,14 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 			objInfo.Expires = t.UTC()
 		}
 	}
+
+	// Add user tags to the object info
+	objInfo.UserTags = m.Meta[xhttp.AmzObjectTagging]
+
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
+	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(m.Meta)
 
 	// All the parts per object.

cmd/fs-v1-multipart.go

@@ -575,7 +575,6 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 		fsMeta.Parts[i] = ObjectPartInfo{
 			Number:     part.PartNumber,
-			ETag:       part.ETag,
 			Size:       fi.Size(),
 			ActualSize: actualSize,
 		}
@@ -683,7 +682,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Deny if WORM is enabled
-	if _, ok := isWORMEnabled(bucket); ok {
+	if isWORMEnabled(bucket) {
 		if _, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)); err == nil {
 			return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
 		}

cmd/fs-v1.go

@@ -19,13 +19,13 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/user"
 	"path"
-	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -35,13 +35,18 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/lifecycle"
+
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
+
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
 	"github.com/minio/minio/pkg/mountinfo"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // Default etag is used for pre-existing objects.
@@ -107,8 +112,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 		return err
 	}
 
-	metaStatsPath := pathJoin(fsPath, minioMetaBackgroundOpsBucket, fsUUID)
-	if err := os.MkdirAll(metaStatsPath, 0777); err != nil {
+	if err := os.MkdirAll(pathJoin(fsPath, minioMetaBackgroundOpsBucket), 0777); err != nil {
 		return err
 	}
 
@@ -129,7 +133,15 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		if err == errMinDiskSize {
 			return nil, err
 		}
-		return nil, config.ErrUnableToWriteInBackend(err)
+		// Show a descriptive error with a hint about how to fix it.
+		var username string
+		if u, err := user.Current(); err == nil {
+			username = u.Username
+		} else {
+			username = "<your-username>"
+		}
+		hint := fmt.Sprintf("Use 'sudo chown %s %s && sudo chmod u+rxw %s' to provide sufficient permissions.", username, fsPath, fsPath)
+		return nil, config.ErrUnableToWriteInBackend(err).Hint(hint)
 	}
 
 	// Assign a new UUID for FS minio mode. Each server instance
@@ -176,9 +188,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
 	// lockers are explicitly 'nil' for FS mode since there are only local lockers
-	return fs.nsMutex.NewNSLock(ctx, nil, bucket, object)
+	return fs.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
 }
 
 // Shutdown - should be called when process shuts down.
@@ -190,7 +202,7 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
 }
 
 // StorageInfo - returns underlying storage statistics.
-func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
+func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) StorageInfo {
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
@@ -216,91 +228,23 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
 	return storageInfo
 }
 
-func (fs *FSObjects) waitForLowActiveIO() error {
-	t := time.NewTicker(lowActiveIOWaitTick)
-	defer t.Stop()
-	for {
-		if atomic.LoadInt64(&fs.activeIOCount) >= fs.maxActiveIOCount {
-			select {
-			case <-GlobalServiceDoneCh:
-				return errors.New("forced exit")
-			case <-t.C:
-				continue
-			}
-		}
-		break
+func (fs *FSObjects) waitForLowActiveIO() {
+	for atomic.LoadInt64(&fs.activeIOCount) >= fs.maxActiveIOCount {
+		time.Sleep(lowActiveIOWaitTick)
 	}
-
-	return nil
-
 }
 
-// crawlAndGetDataUsageInfo returns data usage stats of the current FS deployment
-func (fs *FSObjects) crawlAndGetDataUsageInfo(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-
-	var dataUsageInfoMu sync.Mutex
-	var dataUsageInfo = DataUsageInfo{
-		BucketsSizes:          make(map[string]uint64),
-		ObjectsSizesHistogram: make(map[string]uint64),
-	}
-
-	walkFn := func(origPath string, typ os.FileMode) error {
-
-		select {
-		case <-GlobalServiceDoneCh:
-			return filepath.SkipDir
-		default:
-		}
-
-		if err := fs.waitForLowActiveIO(); err != nil {
-			return filepath.SkipDir
-		}
-
-		path := strings.TrimPrefix(origPath, fs.fsPath)
-		path = strings.TrimPrefix(path, SlashSeparator)
-
-		splits := splitN(path, SlashSeparator, 2)
-		bucket := splits[0]
-		prefix := splits[1]
-
-		if bucket == "" {
-			return nil
-		}
-
-		if isReservedOrInvalidBucket(bucket, false) {
-			return filepath.SkipDir
-		}
-
-		if prefix == "" {
-			dataUsageInfoMu.Lock()
-			dataUsageInfo.BucketsCount++
-			dataUsageInfo.BucketsSizes[bucket] = 0
-			dataUsageInfoMu.Unlock()
-			return nil
-		}
-
-		if typ&os.ModeDir != 0 {
-			return nil
-		}
-
-		// Get file size
-		fi, err := os.Stat(origPath)
+// CrawlAndGetDataUsage returns data usage stats of the current FS deployment
+func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+	dataUsageInfo := updateUsage(fs.fsPath, endCh, fs.waitForLowActiveIO, func(item Item) (int64, error) {
+		// Get file size, symlinks which cannot bex
+		// followed are automatically filtered by fastwalk.
+		fi, err := os.Stat(item.Path)
 		if err != nil {
-			return nil
+			return 0, errSkipFile
 		}
-		size := fi.Size()
-
-		dataUsageInfoMu.Lock()
-		dataUsageInfo.ObjectsCount++
-		dataUsageInfo.ObjectsTotalSize += uint64(size)
-		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
-		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
-		dataUsageInfoMu.Unlock()
-
-		return nil
-	}
-
-	fastWalk(fs.fsPath, walkFn)
+		return fi.Size(), nil
+	})
 
 	dataUsageInfo.LastUpdate = UTCNow()
 	atomic.StoreUint64(&fs.totalUsed, dataUsageInfo.ObjectsTotalSize)
@@ -546,7 +490,6 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 // GetObjectNInfo - returns object info and a reader for object
 // content.
 func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-
 	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
 		return nil, err
 	}
@@ -1022,7 +965,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
 	// Deny if WORM is enabled
-	if _, ok := isWORMEnabled(bucket); ok {
+	if isWORMEnabled(bucket) {
 		if _, err := fsStatFile(ctx, fsNSObjPath); err == nil {
 			return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
 		}
@@ -1115,15 +1058,18 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // is a leaf or non-leaf entry.
 func (fs *FSObjects) listDirFactory() ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
-			return
+			return false, nil
+		}
+		if len(entries) == 0 {
+			return true, nil
 		}
 		sort.Strings(entries)
-		return filterMatchingPrefix(entries, prefixEntry)
+		return false, filterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -1225,6 +1171,58 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
 }
 
+// GetObjectTag - get object tags from an existing object
+func (fs *FSObjects) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	oi, err := fs.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+
+	tags, err := tagging.FromString(oi.UserTags)
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+
+	return tags, nil
+}
+
+// PutObjectTag - replace or add tags to an existing object
+func (fs *FSObjects) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+	fsMeta := fsMetaV1{}
+	wlk, err := fs.rwPool.Write(fsMetaPath)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return toObjectErr(err, bucket, object)
+	}
+	// This close will allow for locks to be synchronized on `fs.json`.
+	defer wlk.Close()
+
+	// Read objects' metadata in `fs.json`.
+	if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil {
+		// For any error to read fsMeta, set default ETag and proceed.
+		fsMeta = fs.defaultFsJSON(object)
+	}
+
+	// clean fsMeta.Meta of tag key, before updating the new tags
+	delete(fsMeta.Meta, xhttp.AmzObjectTagging)
+
+	// Do not update for empty tags
+	if tags != "" {
+		fsMeta.Meta[xhttp.AmzObjectTagging] = tags
+	}
+
+	if _, err = fsMeta.WriteTo(wlk); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+	return nil
+}
+
+// DeleteObjectTag - delete object tags from an existing object
+func (fs *FSObjects) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	return fs.PutObjectTag(ctx, bucket, object, "")
+}
+
 // ReloadFormat - no-op for fs, Valid only for XL.
 func (fs *FSObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
 	logger.LogIf(ctx, NotImplemented{})
@@ -1251,8 +1249,17 @@ func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remo
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 
+// Walk a bucket, optionally prefix recursively, until we have returned
+// all the content to objectInfo channel, it is callers responsibility
+// to allocate a receive channel for ObjectInfo, upon any unhandled
+// error walker returns error. Optionally if context.Done() is received
+// then Walk() stops the walker.
+func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfo, fs.getObjectInfo)
+}
+
 // HealObjects - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
+func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) (e error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }
@@ -1263,12 +1270,6 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 	return []BucketInfo{}, NotImplemented{}
 }
 
-// ListObjectsHeal - list all objects to be healed. Valid only for XL
-func (fs *FSObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return ListObjectsInfo{}, NotImplemented{}
-}
-
 // GetMetrics - no op
 func (fs *FSObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
@@ -1305,6 +1306,21 @@ func (fs *FSObjects) DeleteBucketLifecycle(ctx context.Context, bucket string) e
 	return removeLifecycleConfig(ctx, fs, bucket)
 }
 
+// GetBucketSSEConfig returns bucket encryption config on given bucket
+func (fs *FSObjects) GetBucketSSEConfig(ctx context.Context, bucket string) (*bucketsse.BucketSSEConfig, error) {
+	return getBucketSSEConfig(fs, bucket)
+}
+
+// SetBucketSSEConfig sets bucket encryption config on given bucket
+func (fs *FSObjects) SetBucketSSEConfig(ctx context.Context, bucket string, config *bucketsse.BucketSSEConfig) error {
+	return saveBucketSSEConfig(ctx, fs, bucket, config)
+}
+
+// DeleteBucketSSEConfig deletes bucket encryption config on given bucket
+func (fs *FSObjects) DeleteBucketSSEConfig(ctx context.Context, bucket string) error {
+	return removeBucketSSEConfig(ctx, fs, bucket)
+}
+
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
 	marker := continuationToken
@@ -1346,3 +1362,9 @@ func (fs *FSObjects) IsEncryptionSupported() bool {
 func (fs *FSObjects) IsCompressionSupported() bool {
 	return true
 }
+
+// IsReady - Check if the backend disk is ready to accept traffic.
+func (fs *FSObjects) IsReady(_ context.Context) bool {
+	_, err := os.Stat(fs.fsPath)
+	return err == nil
+}

cmd/fs-v1_test.go

@@ -280,7 +280,7 @@ func TestFSDeleteObject(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with invalid object name
-	if err := fs.DeleteObject(context.Background(), bucketName, "\\"); !isSameType(err, ObjectNameInvalid{}) {
+	if err := fs.DeleteObject(context.Background(), bucketName, "\\"); !isSameType(err, ObjectNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with object does not exist.

cmd/gateway-common.go

@@ -304,9 +304,12 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
 	if err != nil {
 		return false
 	}
-	if _, err = clnt.Do(req); err != nil {
+	resp, err := clnt.Do(req)
+	if err != nil {
+		clnt.CloseIdleConnections()
 		return !xnet.IsNetworkOrHostDown(err)
 	}
+	xhttp.DrainBody(resp.Body)
 	return true
 }
 
@@ -343,7 +346,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = BucketNotEmpty{}
 	case "NoSuchBucketPolicy":
 		err = BucketPolicyNotFound{}
-	case "NoSuchBucketLifecycle":
+	case "NoSuchLifecycleConfiguration":
 		err = BucketLifecycleNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
@@ -414,3 +417,36 @@ func gatewayHandleEnvVars() {
 		}
 	}
 }
+
+// shouldMeterRequest checks whether incoming request should be added to prometheus gateway metrics
+func shouldMeterRequest(req *http.Request) bool {
+	return !(guessIsBrowserReq(req) || guessIsHealthCheckReq(req) || guessIsMetricsReq(req))
+}
+
+// MetricsTransport is a custom wrapper around Transport to track metrics
+type MetricsTransport struct {
+	Transport *http.Transport
+	Metrics   *Metrics
+}
+
+// RoundTrip implements the RoundTrip method for MetricsTransport
+func (m MetricsTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	metered := shouldMeterRequest(r)
+	if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
+		m.Metrics.IncRequests(r.Method)
+		if r.ContentLength > 0 {
+			m.Metrics.IncBytesSent(uint64(r.ContentLength))
+		}
+	}
+	// Make the request to the server.
+	resp, err := m.Transport.RoundTrip(r)
+	if err != nil {
+		return nil, err
+	}
+	if metered && (r.Method == http.MethodGet || r.Method == http.MethodHead) {
+		if r.ContentLength > 0 {
+			m.Metrics.IncBytesReceived(uint64(resp.ContentLength))
+		}
+	}
+	return resp, nil
+}

cmd/gateway-main.go

@@ -35,15 +35,6 @@ import (
 	"github.com/minio/minio/pkg/env"
 )
 
-func init() {
-	logger.Init(GOPATH, GOROOT)
-	logger.RegisterError(config.FmtError)
-
-	// Initialize globalConsoleSys system
-	globalConsoleSys = NewConsoleLogger(context.Background())
-	logger.AddTarget(globalConsoleSys)
-}
-
 var (
 	gatewayCmd = cli.Command{
 		Name:            "gateway",
@@ -165,7 +156,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalServerConfig = srvCfg
 	globalServerConfigMu.Unlock()
 
-	router := mux.NewRouter().SkipClean(true)
+	// Initialize router. `SkipClean(true)` stops gorilla/mux from
+	// normalizing URL path minio/minio#3256
+	// avoid URL path encoding minio/minio#8950
+	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
 
 	if globalEtcdClient != nil {
 		// Enable STS router if etcd is enabled.
@@ -243,12 +237,16 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// sub-systems, make sure that we do not move the above codeblock elsewhere.
 	if enableConfigOps {
 		logger.FatalIf(globalConfigSys.Init(newObject), "Unable to initialize config system")
+		buckets, err := newObject.ListBuckets(context.Background())
+		if err != nil {
+			logger.Fatal(err, "Unable to list buckets")
+		}
 
+		logger.FatalIf(globalNotificationSys.Init(buckets, newObject), "Unable to initialize notification system")
 		// Start watching disk for reloading config, this
 		// is only enabled for "NAS" gateway.
 		globalConfigSys.WatchConfigNASDisk(newObject)
 	}
-
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
 	logger.SetDeploymentID(globalDeploymentID)

cmd/gateway-metrics.go

@@ -37,8 +37,8 @@ type Metrics struct {
 }
 
 // IncBytesReceived - Increase total bytes received from gateway backend
-func (s *Metrics) IncBytesReceived(n int64) {
-	s.bytesReceived.Add(uint64(n))
+func (s *Metrics) IncBytesReceived(n uint64) {
+	s.bytesReceived.Add(n)
 }
 
 // GetBytesReceived - Get total bytes received from gateway backend
@@ -47,8 +47,8 @@ func (s *Metrics) GetBytesReceived() uint64 {
 }
 
 // IncBytesSent - Increase total bytes sent to gateway backend
-func (s *Metrics) IncBytesSent(n int64) {
-	s.bytesSent.Add(uint64(n))
+func (s *Metrics) IncBytesSent(n uint64) {
+	s.bytesSent.Add(n)
 }
 
 // GetBytesSent - Get total bytes received from gateway backend

cmd/gateway-unsupported.go

@@ -21,9 +21,13 @@ import (
 	"errors"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/lifecycle"
+
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/bucket/object/tagging"
+	"github.com/minio/minio/pkg/bucket/policy"
+
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/policy"
 )
 
 // GatewayLocker implements custom NeNSLock implementation
@@ -33,8 +37,8 @@ type GatewayLocker struct {
 }
 
 // NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, object)
+func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
 }
 
 // NewGatewayLayerWithLocker - initialize gateway with locker.
@@ -45,8 +49,14 @@ func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
 // GatewayUnsupported list of unsupported call stubs for gateway.
 type GatewayUnsupported struct{}
 
+// CrawlAndGetDataUsage - crawl is not implemented for gateway
+func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+	logger.CriticalIf(ctx, errors.New("not implemented"))
+	return DataUsageInfo{}
+}
+
 // NewNSLock is a dummy stub for gateway.
-func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
 	return nil
 }
@@ -121,6 +131,21 @@ func (a GatewayUnsupported) DeleteBucketLifecycle(ctx context.Context, bucket st
 	return NotImplemented{}
 }
 
+// GetBucketSSEConfig returns bucket encryption config on given bucket
+func (a GatewayUnsupported) GetBucketSSEConfig(ctx context.Context, bucket string) (*bucketsse.BucketSSEConfig, error) {
+	return nil, NotImplemented{}
+}
+
+// SetBucketSSEConfig sets bucket encryption config on given bucket
+func (a GatewayUnsupported) SetBucketSSEConfig(ctx context.Context, bucket string, config *bucketsse.BucketSSEConfig) error {
+	return NotImplemented{}
+}
+
+// DeleteBucketSSEConfig deletes bucket encryption config on given bucket
+func (a GatewayUnsupported) DeleteBucketSSEConfig(ctx context.Context, bucket string) error {
+	return NotImplemented{}
+}
+
 // ReloadFormat - Not implemented stub.
 func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error {
 	return NotImplemented{}
@@ -141,11 +166,6 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 	return nil, NotImplemented{}
 }
 
-// ListObjectsHeal - Not implemented stub
-func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	return ListObjectsInfo{}, NotImplemented{}
-}
-
 // HealObject - Not implemented stub
 func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
@@ -156,8 +176,13 @@ func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, c
 	return result, NotImplemented{}
 }
 
+// Walk - Not implemented stub
+func (a GatewayUnsupported) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	return NotImplemented{}
+}
+
 // HealObjects - Not implemented stub
-func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
+func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) (e error) {
 	return NotImplemented{}
 }
 
@@ -173,6 +198,24 @@ func (a GatewayUnsupported) GetMetrics(ctx context.Context) (*Metrics, error) {
 	return &Metrics{}, NotImplemented{}
 }
 
+// PutObjectTag - not implemented.
+func (a GatewayUnsupported) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	logger.LogIf(ctx, NotImplemented{})
+	return NotImplemented{}
+}
+
+// GetObjectTag - not implemented.
+func (a GatewayUnsupported) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	logger.LogIf(ctx, NotImplemented{})
+	return tagging.Tagging{}, NotImplemented{}
+}
+
+// DeleteObjectTag - not implemented.
+func (a GatewayUnsupported) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	logger.LogIf(ctx, NotImplemented{})
+	return NotImplemented{}
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (a GatewayUnsupported) IsNotificationSupported() bool {
 	return false
@@ -192,3 +235,8 @@ func (a GatewayUnsupported) IsEncryptionSupported() bool {
 func (a GatewayUnsupported) IsCompressionSupported() bool {
 	return false
 }
+
+// IsReady - No Op.
+func (a GatewayUnsupported) IsReady(_ context.Context) bool {
+	return false
+}