xl: Implement MRF healing (#8470)

Zone abstraction of object layer was returning `nil`
incorrectly under situations where disk healing is
not required. Returning `nil` is considered as healing
successful, which leads to unexpected ReloadFormat()
peer notification calls during startup.

This PR fixes this behavior properly for zones.

.travis.yml

@@ -32,8 +32,7 @@ matrix:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
-        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
-        - make verifiers
+        - make test-race
         - make crosscompile
         - make verify
         - cd browser && npm install && npm run test && cd ..

Dockerfile

@@ -12,12 +12,13 @@ RUN  \
      git clone https://github.com/minio/minio && cd minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
 
-FROM alpine:3.9
+FROM alpine:3.10
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 EXPOSE 9000
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM alpine:3.9
+FROM alpine:3.10
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
@@ -8,7 +8,8 @@ COPY minio /usr/bin/
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.release

@@ -8,7 +8,7 @@ RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio
 
-FROM alpine:3.9
+FROM alpine:3.10
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
@@ -17,7 +17,8 @@ COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_KMS_MASTER_KEY_FILE=kms_master_key
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.simpleci

@@ -34,11 +34,13 @@ USER ci
 RUN make
 RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
 RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
-RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
-RUN make verifiers
+RUN make test-race
 RUN make crosscompile
 RUN make verify
 
+## -- add healing tests
+RUN make verify-healing
+
 #-------------------------------------------------------------
 # Stage 2: Test Frontend
 #-------------------------------------------------------------

Makefile

@@ -64,15 +64,21 @@ test: verifiers build
 	@echo "Running unit tests"
 	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
+test-race: verifiers build
+	@echo "Running unit tests under -race"
+	@(env bash $(PWD)/buildscripts/race.sh)
+
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
-coverage: build
-	@echo "Running all coverage for minio"
-	@(env bash $(PWD)/buildscripts/go-coverage.sh)
+# Verify healing of disks with minio binary
+verify-healing:
+	@echo "Verify healing build with race"
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
 # Builds minio locally.
 build: checks

SECURITY.md

@@ -8,12 +8,12 @@ Whenever there is a security update you just need to upgrade to the latest versi
 ## Reporting a Vulnerability
 
 All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories)
-should be reported by email to security@min.io. Your email will be acknowledged within 48 hours, 
-and you'll receive a more detailed response to your email within 72 hours indicating the next steps 
-in handling your report. 
+should be reported by email to security@min.io. Your email will be acknowledged within 48 hours,
+and you'll receive a more detailed response to your email within 72 hours indicating the next steps
+in handling your report.
 
-Please, provide a detailed explanation of the issue. In particular, outline the type of the security 
-issue (DoS, authentication bypass, information disclouse, ...) and the assumptions you're making (e.g. do 
+Please, provide a detailed explanation of the issue. In particular, outline the type of the security
+issue (DoS, authentication bypass, information disclose, ...) and the assumptions you're making (e.g. do
 you need access credentials for a successful exploit).
 
 If you have not received a reply to your email within 48 hours or you have not heard from the security team
@@ -28,14 +28,14 @@ MinIO uses the following disclosure process:
 
 1. Once the security report is received one member of the security team tries to verify and reproduce
    the issue and determines the impact it has.
-2. A member of the security team will respond and either confrim or reject the security report.
-   If the report is rejected the repsonse explains why.
+2. A member of the security team will respond and either confirm or reject the security report.
+   If the report is rejected the response explains why.
 3. Code is audited to find any potential similar problems.
-4. Fixes are prepared for the latest release. 
+4. Fixes are prepared for the latest release.
 5. On the date that the fixes are applied a security advisory will be published on https://blog.min.io.
    Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing
    the security issue. By default MinIO will **not** publish this information to protect your privacy.
-   
+
 This process can take some time, especially when coordination is required with maintainers of other projects.
-Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we 
-follow the process described above to ensure that disclosures are handled consistently. 
+Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we
+follow the process described above to ensure that disclosures are handled consistently.

buildscripts/go-coverage.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/race.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+for d in $(go list ./... | grep -v browser); do
+    CGO_ENABLED=1 go test -v -race --timeout 20m "$d"
+done

buildscripts/verify-healing.sh

@@ -0,0 +1,197 @@
+#!/bin/bash
+#
+# MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
+
+function start_minio_3_node() {
+    declare -a minio_pids
+    export MINIO_ACCESS_KEY=minio
+    export MINIO_SECRET_KEY=minio123
+
+    for i in $(seq 1 3); do
+        ARGS+=("http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[9000+$i]${WORK_DIR}/$i/6/")
+    done
+
+    "${MINIO[@]}" server --address ":9001" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9001.log" 2>&1 &
+    minio_pids[0]=$!
+
+    "${MINIO[@]}" server --address ":9002" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9002.log" 2>&1 &
+    minio_pids[1]=$!
+
+    "${MINIO[@]}" server --address ":9003" ${ARGS[@]} > "${WORK_DIR}/dist-minio-9003.log" 2>&1 &
+    minio_pids[2]=$!
+
+    sleep "$1"
+    echo "${minio_pids[@]}"
+}
+
+
+function check_online() {
+    for i in $(seq 1 3); do
+        if grep -q 'Server switching to safe mode' ${WORK_DIR}/dist-minio-$[9000+$i].log; then
+            echo "1"
+        fi
+    done
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+function __init__()
+{
+    echo "Initializing environment"
+    mkdir -p "$WORK_DIR"
+    mkdir -p "$MINIO_CONFIG_DIR"
+
+    ## version is purposefully set to '3' for minio to migrate configuration file
+    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+}
+
+function perform_test_1() {
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+
+    echo "Testing in Distributed Erasure setup healing test case 1"
+    echo "Remove the contents of the disks belonging to '2' erasure set"
+
+    rm -rf ${WORK_DIR}/2/*
+
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        if ! kill "$pid"; then
+            cat "${WORK_DIR}/*.log"
+            echo "FAILED"
+            purge "$WORK_DIR"
+            exit 1
+        fi
+    done
+
+    rv=$(check_online)
+    if [ "$rv" == "1" ]; then
+        for pid in "${minio_pids[@]}"; do
+            kill -9 "$pid"
+        done
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-$[9000+$i].log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+}
+
+function perform_test_2() {
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+
+    echo "Testing in Distributed Erasure setup healing test case 2"
+    echo "Remove the contents of the disks belonging to '1' erasure set"
+
+    rm -rf ${WORK_DIR}/1/*/
+
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        if ! kill "$pid"; then
+            cat "${WORK_DIR}/*.log"
+            echo "FAILED"
+            purge "$WORK_DIR"
+            exit 1
+        fi
+    done
+
+    rv=$(check_online)
+
+    if [ "$rv" == "1" ]; then
+        for pid in "${minio_pids[@]}"; do
+            kill -9 "$pid"
+        done
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-$[9000+$i].log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+}
+
+function perform_test_3() {
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+
+    echo "Testing in Distributed Erasure setup healing test case 3"
+    echo "Remove the contents of the disks belonging to '3' erasure set"
+
+    rm -rf ${WORK_DIR}/3/*/
+
+    minio_pids=( $(start_minio_3_node 60) )
+    for pid in "${minio_pids[@]}"; do
+        if ! kill "$pid"; then
+            cat "${WORK_DIR}/*.log"
+            echo "FAILED"
+            purge "$WORK_DIR"
+            exit 1
+        fi
+    done
+
+    rv=$(check_online)
+
+    if [ "$rv" == "1" ]; then
+        for pid in "${minio_pids[@]}"; do
+            kill -9 "$pid"
+        done
+        for i in $(seq 1 3); do
+            echo "server$i log:"
+            cat "${WORK_DIR}/dist-minio-$[9000+$i].log"
+        done
+        echo "FAILED"
+        purge "$WORK_DIR"
+        exit 1
+    fi
+}
+
+function main()
+{
+    perform_test_1
+    perform_test_2
+    perform_test_3
+}
+
+( __init__ "$@" && main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

cmd/admin-handlers-users.go

@@ -484,7 +484,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 

cmd/admin-handlers.go

@@ -571,30 +571,44 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	}
 
 	vars := mux.Vars(r)
-	profiler := vars["profilerType"]
-
+	profiles := strings.Split(vars["profilerType"], ",")
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Start profiling on remote servers.
-	hostErrs := globalNotificationSys.StartProfiling(profiler)
+	globalProfilerMu.Lock()
+	defer globalProfilerMu.Unlock()
 
-	// Start profiling locally as well.
-	{
-		if globalProfiler != nil {
-			globalProfiler.Stop()
+	if globalProfiler == nil {
+		globalProfiler = make(map[string]minioProfiler, 10)
+	}
+
+	// Stop profiler of all types if already running
+	for k, v := range globalProfiler {
+		for _, p := range profiles {
+			if p == k {
+				v.Stop()
+				delete(globalProfiler, k)
+			}
 		}
-		prof, err := startProfiler(profiler, "")
+	}
+
+	// Start profiling on remote servers.
+	var hostErrs []NotificationPeerErr
+	for _, profiler := range profiles {
+		hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(profiler)...)
+
+		// Start profiling locally as well.
+		prof, err := startProfiler(profiler)
 		if err != nil {
 			hostErrs = append(hostErrs, NotificationPeerErr{
 				Host: *thisAddr,
 				Err:  err,
 			})
 		} else {
-			globalProfiler = prof
+			globalProfiler[profiler] = prof
 			hostErrs = append(hostErrs, NotificationPeerErr{
 				Host: *thisAddr,
 			})
@@ -620,7 +634,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 		return
 	}
 
-	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
+	writeSuccessResponseJSON(w, startProfilingResultInBytes)
 }
 
 // dummyFileInfo represents a dummy representation of a profile data file
@@ -979,6 +993,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 
 	var apiErr APIError
 	switch e := err.(type) {
+	case iampolicy.Error:
+		apiErr = APIError{
+			Code:           "XMinioMalformedIAMPolicy",
+			Description:    e.Error(),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
 	case config.Error:
 		apiErr = APIError{
 			Code:           "XMinioConfigError",
@@ -1451,7 +1471,9 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 
 func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
-	targetList, _ := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport())
+
+	// Fetch the targets
+	targetList, _ := notify.RegisterNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport(), nil, true)
 
 	for targetID, target := range targetList.TargetMap() {
 		targetIDStatus := make(map[string]madmin.Status)
@@ -1464,6 +1486,8 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 		list := lambdaMap[targetID.Name]
 		list = append(list, targetIDStatus)
 		lambdaMap[targetID.Name] = list
+		// Close any leaking connections
+		_ = target.Close()
 	}
 
 	notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))

cmd/admin-handlers_test.go

@@ -47,6 +47,10 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// reset global variables to start afresh.
 	resetTestGlobals()
 
+	// Set globalIsXL to indicate that the setup uses an erasure
+	// code backend.
+	globalIsXL = true
+
 	// Initializing objectLayer for HealFormatHandler.
 	objLayer, xlDirs, xlErr := initTestXLObjLayer()
 	if xlErr != nil {
@@ -63,15 +67,6 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 
 	globalEndpoints = mustGetZoneEndpoints(xlDirs...)
 
-	// Set globalIsXL to indicate that the setup uses an erasure
-	// code backend.
-	globalIsXL = true
-
-	// Init global heal state
-	if globalIsXL {
-		globalAllHealState = initHealState()
-	}
-
 	globalConfigSys = NewConfigSys()
 
 	globalIAMSys = NewIAMSys()

cmd/admin-heal-ops.go

@@ -577,44 +577,38 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 func (h *healSequence) healItemsFromSourceCh() error {
 	h.lastHealActivity = UTCNow()
 
-	// Start healing the config prefix.
-	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
+	bucketsOnly := true // heal buckets only, not objects.
+	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)
 	}
 
-	// Start healing the bucket config prefix.
-	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
+	for {
+		select {
+		case path := <-h.sourceCh:
+			var itemType madmin.HealItemType
+			switch {
+			case path == nopHeal:
+				continue
+			case path == SlashSeparator:
+				itemType = madmin.HealItemMetadata
+			case !strings.Contains(path, SlashSeparator):
+				itemType = madmin.HealItemBucket
+			default:
+				itemType = madmin.HealItemObject
+			}
 
-	// Start healing the background ops prefix.
-	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
+			if err := h.queueHealTask(path, itemType); err != nil {
+				logger.LogIf(h.ctx, err)
+			}
 
-	for path := range h.sourceCh {
-
-		var itemType madmin.HealItemType
-		switch {
-		case path == nopHeal:
-			continue
-		case path == SlashSeparator:
-			itemType = madmin.HealItemMetadata
-		case !strings.Contains(path, SlashSeparator):
-			itemType = madmin.HealItemBucket
-		default:
-			itemType = madmin.HealItemObject
+			h.scannedItemsCount++
+			h.lastHealActivity = UTCNow()
+		case <-h.traverseAndHealDoneCh:
+			return nil
+		case <-GlobalServiceDoneCh:
+			return nil
 		}
-
-		if err := h.queueHealTask(path, itemType); err != nil {
-			logger.LogIf(h.ctx, err)
-		}
-
-		h.scannedItemsCount++
-		h.lastHealActivity = UTCNow()
 	}
-
-	return nil
 }
 
 func (h *healSequence) healFromSourceCh() {
@@ -622,7 +616,7 @@ func (h *healSequence) healFromSourceCh() {
 	close(h.traverseAndHealDoneCh)
 }
 
-func (h *healSequence) healItems() error {
+func (h *healSequence) healItems(bucketsOnly bool) error {
 	// Start with format healing
 	if err := h.healDiskFormat(); err != nil {
 		return err
@@ -644,7 +638,7 @@ func (h *healSequence) healItems() error {
 	}
 
 	// Heal buckets and objects
-	return h.healBuckets()
+	return h.healBuckets(bucketsOnly)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -655,7 +649,8 @@ func (h *healSequence) healItems() error {
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
 func (h *healSequence) traverseAndHeal() {
-	if err := h.healItems(); err != nil {
+	bucketsOnly := false // Heals buckets and objects also.
+	if err := h.healItems(bucketsOnly); err != nil {
 		if h.isQuitting() {
 			err = errHealStopSignalled
 		}
@@ -711,14 +706,14 @@ func (h *healSequence) healDiskFormat() error {
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets() error {
+func (h *healSequence) healBuckets(bucketsOnly bool) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(h.bucket)
+		return h.healBucket(h.bucket, bucketsOnly)
 	}
 
 	// Get current object layer instance.
@@ -733,7 +728,7 @@ func (h *healSequence) healBuckets() error {
 	}
 
 	for _, bucket := range buckets {
-		if err = h.healBucket(bucket.Name); err != nil {
+		if err = h.healBucket(bucket.Name, bucketsOnly); err != nil {
 			return err
 		}
 	}
@@ -742,7 +737,7 @@ func (h *healSequence) healBuckets() error {
 }
 
 // healBucket - traverses and heals given bucket
-func (h *healSequence) healBucket(bucket string) error {
+func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
@@ -753,6 +748,10 @@ func (h *healSequence) healBucket(bucket string) error {
 		return err
 	}
 
+	if bucketsOnly {
+		return nil
+	}
+
 	if !h.settings.Recursive {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,

cmd/admin-server-info.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"

cmd/api-errors.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // APIError structure
@@ -142,6 +143,7 @@ const (
 	ErrBadRequest
 	ErrKeyTooLongError
 	ErrInvalidBucketObjectLockConfiguration
+	ErrObjectLockConfigurationNotAllowed
 	ErrObjectLocked
 	ErrInvalidRetentionDate
 	ErrPastObjectLockRetainDate
@@ -731,6 +733,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Bucket is missing ObjectLockConfiguration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectLockConfigurationNotAllowed: {
+		Code:           "InvalidBucketState",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrObjectLocked: {
 		Code:           "InvalidRequest",
 		Description:    "Object is WORM protected and cannot be overwritten",
@@ -1726,8 +1733,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrUnsupportedNotification
 	case BackendDown:
 		apiErr = ErrBackendDown
-	case crypto.Error:
-		apiErr = ErrObjectTampered
 	case ObjectNameTooLong:
 		apiErr = ErrKeyTooLongError
 	default:
@@ -1772,9 +1777,15 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case policy.Error:
+			apiErr = APIError{
+				Code:           "MalformedPolicy",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case crypto.Error:
 			apiErr = APIError{
-				Code:           "XKMSInternalError",
+				Code:           "XMinIOEncryptionError",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}

cmd/api-headers.go

@@ -91,7 +91,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if !objInfo.Expires.IsZero() {
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
-
+	if globalCacheConfig.Enabled {
+		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
+		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
+	}
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		if HasPrefix(k, ReservedMetadataPrefix) {

cmd/auth-handler.go

@@ -226,7 +226,7 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.
-		p, pok := claims[iamPolicyName()]
+		p, pok := claims[iamPolicyClaimName()]
 		if !pok {
 			return nil, errAuthentication
 		}

cmd/background-heal-ops.go

@@ -61,6 +61,7 @@ func (h *healRoutine) run() {
 			if !ok {
 				break
 			}
+
 			if httpServer := newHTTPServerFn(); httpServer != nil {
 				// Wait at max 10 minute for an inprogress request before proceeding to heal
 				waitCount := 600

cmd/background-newdisks-heal-ops.go

@@ -62,10 +62,11 @@ func monitorLocalDisksAndHeal() {
 		time.Sleep(time.Second)
 	}
 
-	// Perform automatic disk healing when a new one is inserted
+	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		time.Sleep(defaultMonitorNewDiskInterval)
 
+		// Attempt a heal as the server starts-up first.
 		localDisksInZoneHeal := make([]Endpoints, len(z.zones))
 		for i, ep := range globalEndpoints {
 			localDisksToHeal := Endpoints{}
@@ -88,6 +89,7 @@ func monitorLocalDisksAndHeal() {
 
 		// Reformat disks
 		bgSeq.sourceCh <- SlashSeparator
+
 		// Ensure that reformatting disks is finished
 		bgSeq.sourceCh <- nopHeal
 

cmd/bootstrap-peer-server.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"

cmd/bucket-handlers.go

@@ -1017,7 +1017,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	if err != nil {
 		aerr := toAPIError(ctx, err)
 		if err == errConfigNotFound {
-			aerr = errorCodes.ToAPIErr(ErrMethodNotAllowed)
+			aerr = errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed)
 		}
 		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-policy-handlers.go

@@ -77,7 +77,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/config-encrypted.go

@@ -84,7 +84,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 			return nil
 		}
 		if !globalActiveCred.IsValid() {
-			return errInvalidArgument
+			return config.ErrMissingCredentialsBackendEncrypted(nil)
 		}
 	}
 
@@ -267,7 +267,12 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 
 		if !utf8.Valid(data) {
-			return errors.New("config data not in plain-text form")
+			_, err = decryptData(data, globalActiveCred)
+			if err == nil {
+				// Config is already encrypted with right keys
+				continue
+			}
+			return errors.New("config data not in plain-text form or encrypted")
 		}
 
 		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -279,9 +284,11 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 			return err
 		}
 	}
-	if encrypted && globalActiveCred.IsValid() {
+
+	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
+
 	return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
 }
 
@@ -309,7 +316,8 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 
 	var marker string
 	for {
-		res, err := objAPI.ListObjects(context.Background(), minioMetaBucket, minioConfigPrefix, marker, "", maxObjectList)
+		res, err := objAPI.ListObjects(context.Background(), minioMetaBucket,
+			minioConfigPrefix, marker, "", maxObjectList)
 		if err != nil {
 			return err
 		}
@@ -339,7 +347,12 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 			}
 
 			if !utf8.Valid(data) {
-				return errors.New("config data not in plain-text form")
+				_, err = decryptData(data, globalActiveCred)
+				if err == nil {
+					// Config is already encrypted with right keys
+					continue
+				}
+				return errors.New("config data not in plain-text form or encrypted")
 			}
 
 			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -359,7 +372,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		marker = res.NextMarker
 	}
 
-	if encrypted && globalActiveCred.IsValid() {
+	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
 

cmd/config.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
@@ -170,6 +171,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	}
 
 	var config = config.New()
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(configData, &config); err != nil {
 		return nil, err
 	}

cmd/config/cache/lookup.go

@@ -121,9 +121,7 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 			return cfg, config.ErrInvalidCacheQuota(err)
 		}
 		cfg.Quota = cfg.MaxUse
-	}
-
-	if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
+	} else if quotaStr := env.Get(EnvCacheQuota, kvs.Get(Quota)); quotaStr != "" {
 		cfg.Quota, err = strconv.Atoi(quotaStr)
 		if err != nil {
 			return cfg, config.ErrInvalidCacheQuota(err)

cmd/config/config.go

@@ -24,7 +24,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
@@ -621,6 +621,12 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 	currKVS, ok := c[subSys][tgt]
 	if !ok {
 		currKVS = defaultKVS[subSys]
+	} else {
+		for _, kv := range defaultKVS[subSys] {
+			if _, ok = currKVS.Lookup(kv.Key); !ok {
+				currKVS.Set(kv.Key, kv.Value)
+			}
+		}
 	}
 
 	for _, kv := range kvs {

cmd/config/errors-utils.go

@@ -20,7 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net"
 	"syscall"
 
 	"github.com/minio/minio/pkg/color"
@@ -106,9 +106,11 @@ func ErrorToErr(err error) Err {
 	// Show a generic message for known golang errors
 	if errors.Is(err, syscall.EADDRINUSE) {
 		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
-	} else if errors.Is(err, syscall.EACCES) {
-		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
-	} else if os.IsPermission(err) {
+	} else if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) {
+		switch err.(type) {
+		case *net.OpError:
+			return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+		}
 		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
 	} else if errors.Is(err, io.ErrUnexpectedEOF) {
 		return ErrUnexpectedDataContent(err)

cmd/config/etcd/dns/etcd_dns.go

@@ -117,9 +117,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		if err != nil {
 			return nil, err
 		}
-		if r.Count == 0 {
-			return nil, ErrNoEntriesFound
-		}
 	}
 
 	var srvRecords []SrvRecord
@@ -143,9 +140,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		srvRecords = append(srvRecords, srvRecord)
 
 	}
-	if len(srvRecords) == 0 {
-		return nil, ErrNoEntriesFound
-	}
 	sort.Slice(srvRecords, func(i int, j int) bool {
 		return srvRecords[i].Key < srvRecords[j].Key
 	})

cmd/config/identity/openid/help.go

@@ -32,9 +32,15 @@ var (
 			Type:        "string",
 			Optional:    true,
 		},
+		config.HelpKV{
+			Key:         ClaimName,
+			Description: `JWT canned policy claim name, defaults to "policy"`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         ClaimPrefix,
-			Description: `JWT claim namespace prefix e.g. "customer1"`,
+			Description: `JWT claim namespace prefix e.g. "customer1/"`,
 			Optional:    true,
 			Type:        "string",
 		},

cmd/config/identity/openid/jwt.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
@@ -41,6 +42,7 @@ type Config struct {
 	} `json:"jwks"`
 	URL          *xnet.URL `json:"url,omitempty"`
 	ClaimPrefix  string    `json:"claimPrefix,omitempty"`
+	ClaimName    string    `json:"claimName,omitempty"`
 	DiscoveryDoc DiscoveryDoc
 	ClientID     string
 	publicKeys   map[string]crypto.PublicKey
@@ -209,12 +211,14 @@ func (p *JWT) ID() ID {
 const (
 	JwksURL     = "jwks_url"
 	ConfigURL   = "config_url"
+	ClaimName   = "claim_name"
 	ClaimPrefix = "claim_prefix"
 	ClientID    = "client_id"
 
 	EnvIdentityOpenIDClientID    = "MINIO_IDENTITY_OPENID_CLIENT_ID"
 	EnvIdentityOpenIDJWKSURL     = "MINIO_IDENTITY_OPENID_JWKS_URL"
 	EnvIdentityOpenIDURL         = "MINIO_IDENTITY_OPENID_CONFIG_URL"
+	EnvIdentityOpenIDClaimName   = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
 	EnvIdentityOpenIDClaimPrefix = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
 )
 
@@ -271,6 +275,10 @@ var (
 			Key:   ClientID,
 			Value: "",
 		},
+		config.KV{
+			Key:   ClaimName,
+			Value: iampolicy.PolicyName,
+		},
 		config.KV{
 			Key:   ClaimPrefix,
 			Value: "",
@@ -299,6 +307,7 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	}
 
 	c = Config{
+		ClaimName:   env.Get(EnvIdentityOpenIDClaimName, kvs.Get(ClaimName)),
 		ClaimPrefix: env.Get(EnvIdentityOpenIDClaimPrefix, kvs.Get(ClaimPrefix)),
 		publicKeys:  make(map[string]crypto.PublicKey),
 		ClientID:    env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
@@ -317,6 +326,11 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 			return c, err
 		}
 	}
+
+	if c.ClaimName == "" {
+		c.ClaimName = iampolicy.PolicyName
+	}
+
 	if jwksURL == "" {
 		// Fallback to discovery document jwksURL
 		jwksURL = c.DiscoveryDoc.JwksURI
@@ -330,9 +344,11 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	if err != nil {
 		return c, err
 	}
+
 	if err = c.PopulatePublicKey(); err != nil {
 		return c, err
 	}
+
 	return c, nil
 }
 

cmd/config/notify/parse.go

@@ -118,7 +118,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -134,7 +134,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 
@@ -152,7 +152,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 			continue
 		}
 		args.TLS.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -169,7 +169,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 			continue
 		}
 		args.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -185,7 +185,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -201,7 +201,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -217,7 +217,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -233,7 +233,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -249,7 +249,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf)
+		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
 			return nil, err
 		}
@@ -265,7 +265,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport)
+		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport, test)
 		if err != nil {
 			return nil, err
 		}

cmd/crypto/config.go

@@ -16,7 +16,6 @@ package crypto
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
 	"reflect"
 	"strconv"
@@ -342,7 +341,7 @@ func LookupVaultConfig(kvs config.KVS) (VaultConfig, error) {
 	if keyVersion := env.Get(EnvKMSVaultKeyVersion, kvs.Get(KMSVaultKeyVersion)); keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
-			return vcfg, fmt.Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
+			return vcfg, Errorf("Unable to parse VaultKeyVersion value (`%s`)", keyVersion)
 		}
 	}
 

cmd/crypto/error.go

@@ -14,60 +14,79 @@
 
 package crypto
 
-import "errors"
+import (
+	"fmt"
+)
 
 // Error is the generic type for any error happening during decrypting
 // an object. It indicates that the object itself or its metadata was
 // modified accidentally or maliciously.
-type Error string
+type Error struct {
+	err error
+}
 
-func (e Error) Error() string { return string(e) }
+// Errorf - formats according to a format specifier and returns
+// the string as a value that satisfies error of type crypto.Error
+func Errorf(format string, a ...interface{}) error {
+	return Error{err: fmt.Errorf(format, a...)}
+}
+
+// Unwrap the internal error.
+func (e Error) Unwrap() error { return e.err }
+
+// Error 'error' compatible method.
+func (e Error) Error() string {
+	if e.err == nil {
+		return "crypto: cause <nil>"
+	}
+	return e.err.Error()
+}
 
 var (
 	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
 	// is not supported.
-	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+	ErrInvalidEncryptionMethod = Errorf("The encryption method is not supported")
 
 	// ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
 	// is not supported.
-	ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
+	ErrInvalidCustomerAlgorithm = Errorf("The SSE-C algorithm is not supported")
 
 	// ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
-	ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
+	ErrMissingCustomerKey = Errorf("The SSE-C request is missing the customer key")
 
 	// ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
 	// MD5 checksum.
-	ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
+	ErrMissingCustomerKeyMD5 = Errorf("The SSE-C request is missing the customer key MD5")
 
 	// ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
 	// base64-encoded string or not 256 bits long.
-	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
+	ErrInvalidCustomerKey = Errorf("The SSE-C client key is invalid")
 
 	// ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key)
 	// does not match the secret key used during encrypting the object.
-	ErrSecretKeyMismatch = errors.New("The secret key does not match the secret key used during upload")
+	ErrSecretKeyMismatch = Errorf("The secret key does not match the secret key used during upload")
 
 	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
 	// computed MD5 sum. This means that the client provided either the wrong key for
 	// a certain MD5 checksum or the wrong MD5 for a certain key.
-	ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
+	ErrCustomerKeyMD5Mismatch = Errorf("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
 	// ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
 	// The client needs to remove the SSE-S3 header or the SSE-C headers
-	ErrIncompatibleEncryptionMethod = errors.New("Server side encryption specified with both SSE-C and SSE-S3 headers")
+	ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers")
 )
 
-const (
-	errMissingInternalIV            Error = "The object metadata is missing the internal encryption IV"
-	errMissingInternalSealAlgorithm Error = "The object metadata is missing the internal seal algorithm"
+var (
+	errMissingInternalIV            = Errorf("The object metadata is missing the internal encryption IV")
+	errMissingInternalSealAlgorithm = Errorf("The object metadata is missing the internal seal algorithm")
 
-	errInvalidInternalIV            Error = "The internal encryption IV is malformed"
-	errInvalidInternalSealAlgorithm Error = "The internal seal algorithm is invalid and not supported"
+	errInvalidInternalIV            = Errorf("The internal encryption IV is malformed")
+	errInvalidInternalSealAlgorithm = Errorf("The internal seal algorithm is invalid and not supported")
 
-	errMissingUpdatedKey Error = "The key update returned no error but also no sealed key"
+	errMissingUpdatedKey = Errorf("The key update returned no error but also no sealed key")
 )
 
 var (
 	// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
 	// to produce enough random data. This is fatal error and should cause a panic.
-	errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
+	errOutOfEntropy = Errorf("Unable to read enough randomness from the system")
 )

cmd/crypto/kes.go

@@ -19,7 +19,6 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -76,13 +75,13 @@ type KesConfig struct {
 func (k KesConfig) Verify() (err error) {
 	switch {
 	case k.Endpoint == "":
-		err = errors.New("crypto: missing kes endpoint")
+		err = Errorf("crypto: missing kes endpoint")
 	case k.CertFile == "":
-		err = errors.New("crypto: missing cert file")
+		err = Errorf("crypto: missing cert file")
 	case k.KeyFile == "":
-		err = errors.New("crypto: missing key file")
+		err = Errorf("crypto: missing key file")
 	case k.DefaultKeyID == "":
-		err = errors.New("crypto: missing default key id")
+		err = Errorf("crypto: missing default key id")
 	}
 	return err
 }
@@ -153,7 +152,7 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 		return key, nil, err
 	}
 	if len(plainKey) != len(key) {
-		return key, nil, errors.New("crypto: received invalid plaintext key size from KMS")
+		return key, nil, Errorf("crypto: received invalid plaintext key size from KMS")
 	}
 	copy(key[:], plainKey)
 	return key, sealedKey, nil
@@ -176,7 +175,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 		return key, err
 	}
 	if len(plainKey) != len(key) {
-		return key, errors.New("crypto: received invalid plaintext key size from KMS")
+		return key, Errorf("crypto: received invalid plaintext key size from KMS")
 	}
 	copy(key[:], plainKey)
 	return key, nil
@@ -301,7 +300,7 @@ func (c *kesClient) parseErrorResponse(resp *http.Response) error {
 	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
 		return err
 	}
-	return fmt.Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
+	return Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
 }
 
 // loadCACertificates returns a new CertPool
@@ -334,7 +333,7 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 		if os.IsNotExist(err) || os.IsPermission(err) {
 			return rootCAs, nil
 		}
-		return nil, fmt.Errorf("crypto: cannot open '%s': %v", path, err)
+		return nil, Errorf("crypto: cannot open '%s': %v", path, err)
 	}
 
 	// If path is a file, parse as PEM-encoded certifcate
@@ -346,7 +345,7 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 			return nil, err
 		}
 		if !rootCAs.AppendCertsFromPEM(cert) {
-			return nil, fmt.Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
+			return nil, Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
 		}
 		return rootCAs, nil
 	}

cmd/crypto/key.go

@@ -21,7 +21,6 @@ import (
 	"crypto/rand"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io"
 	"path"
 
@@ -108,7 +107,7 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 	)
 	switch sealedKey.Algorithm {
 	default:
-		return Error(fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm))
+		return Errorf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)
 	case SealAlgorithm:
 		mac := hmac.New(sha256.New, extKey[:])
 		mac.Write(sealedKey.IV[:])

cmd/crypto/legacy.go

@@ -17,7 +17,6 @@
 package crypto
 
 import (
-	"fmt"
 	"reflect"
 	"strconv"
 
@@ -167,7 +166,7 @@ func lookupConfigLegacy(kvs config.KVS) (VaultConfig, error) {
 	if keyVersion := env.Get(EnvLegacyVaultKeyVersion, ""); keyVersion != "" {
 		vcfg.Key.Version, err = strconv.Atoi(keyVersion)
 		if err != nil {
-			return vcfg, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
+			return vcfg, Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)",
 				EnvLegacyVaultKeyVersion, keyVersion)
 		}
 	}

cmd/crypto/metadata.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/base64"
 	"errors"
-	"fmt"
 
 	"github.com/minio/minio/cmd/logger"
 )
@@ -126,7 +125,7 @@ func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 // is nil.
 func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
 	}
 
 	// There are two possibilites:
@@ -172,7 +171,7 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	b64SealedKey, ok := metadata[S3SealedKey]
 	if !ok {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed key for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
 	}
 
 	// There are two possibilites:
@@ -182,10 +181,10 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	keyID, idPresent := metadata[S3KMSKeyID]
 	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
 	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal KMS key-ID for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
 	}
 	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -198,12 +197,12 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Error("The internal sealed key for SSE-S3 is invalid")
+		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
 		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
 		if err != nil {
-			return keyID, kmsKey, sealedKey, Error("The internal sealed KMS data key for SSE-S3 is invalid")
+			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
 		}
 	}
 
@@ -217,7 +216,7 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 // It allocates a new metadata map if metadata is nil.
 func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
-		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
+		logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
 	}
 
 	if metadata == nil {
@@ -244,7 +243,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	b64SealedKey, ok := metadata[SSECSealedKey]
 	if !ok {
-		return sealedKey, Error("The object metadata is missing the internal sealed key for SSE-C")
+		return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -257,7 +256,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Error("The internal sealed key for SSE-C is invalid")
+		return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
 	}
 
 	sealedKey.Algorithm = algorithm

cmd/crypto/metadata_test.go

@@ -125,15 +125,15 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
-		ExpectedErr: Error("The object metadata is missing the internal KMS key-ID for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
@@ -150,7 +150,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
-		ExpectedErr: Error("The internal sealed key for SSE-S3 is invalid"),
+		ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
 			S3KMSKeyID: "", S3KMSSealedKey: "",
@@ -158,7 +158,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
-		ExpectedErr: Error("The internal sealed KMS data key for SSE-S3 is invalid"),
+		ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
 			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
@@ -188,9 +188,17 @@ var s3ParseMetadataTests = []struct {
 func TestS3ParseMetadata(t *testing.T) {
 	for i, test := range s3ParseMetadataTests {
 		keyID, dataKey, sealedKey, err := S3.ParseMetadata(test.Metadata)
-		if err != test.ExpectedErr {
+		if err != nil && test.ExpectedErr == nil {
 			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
 		}
+		if err == nil && test.ExpectedErr != nil {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if err != nil && test.ExpectedErr != nil {
+			if err.Error() != test.ExpectedErr.Error() {
+				t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+			}
+		}
 		if !bytes.Equal(dataKey, test.DataKey) {
 			t.Errorf("Test %d: got data key '%v' - want data key '%v'", i, dataKey, test.DataKey)
 		}
@@ -218,7 +226,7 @@ var ssecParseMetadataTests = []struct {
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
 	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
-		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-C"),
+		ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
@@ -233,7 +241,7 @@ var ssecParseMetadataTests = []struct {
 		SealedKey: SealedKey{},
 	}, // 4
 	{
-		ExpectedErr: Error("The internal sealed key for SSE-C is invalid"),
+		ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
 		},
@@ -267,9 +275,17 @@ func TestCreateMultipartMetadata(t *testing.T) {
 func TestSSECParseMetadata(t *testing.T) {
 	for i, test := range ssecParseMetadataTests {
 		sealedKey, err := SSEC.ParseMetadata(test.Metadata)
-		if err != test.ExpectedErr {
+		if err != nil && test.ExpectedErr == nil {
 			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
 		}
+		if err == nil && test.ExpectedErr != nil {
+			t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+		}
+		if err != nil && test.ExpectedErr != nil {
+			if err.Error() != test.ExpectedErr.Error() {
+				t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr)
+			}
+		}
 		if sealedKey.Algorithm != test.SealedKey.Algorithm {
 			t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm)
 		}

cmd/crypto/parse.go

@@ -16,7 +16,6 @@ package crypto
 
 import (
 	"encoding/hex"
-	"fmt"
 	"strings"
 )
 
@@ -25,18 +24,18 @@ import (
 func ParseMasterKey(envArg string) (KMS, error) {
 	values := strings.SplitN(envArg, ":", 2)
 	if len(values) != 2 {
-		return nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
+		return nil, Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
 	}
 	var (
 		keyID  = values[0]
 		hexKey = values[1]
 	)
 	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
-		return nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
+		return nil, Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
 	}
 	var masterKey [32]byte
 	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
-		return nil, err
+		return nil, Errorf("Invalid KMS master key: %v", err)
 	}
 	return NewMasterKey(keyID, masterKey), nil
 }

cmd/crypto/vault.go

@@ -17,7 +17,6 @@ package crypto
 import (
 	"bytes"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -27,7 +26,7 @@ import (
 
 var (
 	//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
-	ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
+	ErrKMSAuthLogin = Errorf("Vault service did not return auth info")
 )
 
 // VaultKey represents vault encryption key-ring.
@@ -75,17 +74,17 @@ var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements
 func (v *VaultConfig) Verify() (err error) {
 	switch {
 	case v.Endpoint == "":
-		err = errors.New("crypto: missing hashicorp vault endpoint")
+		err = Errorf("crypto: missing hashicorp vault endpoint")
 	case strings.ToLower(v.Auth.Type) != "approle":
-		err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
+		err = Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
 	case v.Auth.AppRole.ID == "":
-		err = errors.New("crypto: missing hashicorp vault AppRole ID")
+		err = Errorf("crypto: missing hashicorp vault AppRole ID")
 	case v.Auth.AppRole.Secret == "":
-		err = errors.New("crypto: missing hashicorp vault AppSecret ID")
+		err = Errorf("crypto: missing hashicorp vault AppSecret ID")
 	case v.Key.Name == "":
-		err = errors.New("crypto: missing hashicorp vault key name")
+		err = Errorf("crypto: missing hashicorp vault key name")
 	case v.Key.Version < 0:
-		err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
+		err = Errorf("crypto: invalid hashicorp vault key version: The key version must not be negative")
 	}
 	return
 }
@@ -107,7 +106,7 @@ func NewVault(config VaultConfig) (KMS, error) {
 	}
 	client, err := vault.NewClient(&vaultCfg)
 	if err != nil {
-		return nil, err
+		return nil, Errorf("crypto: client error %w", err)
 	}
 	if config.Namespace != "" {
 		client.SetNamespace(config.Namespace)
@@ -167,6 +166,7 @@ func (v *vaultService) authenticate() (err error) {
 	var secret *vault.Secret
 	secret, err = v.client.Logical().Write("auth/approle/login", payload)
 	if err != nil {
+		err = Errorf("crypto: client error %w", err)
 		return
 	}
 	if secret == nil {
@@ -217,14 +217,23 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
-		return key, sealedKey, err
+		return key, sealedKey, Errorf("crypto: client error %w", err)
 	}
-	sealKey := s.Data["ciphertext"].(string)
-	plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
+	sealKey, ok := s.Data["ciphertext"].(string)
+	if !ok {
+		return key, sealedKey, Errorf("crypto: incorrect 'ciphertext' key type %v", s.Data["ciphertext"])
+	}
+
+	plainKeyB64, ok := s.Data["plaintext"].(string)
+	if !ok {
+		return key, sealedKey, Errorf("crypto: incorrect 'plaintext' key type %v", s.Data["plaintext"])
+	}
+
+	plainKey, err := base64.StdEncoding.DecodeString(plainKeyB64)
 	if err != nil {
-		return key, sealedKey, err
+		return key, sealedKey, Errorf("crypto: invalid base64 key %w", err)
 	}
-	copy(key[:], []byte(plainKey))
+	copy(key[:], plainKey)
 	return key, []byte(sealKey), nil
 }
 
@@ -243,16 +252,22 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 		"ciphertext": string(sealedKey),
 		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
 	}
+
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
 	if err != nil {
-		return key, err
+		return key, Errorf("crypto: client error %w", err)
 	}
-	base64Key := s.Data["plaintext"].(string)
+
+	base64Key, ok := s.Data["plaintext"].(string)
+	if !ok {
+		return key, Errorf("crypto: incorrect 'plaintext' key type %v", s.Data["plaintext"])
+	}
+
 	plainKey, err := base64.StdEncoding.DecodeString(base64Key)
 	if err != nil {
-		return key, err
+		return key, Errorf("crypto: invalid base64 key %w", err)
 	}
-	copy(key[:], []byte(plainKey))
+	copy(key[:], plainKey)
 	return key, nil
 }
 
@@ -273,7 +288,7 @@ func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (r
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {
-		return nil, err
+		return nil, Errorf("crypto: client error %w", err)
 	}
 	ciphertext, ok := s.Data["ciphertext"]
 	if !ok {

cmd/data-usage.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -133,6 +134,7 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 	}
 
 	var dataUsageInfo DataUsageInfo
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
 	if err != nil {
 		return DataUsageInfo{}, err

cmd/disk-cache-backend.go

@@ -89,8 +89,10 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 	}
 
 	o = ObjectInfo{
-		Bucket: bucket,
-		Name:   object,
+		Bucket:            bucket,
+		Name:              object,
+		CacheStatus:       CacheHit,
+		CacheLookupStatus: CacheHit,
 	}
 
 	// We set file info only if its valid.

cmd/disk-cache-utils.go

@@ -28,6 +28,24 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 )
 
+// CacheStatusType - whether the request was served from cache.
+type CacheStatusType string
+
+const (
+	// CacheHit - whether object was served from cache.
+	CacheHit CacheStatusType = "HIT"
+
+	// CacheMiss - object served from backend.
+	CacheMiss CacheStatusType = "MISS"
+)
+
+func (c CacheStatusType) String() string {
+	if c != "" {
+		return string(c)
+	}
+	return string(CacheMiss)
+}
+
 type cacheControl struct {
 	expiry       time.Time
 	maxAge       int

cmd/disk-cache.go

@@ -223,7 +223,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		}
 		if cc.noStore {
 			c.cacheStats.incMiss()
-			return c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+			bReader, err := c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+			bReader.ObjInfo.CacheLookupStatus = CacheHit
+			bReader.ObjInfo.CacheStatus = CacheMiss
+			return bReader, err
 		}
 	}
 
@@ -272,8 +275,14 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		default:
 		}
 	}
+
+	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	// Record if cache has a hit that was invalidated by ETag verification
+	if cacheErr == nil {
+		bkReader.ObjInfo.CacheLookupStatus = CacheHit
+	}
 	if !dcache.diskAvailable(objInfo.Size) {
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return bkReader, bkErr
 	}
 
 	if rs != nil {
@@ -290,12 +299,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 				c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
 			}
 		}()
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
-	}
-	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
-	if bkErr != nil {
-		return nil, bkErr
+		return bkReader, bkErr
 	}
+
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)

cmd/endpoint-ellipses.go

@@ -76,7 +76,8 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 	for _, totalSize := range totalSizes {
 		// Check if totalSize has minimum range upto setSize
 		if totalSize < setSizes[0] || totalSize < customSetDriveCount {
-			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
+			msg := fmt.Sprintf("Incorrect number of endpoints provided %s", args)
+			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 		}
 	}
 
@@ -120,7 +121,8 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s", args)
+		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 
 	for i := range totalSizes {
@@ -203,18 +205,7 @@ func parseEndpointSet(customSetDriveCount uint64, args ...string) (ep endpointSe
 // specific set size.
 // For example: {1...64} is divided into 4 sets each of size 16.
 // This applies to even distributed setup syntax as well.
-func GetAllSets(args ...string) ([][]string, error) {
-	var customSetDriveCount uint64
-	if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
-		customSetDriveCount, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return nil, config.ErrInvalidErasureSetSize(err)
-		}
-		if !isValidSetSize(customSetDriveCount) {
-			return nil, config.ErrInvalidErasureSetSize(nil)
-		}
-	}
-
+func GetAllSets(customSetDriveCount uint64, args ...string) ([][]string, error) {
 	var setArgs [][]string
 	if !ellipses.HasEllipses(args...) {
 		var setIndexes [][]uint64
@@ -257,16 +248,26 @@ func GetAllSets(args ...string) ([][]string, error) {
 
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
-func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, int, SetupType, error) {
+func createServerEndpoints(serverAddr string, args ...string) (
+	endpointZones EndpointZones, setDriveCount int,
+	setupType SetupType, err error) {
+
 	if len(args) == 0 {
 		return nil, -1, -1, errInvalidArgument
 	}
 
-	var endpointZones EndpointZones
-	var setupType SetupType
-	var drivesPerSet int
+	if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
+		setDriveCount, err = strconv.Atoi(v)
+		if err != nil {
+			return nil, -1, -1, config.ErrInvalidErasureSetSize(err)
+		}
+		if !isValidSetSize(uint64(setDriveCount)) {
+			return nil, -1, -1, config.ErrInvalidErasureSetSize(nil)
+		}
+	}
+
 	if !ellipses.HasEllipses(args...) {
-		setArgs, err := GetAllSets(args...)
+		setArgs, err := GetAllSets(uint64(setDriveCount), args...)
 		if err != nil {
 			return nil, -1, -1, err
 		}
@@ -283,43 +284,37 @@ func createServerEndpoints(serverAddr string, args ...string) (EndpointZones, in
 		return endpointZones, len(setArgs[0]), setupType, nil
 	}
 
+	var prevSetupType SetupType
 	var foundPrevLocal bool
-
-	// Verify the args setup-type appropriately.
-	{
-		setArgs, err := GetAllSets(args...)
-		if err != nil {
-			return nil, -1, -1, err
-		}
-
-		var endpoints Endpoints
-		endpoints, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
-		if err != nil {
-			return nil, -1, -1, err
-		}
-
-		foundPrevLocal = endpoints.atleastOneEndpointLocal()
-	}
-
 	for _, arg := range args {
-		setArgs, err := GetAllSets(arg)
+		setArgs, err := GetAllSets(uint64(setDriveCount), arg)
 		if err != nil {
 			return nil, -1, -1, err
 		}
-		endpointList, _, err := CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
+		var endpointList Endpoints
+		endpointList, setupType, err = CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
 		if err != nil {
 			return nil, -1, -1, err
 		}
-		if drivesPerSet != 0 && drivesPerSet != len(setArgs[0]) {
-			return nil, -1, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", drivesPerSet, len(setArgs[0]))
+		if setDriveCount != 0 && setDriveCount != len(setArgs[0]) {
+			return nil, -1, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
 		}
-		endpointZones = append(endpointZones, ZoneEndpoints{
+		if prevSetupType != UnknownSetupType && prevSetupType != setupType {
+			return nil, -1, -1, fmt.Errorf("All zones should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
+		}
+		if err = endpointZones.Add(ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
-		})
-		drivesPerSet = len(setArgs[0])
+		}); err != nil {
+			return nil, -1, -1, err
+		}
+		foundPrevLocal = endpointList.atleastOneEndpointLocal()
+		if setDriveCount == 0 {
+			setDriveCount = len(setArgs[0])
+		}
+		prevSetupType = setupType
 	}
 
-	return endpointZones, drivesPerSet, setupType, nil
+	return endpointZones, setDriveCount, setupType, nil
 }

cmd/endpoint.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
@@ -53,8 +52,7 @@ const (
 // Endpoint - any type of endpoint.
 type Endpoint struct {
 	*url.URL
-	IsLocal  bool
-	SetIndex int
+	IsLocal bool
 }
 
 func (endpoint Endpoint) String() string {
@@ -194,6 +192,24 @@ type ZoneEndpoints struct {
 // EndpointZones - list of list of endpoints
 type EndpointZones []ZoneEndpoints
 
+// Add add zone endpoints
+func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
+	existSet := set.NewStringSet()
+	for _, zep := range *l {
+		for _, ep := range zep.Endpoints {
+			existSet.Add(ep.String())
+		}
+	}
+	// Validate if there are duplicate endpoints across zones
+	for _, ep := range zeps.Endpoints {
+		if existSet.Contains(ep.String()) {
+			return fmt.Errorf("duplicate endpoints found")
+		}
+	}
+	*l = append(*l, zeps)
+	return nil
+}
+
 // FirstLocal returns true if the first endpoint is local.
 func (l EndpointZones) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
@@ -442,28 +458,6 @@ func NewEndpoints(args ...string) (endpoints Endpoints, err error) {
 	return endpoints, nil
 }
 
-func checkEndpointsSubOptimal(ctx *cli.Context, setupType SetupType, endpointZones EndpointZones) (err error) {
-	// Validate sub optimal ordering only for distributed setup.
-	if setupType != DistXLSetupType {
-		return nil
-	}
-	var endpointOrder int
-	err = fmt.Errorf("Too many disk args are local, input is in sub-optimal order. Please review input args: %s", ctx.Args())
-	for _, endpoints := range endpointZones {
-		for _, endpoint := range endpoints.Endpoints {
-			if endpoint.IsLocal {
-				endpointOrder++
-			} else {
-				endpointOrder--
-			}
-			if endpointOrder >= 2 {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
 // Checks if there are any cross device mounts.
 func checkCrossDeviceMounts(endpoints Endpoints) (err error) {
 	var absPaths []string
@@ -517,9 +511,8 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		return endpoints, setupType, nil
 	}
 
-	for i, iargs := range args {
+	for _, iargs := range args {
 		// Convert args to endpoints
-		var newEndpoints Endpoints
 		eps, err := NewEndpoints(iargs...)
 		if err != nil {
 			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
@@ -530,11 +523,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
-		for _, ep := range eps {
-			ep.SetIndex = i
-			newEndpoints = append(newEndpoints, ep)
-		}
-		endpoints = append(endpoints, newEndpoints...)
+		endpoints = append(endpoints, eps...)
 	}
 
 	if len(endpoints) == 0 {
@@ -608,13 +597,14 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 
 	// All endpoints are pointing to local host
 	if len(endpoints) == localEndpointCount {
-		// If all endpoints have same port number, then this is XL setup using URL style endpoints.
+		// If all endpoints have same port number, Just treat it as distXL setup
+		// using URL style endpoints.
 		if len(localPortSet) == 1 {
 			if len(localServerHostSet) > 1 {
 				return endpoints, setupType,
 					config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
 			}
-			return endpoints, XLSetupType, nil
+			return endpoints, DistXLSetupType, nil
 		}
 
 		// Even though all endpoints are local, but those endpoints use different ports.

cmd/endpoint_test.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"flag"
 	"fmt"
 	"net"
 	"net/url"
@@ -25,47 +24,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
 )
 
-func TestSubOptimalEndpointInput(t *testing.T) {
-	args1 := []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}
-	args2 := []string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}
-
-	tests := []struct {
-		setupType SetupType
-		ctx       *cli.Context
-		endpoints EndpointZones
-		isErr     bool
-	}{
-		{
-			setupType: DistXLSetupType,
-			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
-			endpoints: mustGetZoneEndpoints(args1...),
-			isErr:     false,
-		},
-		{
-			setupType: DistXLSetupType,
-			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
-			endpoints: mustGetZoneEndpoints(args2...),
-			isErr:     false,
-		},
-	}
-	for i, test := range tests {
-		test := test
-		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
-			err := checkEndpointsSubOptimal(test.ctx, test.setupType, test.endpoints)
-			if test.isErr && err == nil {
-				t.Error("expected err but found nil")
-			}
-			if !test.isErr && err != nil {
-				t.Errorf("expected err nil but found an err %s", err)
-			}
-		})
-	}
-}
-
 func TestNewEndpoint(t *testing.T) {
 	u2, _ := url.Parse("https://example.org/path")
 	u4, _ := url.Parse("http://192.168.253.200/path")
@@ -285,7 +246,7 @@ func TestCreateEndpoints(t *testing.T) {
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d2"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d3"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost", Path: "/d4"}, IsLocal: true},
-		}, XLSetupType, nil},
+		}, DistXLSetupType, nil},
 		// DistXL Setup with URLEndpointType having mixed naming to local host.
 		{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
 

cmd/erasure-encode_test.go

@@ -44,6 +44,10 @@ func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) e
 	return errFaultyDisk
 }
 
+func (badDisk) Hostname() string {
+	return ""
+}
+
 const oneMiByte = 1 * humanize.MiByte
 
 var erasureEncodeTests = []struct {

cmd/format-disk-cache.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -29,6 +28,7 @@ import (
 	"reflect"
 	"strings"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/sio"
 )
@@ -444,6 +444,7 @@ func migrateOldCache(ctx context.Context, c *diskCache) error {
 			}
 			// marshal cache metadata after adding version and stat info
 			meta := &cacheMeta{}
+			var json = jsoniter.ConfigCompatibleWithStandardLibrary
 			if err = json.Unmarshal(metaBytes, &meta); err != nil {
 				return err
 			}

cmd/format-xl.go

@@ -19,15 +19,17 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"reflect"
-
-	"encoding/hex"
+	"sync"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	sha256 "github.com/minio/sha256-simd"
 )
@@ -619,23 +621,28 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
 	return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.XL.Sets)
 }
 
-// Initializes meta volume on all input storage disks.
-func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error {
-	// This happens for the first time, but keep this here since this
-	// is the only place where it can be made expensive optimizing all
-	// other calls. Create minio meta volume, if it doesn't exist yet.
+// Initializes meta volume only on local storage disks.
+func initXLMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*formatXLV3) error {
+
+	// Compute the local disks eligible for meta volumes (re)initialization
+	var disksToInit []StorageAPI
+	for index := range storageDisks {
+		if formats[index] == nil || storageDisks[index] == nil || storageDisks[index].Hostname() != "" {
+			// Ignore create meta volume on disks which are not found or not local.
+			continue
+		}
+		disksToInit = append(disksToInit, storageDisks[index])
+	}
 
 	// Initialize errs to collect errors inside go-routine.
-	g := errgroup.WithNErrs(len(storageDisks))
+	g := errgroup.WithNErrs(len(disksToInit))
 
 	// Initialize all disks in parallel.
-	for index := range storageDisks {
+	for index := range disksToInit {
+		// Initialize a new index variable in each loop so each
+		// goroutine will return its own instance of index variable.
 		index := index
 		g.Go(func() error {
-			if formats[index] == nil || storageDisks[index] == nil {
-				// Ignore create meta volume on disks which are not found.
-				return nil
-			}
 			return makeFormatXLMetaVolumes(storageDisks[index])
 		}, index)
 	}
@@ -753,16 +760,42 @@ func fixFormatXLV3(storageDisks []StorageAPI, endpoints Endpoints, formats []*fo
 func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, drivesPerSet int, deploymentID string) (*formatXLV3, error) {
 	format := newFormatXLV3(setCount, drivesPerSet)
 	formats := make([]*formatXLV3, len(storageDisks))
+	wantAtMost := ecDrivesNoConfig(drivesPerSet)
 
 	for i := 0; i < setCount; i++ {
+		hostCount := make(map[string]int, drivesPerSet)
 		for j := 0; j < drivesPerSet; j++ {
+			disk := storageDisks[i*drivesPerSet+j]
 			newFormat := format.Clone()
 			newFormat.XL.This = format.XL.Sets[i][j]
 			if deploymentID != "" {
 				newFormat.ID = deploymentID
 			}
+			hostCount[disk.Hostname()]++
 			formats[i*drivesPerSet+j] = newFormat
 		}
+		if len(hostCount) > 0 {
+			var once sync.Once
+			for host, count := range hostCount {
+				if count > wantAtMost {
+					if host == "" {
+						host = "local"
+					}
+					once.Do(func() {
+						if len(hostCount) == 1 {
+							return
+						}
+						logger.Info(" * Set %v:", i+1)
+						for j := 0; j < drivesPerSet; j++ {
+							disk := storageDisks[i*drivesPerSet+j]
+							logger.Info("   - Drive: %s", disk.String())
+						}
+					})
+					logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+
+						"A host failure will result in data becoming unavailable.", host, wantAtMost)
+				}
+			}
+		}
 	}
 
 	// Save formats `format.json` across all disks.
@@ -773,16 +806,26 @@ func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, driv
 	return getFormatXLInQuorum(formats)
 }
 
+// ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
+// It will attempt to read it from env variable and fall back to drives/2.
+func ecDrivesNoConfig(drivesPerSet int) int {
+	ecDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
+	if ecDrives == 0 {
+		cfg, err := storageclass.LookupConfig(nil, drivesPerSet)
+		if err == nil {
+			ecDrives = cfg.Standard.Parity
+		}
+		if ecDrives == 0 {
+			ecDrives = drivesPerSet / 2
+		}
+	}
+	return ecDrives
+}
+
 // Make XL backend meta volumes.
 func makeFormatXLMetaVolumes(disk StorageAPI) error {
 	// Attempt to create MinIO internal buckets.
-	err := disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
-	if err != nil {
-		if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
-			return err
-		}
-	}
-	return nil
+	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
 }
 
 var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

cmd/format-xl_test.go

@@ -101,7 +101,7 @@ func TestFixFormatV3(t *testing.T) {
 		formats[j] = newFormat
 	}
 
-	if err = initFormatXLMetaVolume(storageDisks, formats); err != nil {
+	if err = initXLMetaVolumesInLocalDisks(storageDisks, formats); err != nil {
 		t.Fatal(err)
 	}
 

cmd/fs-v1-metadata.go

@@ -90,6 +90,7 @@ func (c *FSChecksumInfoV1) UnmarshalJSON(data []byte) error {
 	}
 
 	var info checksuminfo
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	err := json.Unmarshal(data, &info)
 	if err != nil {
 		return err

cmd/fs-v1-multipart.go

@@ -683,7 +683,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Deny if WORM is enabled
-	if _, ok := isWORMEnabled(bucket); ok {
+	if isWORMEnabled(bucket) {
 		if _, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)); err == nil {
 			return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
 		}

cmd/fs-v1.go

@@ -1032,7 +1032,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
 	// Deny if WORM is enabled
-	if _, ok := isWORMEnabled(bucket); ok {
+	if isWORMEnabled(bucket) {
 		if _, err := fsStatFile(ctx, fsNSObjPath); err == nil {
 			return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
 		}

cmd/healthcheck-handler.go

@@ -26,11 +26,11 @@ import (
 
 // ReadinessCheckHandler -- Checks if the quorum number of disks are available.
 // For FS - Checks if the backend disk is available
-// For Zones - Checks if all the zones have enough quorum
+// For Zones - Checks if all the zones have enough read quorum
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ReadinessCheckHandler")
 
-	objLayer := newObjectLayerFn()
+	objLayer := newObjectLayerWithoutSafeModeFn()
 	// Service not initialized yet
 	if objLayer == nil || !objLayer.IsReady(ctx) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
@@ -47,7 +47,7 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "LivenessCheckHandler")
 
-	objLayer := newObjectLayerFn()
+	objLayer := newObjectLayerWithoutSafeModeFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		// Respond with 200 OK while server initializes to ensure a distributed cluster
@@ -95,5 +95,6 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }

cmd/http/headers.go

@@ -40,6 +40,12 @@ const (
 	Action             = "Action"
 )
 
+// Non standard S3 HTTP response constants
+const (
+	XCache       = "X-Cache"
+	XCacheLookup = "X-Cache-Lookup"
+)
+
 // Standard S3 HTTP request constants
 const (
 	IfModifiedSince   = "If-Modified-Since"

cmd/iam.go

@@ -1288,14 +1288,14 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
 		return combinedPolicy.IsAllowed(args)
 	}
 
-	pname, ok := args.Claims[iamPolicyName()]
+	pnameSlice, ok := args.GetPolicies(iamPolicyClaimName())
 	if !ok {
-		// When claims are set, it should have a "policy" field.
+		// When claims are set, it should have a policy claim field.
 		return false
 	}
-	pnameStr, ok := pname.(string)
-	if !ok {
-		// When claims has "policy" field, it should be string.
+
+	// When claims are set, it should have a policy claim field.
+	if len(pnameSlice) == 0 {
 		return false
 	}
 
@@ -1310,7 +1310,7 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
 	}
 	name := mp.Policy
 
-	if pnameStr != name {
+	if pnameSlice[0] != name {
 		// When claims has a policy, it should match the
 		// policy of args.AccountName which server remembers.
 		// if not reject such requests.
@@ -1319,36 +1319,36 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
 
 	// Now check if we have a sessionPolicy.
 	spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
-	if !ok {
-		// Sub policy not set, this is most common since subPolicy
-		// is optional, use the top level policy only.
-		p, ok := sys.iamPolicyDocsMap[pnameStr]
-		return ok && p.IsAllowed(args)
+	if ok {
+		spolicyStr, ok := spolicy.(string)
+		if !ok {
+			// Sub policy if set, should be a string reject
+			// malformed/malicious requests.
+			return false
+		}
+
+		// Check if policy is parseable.
+		subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+		if err != nil {
+			// Log any error in input session policy config.
+			logger.LogIf(context.Background(), err)
+			return false
+		}
+
+		// Policy without Version string value reject it.
+		if subPolicy.Version == "" {
+			return false
+		}
+
+		// Sub policy is set and valid.
+		p, ok := sys.iamPolicyDocsMap[pnameSlice[0]]
+		return ok && p.IsAllowed(args) && subPolicy.IsAllowed(args)
 	}
 
-	spolicyStr, ok := spolicy.(string)
-	if !ok {
-		// Sub policy if set, should be a string reject
-		// malformed/malicious requests.
-		return false
-	}
-
-	// Check if policy is parseable.
-	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
-	if err != nil {
-		// Log any error in input session policy config.
-		logger.LogIf(context.Background(), err)
-		return false
-	}
-
-	// Policy without Version string value reject it.
-	if subPolicy.Version == "" {
-		return false
-	}
-
-	// Sub policy is set and valid.
-	p, ok := sys.iamPolicyDocsMap[pnameStr]
-	return ok && p.IsAllowed(args) && subPolicy.IsAllowed(args)
+	// Sub policy not set, this is most common since subPolicy
+	// is optional, use the top level policy only.
+	p, ok := sys.iamPolicyDocsMap[pnameSlice[0]]
+	return ok && p.IsAllowed(args)
 }
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.

cmd/local-locker.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -39,40 +38,6 @@ func isWriteLock(lri []lockRequesterInfo) bool {
 	return len(lri) == 1 && lri[0].Writer
 }
 
-type errorLocker struct{}
-
-func (d *errorLocker) String() string {
-	return ""
-}
-
-func (d *errorLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
-	return false, errors.New("unable to lock")
-}
-
-func (d *errorLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
-	return false, errors.New("unable to unlock")
-}
-
-func (d *errorLocker) RLock(args dsync.LockArgs) (reply bool, err error) {
-	return false, errors.New("unable to rlock")
-}
-
-func (d *errorLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
-	return false, errors.New("unable to runlock")
-}
-
-func (d *errorLocker) Close() error {
-	return nil
-}
-
-func (d *errorLocker) IsOnline() bool {
-	return false
-}
-
-func (d *errorLocker) Expired(args dsync.LockArgs) (reply bool, err error) {
-	return false, errors.New("unable to check for lock expiration")
-}
-
 // localLocker implements Dsync.NetLocker
 type localLocker struct {
 	mutex    sync.Mutex

cmd/lock-rest-client.go

@@ -76,8 +76,8 @@ func (client *lockRESTClient) call(method string, values url.Values, body io.Rea
 	}
 
 	if isNetworkError(err) {
-		time.AfterFunc(defaultRetryUnit*3, func() {
-			// After 3 seconds, take this lock client online for a retry.
+		time.AfterFunc(defaultRetryUnit, func() {
+			// After 1 seconds, take this lock client online for a retry.
 			atomic.StoreInt32(&client.connected, 1)
 		})
 

cmd/lock-rest-server.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"math/rand"
 	"net/http"
@@ -29,7 +30,7 @@ import (
 
 const (
 	// Lock maintenance interval.
-	lockMaintenanceInterval = 30 * time.Second
+	lockMaintenanceInterval = 1 * time.Minute
 
 	// Lock validity check interval.
 	lockValidityCheckInterval = 2 * time.Minute
@@ -179,6 +180,8 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 	return nlripMap
 }
 
+var lockMaintenanceTimeout = newDynamicTimeout(60*time.Second, time.Second)
+
 // lockMaintenance loops over locks that have been active for some time and checks back
 // with the original server whether it is still alive or not
 //
@@ -187,7 +190,14 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 // - some network error (and server is up normally)
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
-func lockMaintenance(interval time.Duration) {
+func lockMaintenance(ctx context.Context, interval time.Duration, objAPI ObjectLayer) error {
+	// Lock to avoid concurrent lock maintenance loops
+	maintenanceLock := objAPI.NewNSLock(ctx, "system", "lock-maintenance-ops")
+	if err := maintenanceLock.GetLock(lockMaintenanceTimeout); err != nil {
+		return err
+	}
+	defer maintenanceLock.Unlock()
+
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
 	for lendpoint, nlrips := range getLongLivedLocks(interval) {
@@ -203,7 +213,8 @@ func lockMaintenance(interval time.Duration) {
 						continue
 					}
 
-					// Call back to original server verify whether the lock is still active (based on name & uid)
+					// Call back to original server verify whether the lock is
+					// still active (based on name & uid)
 					expired, err := c.Expired(dsync.LockArgs{
 						UID:      nlrip.lri.UID,
 						Resource: nlrip.name,
@@ -230,15 +241,31 @@ func lockMaintenance(interval time.Duration) {
 			}
 		}
 	}
+
+	return nil
 }
 
 // Start lock maintenance from all lock servers.
 func startLockMaintenance() {
+	var objAPI ObjectLayer
+	var ctx = context.Background()
+
+	// Wait until the object API is ready
+	for {
+		objAPI = newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
 	// Initialize a new ticker with a minute between each ticks.
 	ticker := time.NewTicker(lockMaintenanceInterval)
 	// Stop the timer upon service closure and cleanup the go-routine.
 	defer ticker.Stop()
 
+	r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 	for {
 		// Verifies every minute for locks held more than 2 minutes.
 		select {
@@ -247,10 +274,13 @@ func startLockMaintenance() {
 		case <-ticker.C:
 			// Start with random sleep time, so as to avoid
 			// "synchronous checks" between servers
-			r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 			duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
 			time.Sleep(duration)
-			lockMaintenance(lockValidityCheckInterval)
+			if err := lockMaintenance(ctx, lockValidityCheckInterval, objAPI); err != nil {
+				// Sleep right after an error.
+				duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
+				time.Sleep(duration)
+			}
 		}
 	}
 }

cmd/logger/console.go

@@ -23,9 +23,9 @@ import (
 	"strings"
 	"time"
 
-	c "github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/color"
+	c "github.com/minio/minio/pkg/console"
 )
 
 // Logger interface describes the methods that need to be implemented to satisfy the interface requirements.
@@ -119,16 +119,16 @@ func (f fatalMsg) pretty(msg string, args ...interface{}) {
 			ansiSaveAttributes()
 			// Print banner with or without the log tag
 			if !tagPrinted {
-				fmt.Print(logBanner)
+				c.Print(logBanner)
 				tagPrinted = true
 			} else {
-				fmt.Print(emptyBanner)
+				c.Print(emptyBanner)
 			}
 			// Restore the text color of the error message
 			ansiRestoreAttributes()
 			ansiMoveRight(bannerWidth)
 			// Continue  error message printing
-			fmt.Println(line)
+			c.Println(line)
 			break
 		}
 	}

cmd/logger/target/console/console.go

@@ -25,6 +25,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/console"
 )
 
 // Target implements loggerTarget to send log
@@ -109,7 +110,7 @@ func (c *Target) Send(e interface{}, logKind string) error {
 		apiString, timeString, deploymentID, requestID, remoteHost, host, userAgent,
 		msg, tagString, strings.Join(trace, "\n"))
 
-	fmt.Println(output)
+	console.Println(output)
 	return nil
 }
 

cmd/logger/utils.go

@@ -19,6 +19,7 @@ package logger
 import (
 	"fmt"
 	"regexp"
+	"runtime"
 
 	"github.com/minio/minio/pkg/color"
 )
@@ -32,18 +33,27 @@ func ansiEscape(format string, args ...interface{}) {
 }
 
 func ansiMoveRight(n int) {
+	if runtime.GOOS == "windows" {
+		return
+	}
 	if color.IsTerminal() {
 		ansiEscape("[%dC", n)
 	}
 }
 
 func ansiSaveAttributes() {
+	if runtime.GOOS == "windows" {
+		return
+	}
 	if color.IsTerminal() {
 		ansiEscape("7")
 	}
 }
 
 func ansiRestoreAttributes() {
+	if runtime.GOOS == "windows" {
+		return
+	}
 	if color.IsTerminal() {
 		ansiEscape("8")
 	}

cmd/main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ import (
 	"sort"
 
 	"github.com/minio/cli"
-	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/trie"
 	"github.com/minio/minio/pkg/words"
 )
@@ -74,8 +74,8 @@ FLAGS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
 VERSION:
-  ` + Version +
-	`{{ "\n"}}`
+{{.Version}}
+`
 
 func newApp(name string) *cli.App {
 	// Collection of minio commands currently supported are.
@@ -126,7 +126,7 @@ func newApp(name string) *cli.App {
 	app := cli.NewApp()
 	app.Name = name
 	app.Author = "MinIO, Inc."
-	app.Version = Version
+	app.Version = ReleaseTag
 	app.Usage = "High Performance Object Storage"
 	app.Description = `Build high performance data infrastructure for machine learning, analytics and application data workloads with MinIO`
 	app.Flags = GlobalFlags

cmd/naughty-disk_test.go

@@ -53,8 +53,8 @@ func (d *naughtyDisk) IsOnline() bool {
 	return d.disk.IsOnline()
 }
 
-func (d *naughtyDisk) LastError() (err error) {
-	return nil
+func (*naughtyDisk) Hostname() string {
+	return ""
 }
 
 func (d *naughtyDisk) Close() (err error) {

cmd/notification.go

@@ -319,36 +319,39 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 
 		profilingDataFound = true
 
-		// Send profiling data to zip as file
-		header, zerr := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profiling-%s.pprof", client.host.String()),
-			size:    int64(len(data)),
-			mode:    0600,
-			modTime: UTCNow(),
-			isDir:   false,
-			sys:     nil,
-		})
-		if zerr != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-			ctx := logger.SetReqInfo(ctx, reqInfo)
-			logger.LogIf(ctx, zerr)
-			continue
-		}
-		zwriter, zerr := zipWriter.CreateHeader(header)
-		if zerr != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-			ctx := logger.SetReqInfo(ctx, reqInfo)
-			logger.LogIf(ctx, zerr)
-			continue
-		}
-		if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-			ctx := logger.SetReqInfo(ctx, reqInfo)
-			logger.LogIf(ctx, err)
-			continue
+		for typ, data := range data {
+			// Send profiling data to zip as file
+			header, zerr := zip.FileInfoHeader(dummyFileInfo{
+				name:    fmt.Sprintf("profiling-%s-%s.pprof", client.host.String(), typ),
+				size:    int64(len(data)),
+				mode:    0600,
+				modTime: UTCNow(),
+				isDir:   false,
+				sys:     nil,
+			})
+			if zerr != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
+				ctx := logger.SetReqInfo(ctx, reqInfo)
+				logger.LogIf(ctx, zerr)
+				continue
+			}
+			zwriter, zerr := zipWriter.CreateHeader(header)
+			if zerr != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
+				ctx := logger.SetReqInfo(ctx, reqInfo)
+				logger.LogIf(ctx, zerr)
+				continue
+			}
+			if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
+				ctx := logger.SetReqInfo(ctx, reqInfo)
+				logger.LogIf(ctx, err)
+				continue
+			}
 		}
 	}
 
+	// Local host
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -366,25 +369,27 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 	profilingDataFound = true
 
 	// Send profiling data to zip as file
-	header, zerr := zip.FileInfoHeader(dummyFileInfo{
-		name:    fmt.Sprintf("profiling-%s.pprof", thisAddr),
-		size:    int64(len(data)),
-		mode:    0600,
-		modTime: UTCNow(),
-		isDir:   false,
-		sys:     nil,
-	})
-	if zerr != nil {
-		return profilingDataFound
-	}
+	for typ, data := range data {
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    fmt.Sprintf("profiling-%s-%s.pprof", thisAddr, typ),
+			size:    int64(len(data)),
+			mode:    0600,
+			modTime: UTCNow(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			return profilingDataFound
+		}
 
-	zwriter, zerr := zipWriter.CreateHeader(header)
-	if zerr != nil {
-		return profilingDataFound
-	}
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			return profilingDataFound
+		}
 
-	if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
-		return profilingDataFound
+		if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
+			return profilingDataFound
+		}
 	}
 
 	return profilingDataFound

cmd/object-api-datatypes.go

@@ -143,6 +143,11 @@ type ObjectInfo struct {
 	// Date and time at which the object is no longer able to be cached
 	Expires time.Time
 
+	// CacheStatus sets status of whether this is a cache hit/miss
+	CacheStatus CacheStatusType
+	// CacheLookupStatus sets whether a cacheable response is present in the cache
+	CacheLookupStatus CacheStatusType
+
 	// Specify object storage class
 	StorageClass string
 

cmd/object-api-errors.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"path"
@@ -429,14 +430,14 @@ func (e BackendDown) Error() string {
 
 // isErrBucketNotFound - Check if error type is BucketNotFound.
 func isErrBucketNotFound(err error) bool {
-	_, ok := err.(BucketNotFound)
-	return ok
+	var bkNotFound BucketNotFound
+	return errors.As(err, &bkNotFound)
 }
 
 // isErrObjectNotFound - Check if error type is ObjectNotFound.
 func isErrObjectNotFound(err error) bool {
-	_, ok := err.(ObjectNotFound)
-	return ok
+	var objNotFound ObjectNotFound
+	return errors.As(err, &objNotFound)
 }
 
 // PreConditionFailed - Check if copy precondition failed

cmd/object-api-multipart_test.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -160,108 +159,6 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
 	}
 }
 
-// Wrapper for calling TestPutObjectPartDiskNotFound tests for both XL
-// write quorum.
-func TestPutObjectPartDiskNotFound(t *testing.T) {
-	ExecObjectLayerDiskAlteredTest(t, testPutObjectPartDiskNotFound)
-}
-
-// testPutObjectPartDiskNotFound - Tests validate PutObjectPart behavior when disks go offline.
-func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
-	bucketNames := []string{"minio-bucket", "minio-2-bucket"}
-	objectNames := []string{"minio-object-1.txt"}
-	uploadIDs := []string{}
-
-	// bucketnames[0].
-	// objectNames[0].
-	// uploadIds [0].
-	// Create bucket before intiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], "")
-	if err != nil {
-		// Failed to create newbucket, abort.
-		t.Fatalf("%s : %s", instanceType, err.Error())
-	}
-
-	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], ObjectOptions{})
-	if err != nil {
-		// Failed to create NewMultipartUpload, abort.
-		t.Fatalf("%s : %s", instanceType, err.Error())
-	}
-
-	// Remove some random disk.
-	for _, disk := range disks[:6] {
-		os.RemoveAll(disk)
-	}
-
-	uploadIDs = append(uploadIDs, uploadID)
-
-	// Create multipart parts.
-	// Need parts to be uploaded before MultipartLists can be called and tested.
-	createPartCases := []struct {
-		bucketName      string
-		objName         string
-		uploadID        string
-		PartID          int
-		inputReaderData string
-		inputMd5        string
-		intputDataSize  int64
-		expectedMd5     string
-	}{
-		// Case 1-5.
-		// Creating sequence of parts for same uploadID.
-		// Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID.
-		{bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"},
-		{bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"},
-		{bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("ijkl")), "09a0877d04abf8759f99adec02baf579"},
-		{bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"},
-		{bucketNames[0], objectNames[0], uploadIDs[0], 5, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("mnop")), "e132e96a5ddad6da8b07bba6f6131fef"},
-	}
-	sha256sum := ""
-	// Iterating over creatPartCases to generate multipart chunks.
-	for _, testCase := range createPartCases {
-		_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
-		if err != nil {
-			t.Fatalf("%s : %s", instanceType, err.Error())
-		}
-	}
-
-	// This causes quorum failure verify.
-	for _, disk := range disks[len(disks)-3:] {
-		os.RemoveAll(disk)
-	}
-
-	// Object part upload should fail with quorum not available.
-	testCase := createPartCases[len(createPartCases)-1]
-	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
-	if err == nil {
-		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
-	}
-
-	// as majority of xl.json are not available, we expect uploadID to be not available.
-	expectedErr1 := BucketNotFound{Bucket: testCase.bucketName}
-	if err.Error() != expectedErr1.Error() {
-		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr1, err)
-	}
-
-	// This causes invalid upload id.
-	for _, disk := range disks {
-		os.RemoveAll(disk)
-	}
-
-	// Object part upload should fail with bucket not found.
-	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
-	if err == nil {
-		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
-	}
-
-	// As all disks at not available, bucket not found.
-	expectedErr2 := BucketNotFound{Bucket: testCase.bucketName}
-	if err.Error() != expectedErr2.Error() {
-		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr2, err)
-	}
-}
-
 // Wrapper for calling PutObjectPart tests for both XL multiple disks and single node setup.
 func TestObjectAPIPutObjectPart(t *testing.T) {
 	ExecObjectLayerTest(t, testObjectAPIPutObjectPart)

cmd/object-handlers.go

@@ -2563,7 +2563,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if _, isWORMBucket := isWORMEnabled(bucket); !isWORMBucket {
+	if _, ok := globalBucketObjectLockConfig.Get(bucket); !ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/object-lock.go

@@ -477,8 +477,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 	ret := getObjectRetentionMeta(oi.UserDefined)
 	// no retention metadata on object
 	if ret.Mode == Invalid {
-		_, isWORMBucket := isWORMEnabled(bucket)
-		if !isWORMBucket {
+		if _, isWORMBucket := globalBucketObjectLockConfig.Get(bucket); !isWORMBucket {
 			return oi, ErrInvalidBucketObjectLockConfiguration
 		}
 		return oi, ErrNone
@@ -527,7 +526,7 @@ func checkPutObjectRetentionAllowed(ctx context.Context, r *http.Request, bucket
 	var mode RetentionMode
 	var retainDate RetentionDate
 
-	retention, isWORMBucket := isWORMEnabled(bucket)
+	retention, isWORMBucket := globalBucketObjectLockConfig.Get(bucket)
 
 	retentionRequested := isObjectLockRequested(r.Header)
 
@@ -595,16 +594,32 @@ func checkPutObjectRetentionAllowed(ctx context.Context, r *http.Request, bucket
 
 // filter object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set.
 func filterObjectLockMetadata(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string, isCopy bool, getRetPerms APIErrorCode) map[string]string {
+	// Copy on write
+	dst := metadata
+	var copied bool
+	delKey := func(key string) {
+		if _, ok := metadata[key]; !ok {
+			return
+		}
+		if !copied {
+			dst = make(map[string]string, len(metadata))
+			for k, v := range metadata {
+				dst[k] = v
+			}
+			copied = true
+		}
+		delete(dst, key)
+	}
 	ret := getObjectRetentionMeta(metadata)
 	if ret.Mode == Invalid || isCopy {
-		delete(metadata, xhttp.AmzObjectLockMode)
-		delete(metadata, xhttp.AmzObjectLockRetainUntilDate)
+		delKey(xhttp.AmzObjectLockMode)
+		delKey(xhttp.AmzObjectLockRetainUntilDate)
 		return metadata
 	}
 	if getRetPerms == ErrNone {
-		return metadata
+		return dst
 	}
-	delete(metadata, xhttp.AmzObjectLockMode)
-	delete(metadata, xhttp.AmzObjectLockRetainUntilDate)
-	return metadata
+	delKey(xhttp.AmzObjectLockMode)
+	delKey(xhttp.AmzObjectLockRetainUntilDate)
+	return dst
 }

cmd/peer-rest-client.go

@@ -225,7 +225,7 @@ func (client *peerRESTClient) StartProfiling(profiler string) error {
 }
 
 // DownloadProfileData - download profiled data from a remote node.
-func (client *peerRESTClient) DownloadProfileData() (data []byte, err error) {
+func (client *peerRESTClient) DownloadProfileData() (data map[string][]byte, err error) {
 	respBody, err := client.call(peerRESTMethodDownloadProfilingData, nil, nil, -1)
 	if err != nil {
 		return

cmd/peer-rest-server.go

@@ -395,28 +395,41 @@ func (s *peerRESTServer) StartProfilingHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	vars := mux.Vars(r)
-	profiler := vars[peerRESTProfiler]
-	if profiler == "" {
+	profiles := strings.Split(vars[peerRESTProfiler], ",")
+	if len(profiles) == 0 {
 		s.writeErrorResponse(w, errors.New("profiler name is missing"))
 		return
 	}
-
-	if globalProfiler != nil {
-		globalProfiler.Stop()
+	globalProfilerMu.Lock()
+	defer globalProfilerMu.Unlock()
+	if globalProfiler == nil {
+		globalProfiler = make(map[string]minioProfiler, 10)
 	}
 
-	var err error
-	globalProfiler, err = startProfiler(profiler, "")
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
+	// Stop profiler of all types if already running
+	for k, v := range globalProfiler {
+		for _, p := range profiles {
+			if p == k {
+				v.Stop()
+				delete(globalProfiler, k)
+			}
+		}
+	}
+
+	for _, profiler := range profiles {
+		prof, err := startProfiler(profiler)
+		if err != nil {
+			s.writeErrorResponse(w, err)
+			return
+		}
+		globalProfiler[profiler] = prof
 	}
 
 	w.(http.Flusher).Flush()
 }
 
-// DownloadProflingDataHandler - returns proflied data.
-func (s *peerRESTServer) DownloadProflingDataHandler(w http.ResponseWriter, r *http.Request) {
+// DownloadProfilingDataHandler - returns profiled data.
+func (s *peerRESTServer) DownloadProfilingDataHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
 		return
@@ -563,7 +576,7 @@ func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Requ
 	vars := mux.Vars(r)
 	dryRunString := vars[peerRESTDryRun]
 	if dryRunString == "" {
-		s.writeErrorResponse(w, errors.New("dry run parameter is missing"))
+		s.writeErrorResponse(w, errors.New("dry-run parameter is missing"))
 		return
 	}
 
@@ -583,6 +596,7 @@ func (s *peerRESTServer) ReloadFormatHandler(w http.ResponseWriter, r *http.Requ
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
 	}
+
 	err := objAPI.ReloadFormat(context.Background(), dryRun)
 	if err != nil {
 		s.writeErrorResponse(w, err)
@@ -1155,7 +1169,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadGroup).HandlerFunc(httpTraceAll(server.LoadGroupHandler)).Queries(restQueries(peerRESTGroup)...)
 
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProflingDataHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProfilingDataHandler))
 
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodTargetExists).HandlerFunc(httpTraceHdrs(server.TargetExistsHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSendEvent).HandlerFunc(httpTraceHdrs(server.SendEventHandler)).Queries(restQueries(peerRESTBucket)...)

cmd/policy.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"sync"
 
+	jsoniter "github.com/json-iterator/go"
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
@@ -275,6 +276,7 @@ func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.Buc
 	}
 
 	var policyInfo miniogopolicy.BucketAccessPolicy
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(data, &policyInfo); err != nil {
 		// This should not happen because data is valid to JSON data.
 		return nil, err
@@ -292,6 +294,7 @@ func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*
 	}
 
 	var bucketPolicy policy.Policy
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(data, &bucketPolicy); err != nil {
 		// This should not happen because data is valid to JSON data.
 		return nil, err

cmd/posix-diskid-check.go

@@ -42,8 +42,8 @@ func (p *posixDiskIDCheck) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsag
 	return p.storage.CrawlAndGetDataUsage(endCh)
 }
 
-func (p *posixDiskIDCheck) LastError() error {
-	return p.storage.LastError()
+func (p *posixDiskIDCheck) Hostname() string {
+	return p.storage.Hostname()
 }
 
 func (p *posixDiskIDCheck) Close() error {

cmd/posix.go

@@ -309,11 +309,8 @@ func (s *posix) String() string {
 	return s.diskPath
 }
 
-func (s *posix) LastError() error {
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-	return nil
+func (*posix) Hostname() string {
+	return ""
 }
 
 func (s *posix) Close() error {
@@ -554,7 +551,9 @@ func (s *posix) SetDiskID(id string) {
 func (s *posix) MakeVolBulk(volumes ...string) (err error) {
 	for _, volume := range volumes {
 		if err = s.MakeVol(volume); err != nil {
-			return err
+			if err != errVolumeExists {
+				return err
+			}
 		}
 	}
 	return nil

cmd/prepare-storage.go

@@ -18,12 +18,18 @@ package cmd
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
+	"path"
 	"sync"
 	"time"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -160,6 +166,7 @@ func validateXLFormats(format *formatXLV3, formats []*formatXLV3, endpoints Endp
 	if len(format.XL.Sets[0]) != drivesPerSet {
 		return fmt.Errorf("Current backend format is inconsistent with input args (%s), Expected drive count per set %d, got %d", endpoints, len(format.XL.Sets[0]), drivesPerSet)
 	}
+
 	return nil
 }
 
@@ -171,6 +178,45 @@ func validateXLFormats(format *formatXLV3, formats []*formatXLV3, endpoints Endp
 // https://github.com/minio/minio/issues/5667
 var errXLV3ThisEmpty = fmt.Errorf("XL format version 3 has This field empty")
 
+// IsServerResolvable - checks if the endpoint is resolvable
+// by sending a naked HTTP request with liveness checks.
+func IsServerResolvable(endpoint Endpoint) error {
+	serverURL := &url.URL{
+		Scheme: endpoint.Scheme,
+		Host:   endpoint.Host,
+		Path:   path.Join(healthCheckPathPrefix, healthCheckLivenessPath),
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: endpoint.Hostname(),
+			RootCAs:    globalRootCAs,
+			NextProtos: []string{"http/1.1"}, // Force http1.1
+		}
+	}
+
+	req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
+	if err != nil {
+		return err
+	}
+
+	httpClient := &http.Client{
+		Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)(),
+	}
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer xhttp.DrainBody(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		return StorageErr(resp.Status)
+	}
+	return nil
+}
+
 // connect to list of endpoints and load all XL disk formats, validate the formats are correct
 // and are in quorum, if no formats are found attempt to initialize all of them for the first
 // time. additionally make sure to close all the disks used in this attempt.
@@ -178,9 +224,15 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 	// Initialize all storage disks
 	storageDisks, errs := initStorageDisksWithErrors(endpoints)
 	defer closeStorageDisks(storageDisks)
+
 	for i, err := range errs {
-		if err != nil && err != errDiskNotFound {
-			return nil, fmt.Errorf("Disk %s: %w", endpoints[i], err)
+		if err != nil {
+			if err != errDiskNotFound {
+				return nil, fmt.Errorf("Disk %s: %w", endpoints[i], err)
+			}
+			if retryCount >= 5 {
+				logger.Info("Unable to connect to %s: %v\n", endpoints[i], IsServerResolvable(endpoints[i]))
+			}
 		}
 	}
 
@@ -193,17 +245,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		}
 	}
 
-	// Connect to all storage disks, a connection failure will be
-	// only logged after some retries.
-	for _, disk := range storageDisks {
-		if disk != nil {
-			connectErr := disk.LastError()
-			if connectErr != nil && retryCount >= 5 {
-				logger.Info("Unable to connect to %s: %v\n", disk.String(), connectErr.Error())
-			}
-		}
-	}
-
 	// Pre-emptively check if one of the formatted disks
 	// is invalid. This function returns success for the
 	// most part unless one of the formats is not consistent
@@ -226,12 +267,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		return format, nil
 	}
 
-	// The first will always recreate some directories inside .minio.sys
-	// such as, tmp, multipart and background-ops
-	if firstDisk {
-		initFormatXLMetaVolume(storageDisks, formatConfigs)
-	}
-
 	// Return error when quorum unformatted disks - indicating we are
 	// waiting for first server to be online.
 	if quorumUnformattedDisks(sErrs) && !firstDisk {
@@ -290,6 +325,10 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		return nil, err
 	}
 
+	// The will always recreate some directories inside .minio.sys of
+	// the local disk such as tmp, multipart and background-ops
+	initXLMetaVolumesInLocalDisks(storageDisks, formatConfigs)
+
 	return format, nil
 }
 
@@ -307,13 +346,6 @@ func waitForFormatXL(firstDisk bool, endpoints Endpoints, setCount, drivesPerSet
 		return nil, err
 	}
 
-	// Done channel is used to close any lingering retry routine, as soon
-	// as this function returns.
-	doneCh := make(chan struct{})
-
-	// Indicate to our retry routine to exit cleanly, upon this function return.
-	defer close(doneCh)
-
 	// prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks.
 	// All times are rounded to avoid showing milli, micro and nano seconds
 	formatStartTime := time.Now().Round(time.Second)
@@ -321,13 +353,16 @@ func waitForFormatXL(firstDisk bool, endpoints Endpoints, setCount, drivesPerSet
 		return time.Now().Round(time.Second).Sub(formatStartTime).String()
 	}
 
-	// Wait on the jitter retry loop.
-	retryTimerCh := newRetryTimerSimple(doneCh)
+	// Wait on each try for an update.
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+	var tries int
 	for {
 		select {
-		case retryCount := <-retryTimerCh:
-			format, err := connectLoadInitFormats(retryCount, firstDisk, endpoints, setCount, drivesPerSet, deploymentID)
+		case <-ticker.C:
+			format, err := connectLoadInitFormats(tries, firstDisk, endpoints, setCount, drivesPerSet, deploymentID)
 			if err != nil {
+				tries++
 				switch err {
 				case errNotFirstDisk:
 					// Fresh setup, wait for first server to be up.

cmd/server-main.go

@@ -45,8 +45,7 @@ func init() {
 	globalConsoleSys = NewConsoleLogger(context.Background())
 	logger.AddTarget(globalConsoleSys)
 
-	gob.Register(VerifyFileError(""))
-	gob.Register(DeleteFileError(""))
+	gob.Register(StorageErr(""))
 }
 
 // ServerFlags - server command specific flags
@@ -127,14 +126,10 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	}
 	logger.FatalIf(err, "Invalid command line arguments")
 
-	if err = checkEndpointsSubOptimal(ctx, setupType, globalEndpoints); err != nil {
-		logger.Info("Optimal endpoint check failed %s", err)
-	}
-
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
-	// To avoid this error sutiation we check for port availability.
+	// To avoid this error situation we check for port availability.
 	logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the server")
 
 	globalIsXL = (setupType == XLSetupType)
@@ -200,9 +195,6 @@ func initSafeMode(buckets []BucketInfo) (err error) {
 		}
 	}(objLock)
 
-	// Calls New() and initializes all sub-systems.
-	newAllSubsystems()
-
 	// Migrate all backend configs to encrypted backend configs, optionally
 	// handles rotating keys for encryption.
 	if err = handleEncryptedConfigBackend(newObject, true); err != nil {
@@ -365,6 +357,16 @@ func serverMain(ctx *cli.Context) {
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()
 
+	// Calls New() and initializes all sub-systems.
+	newAllSubsystems()
+
+	// Enable healing to heal drives if possible
+	if globalIsXL {
+		initBackgroundHealing()
+		initLocalDisksAutoHeal()
+		initGlobalHeal()
+	}
+
 	buckets, err := newObject.ListBuckets(context.Background())
 	if err != nil {
 		logger.Fatal(err, "Unable to list buckets")
@@ -391,12 +393,6 @@ func serverMain(ctx *cli.Context) {
 	initDataUsageStats()
 	initDailyLifecycle()
 
-	if globalIsXL {
-		initBackgroundHealing()
-		initLocalDisksAutoHeal()
-		initGlobalHeal()
-	}
-
 	// Disable safe mode operation, after all initialization is over.
 	globalObjLayerMutex.Lock()
 	globalSafeMode = false

cmd/server-startup-msg.go

@@ -85,7 +85,7 @@ func printStartupSafeModeMessage(apiEndpoints []string, err error) {
 	endPoint := strippedAPIEndpoints[0]
 
 	// Configure 'mc', following block prints platform specific information for minio client admin commands.
-	if color.IsTerminal() {
+	if color.IsTerminal() && !globalCLIContext.Anonymous {
 		logStartupMessage(color.RedBold("\nCommand-line Access: ") + mcAdminQuickStartGuide)
 		if runtime.GOOS == globalWindowsOSName {
 			mcMessage := fmt.Sprintf("> mc.exe config host add %s %s %s %s --api s3v4", alias,
@@ -222,7 +222,7 @@ func printCLIAccessMsg(endPoint string, alias string) {
 	cred := globalActiveCred
 
 	// Configure 'mc', following block prints platform specific information for minio client.
-	if color.IsTerminal() {
+	if color.IsTerminal() && !globalCLIContext.Anonymous {
 		logStartupMessage(color.Blue("\nCommand-line Access: ") + mcQuickStartGuide)
 		if runtime.GOOS == globalWindowsOSName {
 			mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias,

cmd/server_test.go

@@ -353,7 +353,7 @@ func (s *TestSuiteCommon) TestBucketPolicy(c *check) {
 
 	bucketPolicyReadBuf, err := ioutil.ReadAll(response.Body)
 	c.Assert(err, nil)
-	// Verify if downloaded policy matches with previousy uploaded.
+	// Verify if downloaded policy matches with previously uploaded.
 	expectedPolicy, err := policy.ParseConfig(strings.NewReader(bucketPolicyStr), bucketName)
 	c.Assert(err, nil)
 	gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), bucketName)
@@ -521,7 +521,7 @@ func (s *TestSuiteCommon) TestListenBucketNotificationHandler(c *check) {
 	}
 }
 
-// Test deletes multple objects and verifies server resonse.
+// Test deletes multiple objects and verifies server response.
 func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *check) {
 	// generate a random bucket name.
 	bucketName := getRandomBucketName()
@@ -1014,7 +1014,7 @@ func (s *TestSuiteCommon) TestHeader(c *check) {
 func (s *TestSuiteCommon) TestPutBucket(c *check) {
 	// generate a random bucket name.
 	bucketName := getRandomBucketName()
-	// Block 1: Testing for racey access
+	// Block 1: Testing for racy access
 	// The assertion is removed from this block since the purpose of this block is to find races
 	// The purpose this block is not to check for correctness of functionality
 	// Run the test with -race flag to utilize this
@@ -1369,7 +1369,7 @@ func (s *TestSuiteCommon) TestPutObjectLongName(c *check) {
 	c.Assert(response.StatusCode, http.StatusBadRequest)
 	verifyError(c, response, "XMinioInvalidObjectName", "Object name contains a leading slash.", http.StatusBadRequest)
 
-	//make object name as unsuported
+	//make object name as unsupported
 	longObjName = fmt.Sprintf("%0256d", 1)
 	buffer = bytes.NewReader([]byte("hello world"))
 	request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, longObjName),
@@ -1818,7 +1818,7 @@ func (s *TestSuiteCommon) TestPutBucketErrors(c *check) {
 func (s *TestSuiteCommon) TestGetObjectLarge10MiB(c *check) {
 	// generate a random bucket name.
 	bucketName := getRandomBucketName()
-	// form HTTP reqest to create the bucket.
+	// form HTTP request to create the bucket.
 	request, err := newTestSignedRequest("PUT", getMakeBucketURL(s.endPoint, bucketName),
 		0, nil, s.accessKey, s.secretKey, s.signer)
 	c.Assert(err, nil)
@@ -2005,7 +2005,7 @@ func (s *TestSuiteCommon) TestGetPartialObjectMisAligned(c *check) {
 		{"1-", putContent[1:]},
 		// request for object data after the first byte.
 		{"6-", putContent[6:]},
-		// request for last 2 bytes of th object.
+		// request for last 2 bytes of the object.
 		{"-2", putContent[len(putContent)-2:]},
 		// request for last 7 bytes of the object.
 		{"-7", putContent[len(putContent)-7:]},
@@ -2159,7 +2159,7 @@ func (s *TestSuiteCommon) TestGetPartialObjectLarge10MiB(c *check) {
 	request.Header.Add("Range", "bytes=2048-2058")
 
 	client = http.Client{Transport: s.transport}
-	// execute the HTTP request to download the partila content.
+	// execute the HTTP request to download the partial content.
 	response, err = client.Do(request)
 	c.Assert(err, nil)
 	// Since only part of the object is requested, expecting response status to be http.StatusPartialContent .
@@ -2424,11 +2424,11 @@ func (s *TestSuiteCommon) TestBucketMultipartList(c *check) {
 	// The reason to duplicate this structure here is to verify if the
 	// unmarshalling works from a client perspective, specifically
 	// while unmarshalling time.Time type for 'Initiated' field.
-	// time.Time does not honor xml marshaler, it means that we need
+	// time.Time does not honor xml marshaller, it means that we need
 	// to encode/format it before giving it to xml marshaling.
 
 	// This below check adds client side verification to see if its
-	// truly parseable.
+	// truly parsable.
 
 	// listMultipartUploadsResponse - format for list multipart uploads response.
 	type listMultipartUploadsResponse struct {

cmd/setup-type.go

@@ -20,8 +20,11 @@ package cmd
 type SetupType int
 
 const (
+	// UnknownSetupType - starts with unknown setup type.
+	UnknownSetupType SetupType = iota
+
 	// FSSetupType - FS setup type enum.
-	FSSetupType SetupType = iota + 1
+	FSSetupType
 
 	// XLSetupType - XL setup type enum.
 	XLSetupType
@@ -45,5 +48,5 @@ func (setupType SetupType) String() string {
 		return globalMinioModeGatewayPrefix
 	}
 
-	return ""
+	return "unknown"
 }

cmd/signals.go

@@ -28,8 +28,12 @@ func handleSignals() {
 	// Custom exit function
 	exit := func(success bool) {
 		// If global profiler is set stop before we exit.
-		if globalProfiler != nil {
-			globalProfiler.Stop()
+		globalProfilerMu.Lock()
+		defer globalProfilerMu.Unlock()
+		if len(globalProfiler) > 0 {
+			for _, p := range globalProfiler {
+				p.Stop()
+			}
 		}
 
 		if success {

cmd/storage-errors.go

@@ -16,88 +16,83 @@
 
 package cmd
 
-import (
-	"errors"
-)
-
 // errUnexpected - unexpected error, requires manual intervention.
-var errUnexpected = errors.New("Unexpected error, please report this issue at https://github.com/minio/minio/issues")
+var errUnexpected = StorageErr("Unexpected error, please report this issue at https://github.com/minio/minio/issues")
 
 // errCorruptedFormat - corrupted backend format.
-var errCorruptedFormat = errors.New("corrupted backend format, please join https://slack.min.io for assistance")
+var errCorruptedFormat = StorageErr("corrupted backend format, please join https://slack.min.io for assistance")
 
 // errUnformattedDisk - unformatted disk found.
-var errUnformattedDisk = errors.New("unformatted disk found")
+var errUnformattedDisk = StorageErr("unformatted disk found")
 
 // errDiskFull - cannot create volume or files when disk is full.
-var errDiskFull = errors.New("disk path full")
+var errDiskFull = StorageErr("disk path full")
 
 // errDiskNotFound - cannot find the underlying configured disk anymore.
-var errDiskNotFound = errors.New("disk not found")
+var errDiskNotFound = StorageErr("disk not found")
 
 // errFaultyRemoteDisk - remote disk is faulty.
-var errFaultyRemoteDisk = errors.New("remote disk is faulty")
+var errFaultyRemoteDisk = StorageErr("remote disk is faulty")
 
 // errFaultyDisk - disk is faulty.
-var errFaultyDisk = errors.New("disk is faulty")
+var errFaultyDisk = StorageErr("disk is faulty")
 
 // errDiskAccessDenied - we don't have write permissions on disk.
-var errDiskAccessDenied = errors.New("disk access denied")
+var errDiskAccessDenied = StorageErr("disk access denied")
 
 // errFileNotFound - cannot find the file.
-var errFileNotFound = errors.New("file not found")
+var errFileNotFound = StorageErr("file not found")
 
 // errTooManyOpenFiles - too many open files.
-var errTooManyOpenFiles = errors.New("too many open files")
+var errTooManyOpenFiles = StorageErr("too many open files")
 
 // errFileNameTooLong - given file name is too long than supported length.
-var errFileNameTooLong = errors.New("file name too long")
+var errFileNameTooLong = StorageErr("file name too long")
 
 // errVolumeExists - cannot create same volume again.
-var errVolumeExists = errors.New("volume already exists")
+var errVolumeExists = StorageErr("volume already exists")
 
 // errIsNotRegular - not of regular file type.
-var errIsNotRegular = errors.New("not of regular file type")
+var errIsNotRegular = StorageErr("not of regular file type")
 
 // errVolumeNotFound - cannot find the volume.
-var errVolumeNotFound = errors.New("volume not found")
+var errVolumeNotFound = StorageErr("volume not found")
 
 // errVolumeNotEmpty - volume not empty.
-var errVolumeNotEmpty = errors.New("volume is not empty")
+var errVolumeNotEmpty = StorageErr("volume is not empty")
 
 // errVolumeAccessDenied - cannot access volume, insufficient permissions.
-var errVolumeAccessDenied = errors.New("volume access denied")
+var errVolumeAccessDenied = StorageErr("volume access denied")
 
 // errFileAccessDenied - cannot access file, insufficient permissions.
-var errFileAccessDenied = errors.New("file access denied")
+var errFileAccessDenied = StorageErr("file access denied")
 
 // errFileCorrupt - file has an unexpected size, or is not readable
-var errFileCorrupt = errors.New("file is corrupted")
+var errFileCorrupt = StorageErr("file is corrupted")
 
 // errFileParentIsFile - cannot have overlapping objects, parent is already a file.
-var errFileParentIsFile = errors.New("parent is a file")
+var errFileParentIsFile = StorageErr("parent is a file")
 
 // errBitrotHashAlgoInvalid - the algo for bit-rot hash
 // verification is empty or invalid.
-var errBitrotHashAlgoInvalid = errors.New("bit-rot hash algorithm is invalid")
+var errBitrotHashAlgoInvalid = StorageErr("bit-rot hash algorithm is invalid")
 
 // errCrossDeviceLink - rename across devices not allowed.
-var errCrossDeviceLink = errors.New("Rename across devices not allowed, please fix your backend configuration")
+var errCrossDeviceLink = StorageErr("Rename across devices not allowed, please fix your backend configuration")
 
 // errMinDiskSize - cannot create volume or files when disk size is less than threshold.
-var errMinDiskSize = errors.New("The disk size is less than the minimum threshold")
+var errMinDiskSize = StorageErr("The disk size is less than the minimum threshold")
 
 // errLessData - returned when less data available than what was requested.
-var errLessData = errors.New("less data available than what was requested")
+var errLessData = StorageErr("less data available than what was requested")
 
 // errMoreData = returned when more data was sent by the caller than what it was supposed to.
-var errMoreData = errors.New("more data was sent than what was advertised")
+var errMoreData = StorageErr("more data was sent than what was advertised")
 
-// VerifyFileError represents error generated by VerifyFile posix call.
-type VerifyFileError string
+// StorageErr represents error generated by posix call.
+type StorageErr string
 
-// Error method for the hashMismatchError
-func (h VerifyFileError) Error() string {
+func (h StorageErr) Error() string {
 	return string(h)
 }
 

cmd/storage-interface.go

@@ -26,8 +26,8 @@ type StorageAPI interface {
 	String() string
 
 	// Storage operations.
-	IsOnline() bool // Returns true if disk is online.
-	LastError() error
+	IsOnline() bool   // Returns true if disk is online.
+	Hostname() string // Returns host name if remote host.
 	Close() error
 	SetDiskID(id string)
 

cmd/storage-rest-client.go

@@ -47,9 +47,9 @@ func isNetworkError(err error) bool {
 	return false
 }
 
-// Converts rpc.ServerError to underlying error. This function is
-// written so that the storageAPI errors are consistent across network
-// disks as well.
+// Converts network error to storageErr. This function is
+// written so that the storageAPI errors are consistent
+// across network disks.
 func toStorageErr(err error) error {
 	if err == nil {
 		return nil
@@ -111,14 +111,13 @@ type storageRESTClient struct {
 	endpoint   Endpoint
 	restClient *rest.Client
 	connected  int32
-	lastError  error
 	diskID     string
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is makred disconnected
 // permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
-func (client *storageRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+func (client *storageRESTClient) call(method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) {
 	if !client.IsOnline() {
 		return nil, errDiskNotFound
 	}
@@ -126,16 +125,17 @@ func (client *storageRESTClient) call(method string, values url.Values, body io.
 		values = make(url.Values)
 	}
 	values.Set(storageRESTDiskID, client.diskID)
-	respBody, err = client.restClient.Call(method, values, body, length)
+	respBody, err := client.restClient.Call(method, values, body, length)
 	if err == nil {
 		return respBody, nil
 	}
-	client.lastError = err
-	if isNetworkError(err) || err.Error() == errDiskStale.Error() {
+
+	err = toStorageErr(err)
+	if err == errDiskNotFound {
 		atomic.StoreInt32(&client.connected, 0)
 	}
 
-	return nil, toStorageErr(err)
+	return nil, err
 }
 
 // Stringer provides a canonicalized representation of network device.
@@ -148,6 +148,10 @@ func (client *storageRESTClient) IsOnline() bool {
 	return atomic.LoadInt32(&client.connected) == 1
 }
 
+func (client *storageRESTClient) Hostname() string {
+	return client.endpoint.Host
+}
+
 func (client *storageRESTClient) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
 	respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage, nil, nil, -1)
 	defer http.DrainBody(respBody)
@@ -170,11 +174,6 @@ func (client *storageRESTClient) CrawlAndGetDataUsage(endCh <-chan struct{}) (Da
 	return usageInfo, err
 }
 
-// LastError - returns the network error if any.
-func (client *storageRESTClient) LastError() error {
-	return client.lastError
-}
-
 func (client *storageRESTClient) SetDiskID(id string) {
 	client.diskID = id
 }

cmd/storage-rest-common.go

@@ -17,7 +17,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v12"
+	storageRESTVersion       = "v13" // Introduced StorageErr error type.
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )

cmd/storage-rest-server.go

@@ -480,13 +480,6 @@ type DeleteFileBulkErrsResp struct {
 	Errs []error
 }
 
-// DeleteFileError - error captured per delete operation
-type DeleteFileError string
-
-func (d DeleteFileError) Error() string {
-	return string(d)
-}
-
 // DeleteFileBulkHandler - delete a file.
 func (s *storageRESTServer) DeleteFileBulkHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -505,7 +498,7 @@ func (s *storageRESTServer) DeleteFileBulkHandler(w http.ResponseWriter, r *http
 	derrsResp := &DeleteFileBulkErrsResp{Errs: make([]error, len(errs))}
 	for idx, err := range errs {
 		if err != nil {
-			derrsResp.Errs[idx] = DeleteFileError(err.Error())
+			derrsResp.Errs[idx] = StorageErr(err.Error())
 		}
 	}
 
@@ -594,7 +587,7 @@ func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
 	<-doneCh
 	vresp := &VerifyFileResp{}
 	if err != nil {
-		vresp.Err = VerifyFileError(err.Error())
+		vresp.Err = StorageErr(err.Error())
 	}
 	encoder.Encode(vresp)
 	w.(http.Flusher).Flush()

cmd/sts-handlers.go

@@ -214,7 +214,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	// This policy is the policy associated with the user
 	// requesting for temporary credentials. The temporary
 	// credentials will inherit the same policy requirements.
-	m[iamPolicyName()] = policyName
+	m[iamPolicyClaimName()] = policyName
 
 	if len(sessionPolicyStr) > 0 {
 		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
@@ -350,7 +350,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 	// be set and configured on your identity provider as part of
 	// JWT custom claims.
 	var policyName string
-	if v, ok := m[iamPolicyName()]; ok {
+	if v, ok := m[iamPolicyClaimName()]; ok {
 		policyName, _ = v.(string)
 	}
 

cmd/test-utils_test.go

@@ -459,15 +459,30 @@ func resetGlobalIsXL() {
 
 // reset global heal state
 func resetGlobalHealState() {
+	// Init global heal state
 	if globalAllHealState == nil {
-		return
-	}
-	globalAllHealState.Lock()
-	defer globalAllHealState.Unlock()
-	for _, v := range globalAllHealState.healSeqMap {
-		if !v.hasEnded() {
-			v.stop()
+		globalAllHealState = initHealState()
+	} else {
+		globalAllHealState.Lock()
+		for _, v := range globalAllHealState.healSeqMap {
+			if !v.hasEnded() {
+				v.stop()
+			}
 		}
+		globalAllHealState.Unlock()
+	}
+
+	// Init background heal state
+	if globalBackgroundHealState == nil {
+		globalBackgroundHealState = initHealState()
+	} else {
+		globalBackgroundHealState.Lock()
+		for _, v := range globalBackgroundHealState.healSeqMap {
+			if !v.hasEnded() {
+				v.stop()
+			}
+		}
+		globalBackgroundHealState.Unlock()
 	}
 }
 
@@ -1894,7 +1909,8 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
 
-	globalIAMSys = NewIAMSys()
+	newAllSubsystems()
+
 	globalIAMSys.Init(objLayer)
 
 	buckets, err := objLayer.ListBuckets(context.Background())
@@ -1902,7 +1918,6 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 		t.Fatalf("Unable to list buckets on backend %s", err)
 	}
 
-	globalPolicySys = NewPolicySys()
 	globalPolicySys.Init(buckets, objLayer)
 
 	credentials := globalActiveCred

cmd/utils.go

@@ -32,18 +32,20 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/beevik/ntp"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
-	"github.com/pkg/profile"
 )
 
 // IsErrIgnored returns whether given error is ignored or not.
@@ -146,9 +148,10 @@ const (
 	// (Acceptable values range from 1 to 10000 inclusive)
 	globalMaxPartID = 10000
 
-	// Default values used while communicating with the cloud backends
-	defaultDialTimeout   = 30 * time.Second
-	defaultDialKeepAlive = 30 * time.Second
+	// Default values used while communicating for
+	// internode communication.
+	defaultDialTimeout   = 15 * time.Second
+	defaultDialKeepAlive = 20 * time.Second
 )
 
 // isMaxObjectSize - verify if max object size
@@ -187,94 +190,130 @@ func contains(slice interface{}, elem interface{}) bool {
 // provide any API to calculate the profiler file path in the
 // disk since the name of this latter is randomly generated.
 type profilerWrapper struct {
-	stopFn func()
-	pathFn func() string
+	stopFn func() ([]byte, error)
 }
 
-func (p profilerWrapper) Stop() {
-	p.stopFn()
-}
-
-func (p profilerWrapper) Path() string {
-	return p.pathFn()
+func (p profilerWrapper) Stop() ([]byte, error) {
+	return p.stopFn()
 }
 
 // Returns current profile data, returns error if there is no active
 // profiling in progress. Stops an active profile.
-func getProfileData() ([]byte, error) {
-	if globalProfiler == nil {
+func getProfileData() (map[string][]byte, error) {
+	globalProfilerMu.Lock()
+	defer globalProfilerMu.Unlock()
+
+	if len(globalProfiler) == 0 {
 		return nil, errors.New("profiler not enabled")
 	}
 
-	profilerPath := globalProfiler.Path()
-
-	// Stop the profiler
-	globalProfiler.Stop()
-
-	profilerFile, err := os.Open(profilerPath)
-	if err != nil {
-		return nil, err
+	dst := make(map[string][]byte, len(globalProfiler))
+	for typ, prof := range globalProfiler {
+		// Stop the profiler
+		var err error
+		buf, err := prof.Stop()
+		delete(globalProfiler, typ)
+		if err == nil {
+			dst[typ] = buf
+		}
 	}
-
-	return ioutil.ReadAll(profilerFile)
+	return dst, nil
 }
 
 // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
-func startProfiler(profilerType, dirPath string) (minioProfiler, error) {
-	var err error
-	if dirPath == "" {
-		dirPath, err = ioutil.TempDir("", "profile")
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	var profiler interface {
-		Stop()
-	}
-
-	var profilerFileName string
+func startProfiler(profilerType string) (minioProfiler, error) {
+	var prof profilerWrapper
 
 	// Enable profiler and set the name of the file that pkg/pprof
 	// library creates to store profiling data.
 	switch profilerType {
 	case "cpu":
-		profiler = profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
-		profilerFileName = "cpu.pprof"
+		dirPath, err := ioutil.TempDir("", "profile")
+		if err != nil {
+			return nil, err
+		}
+		fn := filepath.Join(dirPath, "cpu.out")
+		f, err := os.Create(fn)
+		if err != nil {
+			return nil, err
+		}
+		err = pprof.StartCPUProfile(f)
+		if err != nil {
+			return nil, err
+		}
+		prof.stopFn = func() ([]byte, error) {
+			pprof.StopCPUProfile()
+			err := f.Close()
+			if err != nil {
+				return nil, err
+			}
+			defer os.RemoveAll(dirPath)
+			return ioutil.ReadFile(fn)
+		}
 	case "mem":
-		profiler = profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
-		profilerFileName = "mem.pprof"
+		old := runtime.MemProfileRate
+		runtime.MemProfileRate = 4096
+		prof.stopFn = func() ([]byte, error) {
+			var buf bytes.Buffer
+			runtime.MemProfileRate = old
+			err := pprof.Lookup("heap").WriteTo(&buf, 0)
+			return buf.Bytes(), err
+		}
 	case "block":
-		profiler = profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
-		profilerFileName = "block.pprof"
+		runtime.SetBlockProfileRate(1)
+		prof.stopFn = func() ([]byte, error) {
+			var buf bytes.Buffer
+			runtime.SetBlockProfileRate(0)
+			err := pprof.Lookup("block").WriteTo(&buf, 0)
+			return buf.Bytes(), err
+		}
 	case "mutex":
-		profiler = profile.Start(profile.MutexProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
-		profilerFileName = "mutex.pprof"
+		runtime.SetMutexProfileFraction(1)
+		prof.stopFn = func() ([]byte, error) {
+			var buf bytes.Buffer
+			runtime.SetMutexProfileFraction(0)
+			err := pprof.Lookup("mutex").WriteTo(&buf, 0)
+			return buf.Bytes(), err
+		}
 	case "trace":
-		profiler = profile.Start(profile.TraceProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
-		profilerFileName = "trace.out"
+		dirPath, err := ioutil.TempDir("", "profile")
+		if err != nil {
+			return nil, err
+		}
+		fn := filepath.Join(dirPath, "trace.out")
+		f, err := os.Create(fn)
+		if err != nil {
+			return nil, err
+		}
+		err = trace.Start(f)
+		if err != nil {
+			return nil, err
+		}
+		prof.stopFn = func() ([]byte, error) {
+			trace.Stop()
+			err := f.Close()
+			if err != nil {
+				return nil, err
+			}
+			defer os.RemoveAll(dirPath)
+			return ioutil.ReadFile(fn)
+		}
 	default:
 		return nil, errors.New("profiler type unknown")
 	}
 
-	return &profilerWrapper{
-		stopFn: profiler.Stop,
-		pathFn: func() string {
-			return filepath.Join(dirPath, profilerFileName)
-		},
-	}, nil
+	return prof, nil
 }
 
 // minioProfiler - minio profiler interface.
 type minioProfiler interface {
 	// Stop the profiler
-	Stop()
-	// Return the path of the profiling file
-	Path() string
+	Stop() ([]byte, error)
 }
 
 // Global profiler to be used by service go-routine.
-var globalProfiler minioProfiler
+var globalProfiler map[string]minioProfiler
+var globalProfilerMu sync.Mutex
 
 // dump the request into a string in JSON format.
 func dumpRequest(r *http.Request) string {
@@ -351,7 +390,6 @@ func newCustomDialContext(dialTimeout, dialKeepAlive time.Duration) dialContext
 		dialer := &net.Dialer{
 			Timeout:   dialTimeout,
 			KeepAlive: dialKeepAlive,
-			DualStack: true,
 		}
 
 		return dialer.DialContext(ctx, network, addr)
@@ -364,10 +402,12 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive ti
 	tr := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		DialContext:           newCustomDialContext(dialTimeout, dialKeepAlive),
+		MaxIdleConns:          256,
 		MaxIdleConnsPerHost:   256,
-		IdleConnTimeout:       60 * time.Second,
-		TLSHandshakeTimeout:   30 * time.Second,
-		ExpectContinueTimeout: 10 * time.Second,
+		IdleConnTimeout:       30 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ResponseHeaderTimeout: 10 * time.Second,
+		ExpectContinueTimeout: 3 * time.Second,
 		TLSClientConfig:       tlsConfig,
 		// Go net/http automatically unzip if content-type is
 		// gzip disable this feature, as we are always interested
@@ -535,16 +575,13 @@ func splitN(str, delim string, num int) []string {
 	return retSplit
 }
 
-func iamPolicyName() string {
-	return globalOpenIDConfig.ClaimPrefix + iampolicy.PolicyName
+func iamPolicyClaimName() string {
+	return globalOpenIDConfig.ClaimPrefix + globalOpenIDConfig.ClaimName
 }
 
-func isWORMEnabled(bucket string) (Retention, bool) {
+func isWORMEnabled(bucket string) bool {
 	if isMinioMetaBucketName(bucket) {
-		return Retention{}, false
+		return false
 	}
-	if globalWORMEnabled {
-		return Retention{}, true
-	}
-	return globalBucketObjectLockConfig.Get(bucket)
+	return globalWORMEnabled
 }

cmd/utils_test.go

@@ -191,7 +191,7 @@ func TestURL2BucketObjectName(t *testing.T) {
 
 // Add tests for starting and stopping different profilers.
 func TestStartProfiler(t *testing.T) {
-	_, err := startProfiler("", "")
+	_, err := startProfiler("")
 	if err == nil {
 		t.Fatal("Expected a non nil error, but nil error returned for invalid profiler.")
 	}

cmd/web-handlers.go

@@ -306,7 +306,7 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
 	r.Header.Set("delimiter", SlashSeparator)
 
 	// If etcd, dns federation configured list buckets from etcd.
-	if globalDNSConfig != nil {
+	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
 			return toJSONError(ctx, err)
@@ -542,7 +542,7 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 					return toJSONError(ctx, err)
 				}
 			} else if lo.Objects[i].IsCompressed() {
-				var actualSize int64 = lo.Objects[i].GetActualSize()
+				actualSize := lo.Objects[i].GetActualSize()
 				if actualSize < 0 {
 					return toJSONError(ctx, errInvalidDecompressedSize)
 				}
@@ -1401,8 +1401,10 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 				info.Size = info.GetActualSize()
 			}
 			header := &zip.FileHeader{
-				Name:   strings.TrimPrefix(objectName, args.Prefix),
-				Method: zip.Deflate,
+				Name:     strings.TrimPrefix(objectName, args.Prefix),
+				Method:   zip.Deflate,
+				Flags:    1 << 11,
+				Modified: info.ModTime,
 			}
 			if hasStringSuffixInSlice(info.Name, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, info.ContentType) {
 				// We strictly disable compression for standard extensions/content-types.
@@ -2024,13 +2026,20 @@ func toJSONError(ctx context.Context, err error, params ...string) (jerr *json2.
 // toWebAPIError - convert into error into APIError.
 func toWebAPIError(ctx context.Context, err error) APIError {
 	switch err {
+	case errNoAuthToken:
+		return APIError{
+			Code:           "WebTokenMissing",
+			HTTPStatusCode: http.StatusBadRequest,
+			Description:    err.Error(),
+		}
 	case errServerNotInitialized:
 		return APIError{
 			Code:           "XMinioServerNotInitialized",
 			HTTPStatusCode: http.StatusServiceUnavailable,
 			Description:    err.Error(),
 		}
-	case errAuthentication, auth.ErrInvalidAccessKeyLength, auth.ErrInvalidSecretKeyLength, errInvalidAccessKeyID:
+	case errAuthentication, auth.ErrInvalidAccessKeyLength,
+		auth.ErrInvalidSecretKeyLength, errInvalidAccessKeyID:
 		return APIError{
 			Code:           "AccessDenied",
 			HTTPStatusCode: http.StatusForbidden,
@@ -2106,11 +2115,7 @@ func toWebAPIError(ctx context.Context, err error) APIError {
 
 	// Log unexpected and unhandled errors.
 	logger.LogIf(ctx, err)
-	return APIError{
-		Code:           "InternalError",
-		HTTPStatusCode: http.StatusInternalServerError,
-		Description:    err.Error(),
-	}
+	return toAPIError(ctx, err)
 }
 
 // writeWebErrorResponse - set HTTP status code and write error description to the body.

cmd/xl-sets.go

@@ -56,6 +56,11 @@ func (s setsStorageAPI) Close() error {
 	return nil
 }
 
+// Information of a new disk connection
+type diskConnectInfo struct {
+	setIndex int
+}
+
 // xlSets implements ObjectLayer combining a static list of erasure coded
 // object sets. NOTE: There is no dynamic scaling allowed or intended in
 // current design.
@@ -74,15 +79,14 @@ type xlSets struct {
 	// Distributed locker clients.
 	xlLockers setsDsyncLockers
 
-	// Lockers map holds dsync lockers for each endpoint
-	lockersMap map[Endpoint]dsync.NetLocker
-
 	// List of endpoints provided on the command line.
 	endpoints Endpoints
 
 	// Total number of sets and the number of disks per set.
 	setCount, drivesPerSet int
 
+	disksConnectEvent chan diskConnectInfo
+
 	// Done channel to control monitoring loop.
 	disksConnectDoneCh chan struct{}
 
@@ -91,6 +95,9 @@ type xlSets struct {
 
 	// Merge tree walk
 	pool *MergeWalkPool
+
+	mrfMU      sync.Mutex
+	mrfUploads map[string]int
 }
 
 // isConnected - checks if the endpoint is connected or not.
@@ -112,9 +119,6 @@ func (s *xlSets) isConnected(endpoint Endpoint) bool {
 			if s.xlDisks[i][j].String() != endpointStr {
 				continue
 			}
-			if !s.xlLockers[i][j].IsOnline() {
-				continue
-			}
 			return s.xlDisks[i][j].IsOnline()
 		}
 	}
@@ -141,6 +145,8 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatXLV3, error) {
 
 // findDiskIndex - returns the i,j'th position of the input `format` against the reference
 // format, after successful validation.
+//   - i'th position is the set index
+//   - j'th position is the disk index in the current set
 func findDiskIndex(refFormat, format *formatXLV3) (int, int, error) {
 	if err := formatXLV3Check(refFormat, format); err != nil {
 		return 0, 0, err
@@ -161,9 +167,9 @@ func findDiskIndex(refFormat, format *formatXLV3) (int, int, error) {
 	return -1, -1, fmt.Errorf("diskID: %s not found", format.XL.This)
 }
 
-// connectDisksAndLockersWithQuorum is same as connectDisksAndLockers but waits
+// connectDisksWithQuorum is same as connectDisks but waits
 // for quorum number of formatted disks to be online in any given sets.
-func (s *xlSets) connectDisksAndLockersWithQuorum() {
+func (s *xlSets) connectDisksWithQuorum() {
 	var onlineDisks int
 	for onlineDisks < len(s.endpoints)/2 {
 		for _, endpoint := range s.endpoints {
@@ -184,7 +190,6 @@ func (s *xlSets) connectDisksAndLockersWithQuorum() {
 			}
 			disk.SetDiskID(format.XL.This)
 			s.xlDisks[i][j] = disk
-			s.xlLockers[i][j] = s.lockersMap[endpoint]
 			onlineDisks++
 		}
 		// Sleep for a while - so that we don't go into
@@ -193,9 +198,9 @@ func (s *xlSets) connectDisksAndLockersWithQuorum() {
 	}
 }
 
-// connectDisksAndLockers - attempt to connect all the endpoints, loads format
+// connectDisks - attempt to connect all the endpoints, loads format
 // and re-arranges the disks in proper position.
-func (s *xlSets) connectDisksAndLockers() {
+func (s *xlSets) connectDisks() {
 	for _, endpoint := range s.endpoints {
 		if s.isConnected(endpoint) {
 			continue
@@ -205,7 +210,7 @@ func (s *xlSets) connectDisksAndLockers() {
 			printEndpointError(endpoint, err)
 			continue
 		}
-		i, j, err := findDiskIndex(s.format, format)
+		setIndex, diskIndex, err := findDiskIndex(s.format, format)
 		if err != nil {
 			// Close the internal connection to avoid connection leaks.
 			disk.Close()
@@ -214,9 +219,14 @@ func (s *xlSets) connectDisksAndLockers() {
 		}
 		disk.SetDiskID(format.XL.This)
 		s.xlDisksMu.Lock()
-		s.xlDisks[i][j] = disk
-		s.xlLockers[i][j] = s.lockersMap[endpoint]
+		s.xlDisks[setIndex][diskIndex] = disk
 		s.xlDisksMu.Unlock()
+
+		// Send a new disk connect event with a timeout
+		select {
+		case s.disksConnectEvent <- diskConnectInfo{setIndex: setIndex}:
+		case <-time.After(100 * time.Millisecond):
+		}
 	}
 }
 
@@ -224,6 +234,7 @@ func (s *xlSets) connectDisksAndLockers() {
 // endpoints by reconnecting them and making sure to place them into right position in
 // the set topology, this monitoring happens at a given monitoring interval.
 func (s *xlSets) monitorAndConnectEndpoints(monitorInterval time.Duration) {
+
 	ticker := time.NewTicker(monitorInterval)
 	// Stop the timer.
 	defer ticker.Stop()
@@ -235,23 +246,15 @@ func (s *xlSets) monitorAndConnectEndpoints(monitorInterval time.Duration) {
 		case <-s.disksConnectDoneCh:
 			return
 		case <-ticker.C:
-			s.connectDisksAndLockers()
+			s.connectDisks()
 		}
 	}
 }
 
 func (s *xlSets) GetLockers(setIndex int) func() []dsync.NetLocker {
 	return func() []dsync.NetLocker {
-		s.xlDisksMu.Lock()
-		defer s.xlDisksMu.Unlock()
 		lockers := make([]dsync.NetLocker, s.drivesPerSet)
 		copy(lockers, s.xlLockers[setIndex])
-		for i, lk := range lockers {
-			// Add error lockers for unavailable locker.
-			if lk == nil {
-				lockers[i] = &errorLocker{}
-			}
-		}
 		return lockers
 	}
 }
@@ -271,24 +274,20 @@ const defaultMonitorConnectEndpointInterval = time.Second * 10 // Set to 10 secs
 
 // Initialize new set of erasure coded sets.
 func newXLSets(endpoints Endpoints, format *formatXLV3, setCount int, drivesPerSet int) (*xlSets, error) {
-	lockersMap := make(map[Endpoint]dsync.NetLocker)
-	for _, endpoint := range endpoints {
-		lockersMap[endpoint] = newLockAPI(endpoint)
-	}
-
 	// Initialize the XL sets instance.
 	s := &xlSets{
 		sets:               make([]*xlObjects, setCount),
 		xlDisks:            make([][]StorageAPI, setCount),
 		xlLockers:          make([][]dsync.NetLocker, setCount),
-		lockersMap:         lockersMap,
 		endpoints:          endpoints,
 		setCount:           setCount,
 		drivesPerSet:       drivesPerSet,
 		format:             format,
+		disksConnectEvent:  make(chan diskConnectInfo),
 		disksConnectDoneCh: make(chan struct{}),
 		distributionAlgo:   format.XL.DistributionAlgo,
 		pool:               NewMergeWalkPool(globalMergeLookupTimeout),
+		mrfUploads:         make(map[string]int),
 	}
 
 	mutex := newNSLock(globalIsDistXL)
@@ -303,21 +302,33 @@ func newXLSets(endpoints Endpoints, format *formatXLV3, setCount int, drivesPerS
 
 		// Initialize xl objects for a given set.
 		s.sets[i] = &xlObjects{
-			getDisks:   s.GetDisks(i),
-			getLockers: s.GetLockers(i),
-			nsMutex:    mutex,
-			bp:         bp,
+			getDisks:    s.GetDisks(i),
+			getLockers:  s.GetLockers(i),
+			nsMutex:     mutex,
+			bp:          bp,
+			mrfUploadCh: make(chan partialUpload, 10000),
 		}
+
 		go s.sets[i].cleanupStaleMultipartUploads(context.Background(),
 			GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
 	}
 
+	// Rely on endpoints list to initialize, init lockers.
+	for i := 0; i < s.setCount; i++ {
+		for j := 0; j < s.drivesPerSet; j++ {
+			s.xlLockers[i][j] = newLockAPI(s.endpoints[i*s.drivesPerSet+j])
+		}
+	}
+
 	// Connect disks right away, but wait until we have `format.json` quorum.
-	s.connectDisksAndLockersWithQuorum()
+	s.connectDisksWithQuorum()
 
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
 
+	go s.maintainMRFList()
+	go s.healMRFRoutine()
+
 	return s, nil
 }
 
@@ -1312,9 +1323,11 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
 	// Replace the new format.
 	s.format = refFormat
 
-	// Close all existing disks, lockers and reconnect all the disks/lockers.
+	// Close all existing disks and reconnect all the disks.
+	s.xlDisksMu.Lock()
 	s.xlDisks.Close()
-	s.connectDisksAndLockers()
+	s.xlDisksMu.Unlock()
+	s.connectDisks()
 
 	// Restart monitoring loop to monitor reformatted disks again.
 	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
@@ -1435,11 +1448,6 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 		return res, errNoHealRequired
 	}
 
-	// All disks are unformatted, return quorum error.
-	if shouldInitXLDisks(sErrs) {
-		return res, errXLReadQuorum
-	}
-
 	refFormat, err := getFormatXLInQuorum(formats)
 	if err != nil {
 		return res, err
@@ -1505,8 +1513,10 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 		s.format = refFormat
 
 		// Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers.
+		s.xlDisksMu.Lock()
 		s.xlDisks.Close()
-		s.connectDisksAndLockers()
+		s.xlDisksMu.Unlock()
+		s.connectDisks()
 
 		// Restart our monitoring loop to start monitoring newly formatted disks.
 		go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
@@ -1657,7 +1667,7 @@ func (s *xlSets) GetMetrics(ctx context.Context) (*Metrics, error) {
 	return &Metrics{}, NotImplemented{}
 }
 
-// IsReady - Returns true if more than n/2 disks (quorum) are online
+// IsReady - Returns true if atleast n/2 disks (read quorum) are online
 func (s *xlSets) IsReady(_ context.Context) bool {
 	s.xlDisksMu.RLock()
 	defer s.xlDisksMu.RUnlock()
@@ -1668,14 +1678,11 @@ func (s *xlSets) IsReady(_ context.Context) bool {
 			if s.xlDisks[i][j] == nil {
 				continue
 			}
-			if !s.xlLockers[i][j].IsOnline() {
-				continue
-			}
 			if s.xlDisks[i][j].IsOnline() {
 				activeDisks++
 			}
-			// Return if more than n/2 disks are online.
-			if activeDisks > len(s.endpoints)/2 {
+			// Return true if read quorum is available.
+			if activeDisks >= len(s.endpoints)/2 {
 				return true
 			}
 		}
@@ -1683,3 +1690,72 @@ func (s *xlSets) IsReady(_ context.Context) bool {
 	// Disks are not ready
 	return false
 }
+
+// maintainMRFList gathers the list of successful partial uploads
+// from all underlying xl sets and puts them in a global map which
+// should not have more than 10000 entries.
+func (s *xlSets) maintainMRFList() {
+	var agg = make(chan partialUpload, 10000)
+	for i, xl := range s.sets {
+		go func(c <-chan partialUpload, setIndex int) {
+			for msg := range c {
+				msg.failedSet = setIndex
+				select {
+				case agg <- msg:
+				default:
+				}
+			}
+		}(xl.mrfUploadCh, i)
+	}
+
+	for fUpload := range agg {
+		s.mrfMU.Lock()
+		if len(s.mrfUploads) > 10000 {
+			s.mrfMU.Unlock()
+			continue
+		}
+		s.mrfUploads[pathJoin(fUpload.bucket, fUpload.object)] = fUpload.failedSet
+		s.mrfMU.Unlock()
+	}
+}
+
+// healMRFRoutine monitors new disks connection, sweep the MRF list
+// to find objects related to the new disk that needs to be healed.
+func (s *xlSets) healMRFRoutine() {
+	// Wait until background heal state is initialized
+	var bgSeq *healSequence
+	for {
+		var ok bool
+		bgSeq, ok = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+		if ok {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+
+	for e := range s.disksConnectEvent {
+		// Get the list of objects related the xl set
+		// to which the connected disk belongs.
+		var mrfUploads []string
+		s.mrfMU.Lock()
+		for k, v := range s.mrfUploads {
+			if v == e.setIndex {
+				mrfUploads = append(mrfUploads, k)
+			}
+		}
+		s.mrfMU.Unlock()
+
+		// Heal objects
+		for _, u := range mrfUploads {
+			// Send an object to be healed with a timeout
+			select {
+			case bgSeq.sourceCh <- u:
+			case <-time.After(100 * time.Millisecond):
+			}
+
+			s.mrfMU.Lock()
+			delete(s.mrfUploads, u)
+			s.mrfMU.Unlock()
+		}
+	}
+}

cmd/xl-v1-metadata.go

@@ -26,6 +26,7 @@ import (
 	"sort"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -77,6 +78,7 @@ func (c ChecksumInfo) MarshalJSON() ([]byte, error) {
 // UnmarshalJSON - should never be called, instead xlMetaV1UnmarshalJSON() should be used.
 func (c *ChecksumInfo) UnmarshalJSON(data []byte) error {
 	var info checksumInfoJSON
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err := json.Unmarshal(data, &info); err != nil {
 		return err
 	}

cmd/xl-v1-multipart.go

@@ -584,8 +584,10 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 	uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
 
+	storageDisks := xl.getDisks()
+
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllXLMetadata(ctx, xl.getDisks(), minioMetaMultipartBucket, uploadIDPath)
+	partsMetadata, errs := readAllXLMetadata(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath)
 
 	// get Quorum for this object
 	_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, partsMetadata, errs)
@@ -598,7 +600,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, toObjectErr(reducedErr, bucket, object)
 	}
 
-	onlineDisks, modTime := listOnlineDisks(xl.getDisks(), partsMetadata, errs)
+	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// Calculate full object size.
 	var objectSize int64
@@ -708,7 +710,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 	if xl.isObject(bucket, object) {
 		// Deny if WORM is enabled
-		if _, ok := isWORMEnabled(bucket); ok {
+		if isWORMEnabled(bucket) {
 			if _, err := xl.getObjectInfo(ctx, bucket, object); err == nil {
 				return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
 			}
@@ -743,10 +745,17 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Rename the multipart object to final location.
-	if _, err = rename(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, bucket, object, true, writeQuorum, nil); err != nil {
+	if onlineDisks, err = rename(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, bucket, object, true, writeQuorum, nil); err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
 
+	// Check if there is any offline disk and add it to the MRF list
+	for i := 0; i < len(onlineDisks); i++ {
+		if onlineDisks[i] == nil || storageDisks[i] == nil {
+			xl.addPartialUpload(bucket, object)
+		}
+	}
+
 	// Success, return object info.
 	return xlMeta.ToObjectInfo(bucket, object), nil
 }

cmd/xl-v1-object.go

@@ -611,7 +611,7 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 
 	if xl.isObject(bucket, object) {
 		// Deny if WORM is enabled
-		if _, ok := isWORMEnabled(bucket); ok {
+		if isWORMEnabled(bucket) {
 			if _, err := xl.getObjectInfo(ctx, bucket, object); err == nil {
 				return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
 			}
@@ -626,7 +626,7 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 		// NOTE: Do not use online disks slice here: the reason is that existing object should be purged
 		// regardless of `xl.json` status and rolled back in case of errors. Also allow renaming the
 		// existing object if it is not present in quorum disks so users can overwrite stale objects.
-		_, err = rename(ctx, xl.getDisks(), bucket, object, minioMetaTmpBucket, newUniqueID, true, writeQuorum, []error{errFileNotFound})
+		_, err = rename(ctx, storageDisks, bucket, object, minioMetaTmpBucket, newUniqueID, true, writeQuorum, []error{errFileNotFound})
 		if err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
 		}
@@ -646,11 +646,19 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 	}
 
 	// Rename the successfully written temporary object to final location.
-	_, err = rename(ctx, onlineDisks, minioMetaTmpBucket, tempObj, bucket, object, true, writeQuorum, nil)
-	if err != nil {
+	if onlineDisks, err = rename(ctx, onlineDisks, minioMetaTmpBucket, tempObj, bucket, object, true, writeQuorum, nil); err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
+	// Whether a disk was initially or becomes offline
+	// during this upload, send it to the MRF list.
+	for i := 0; i < len(onlineDisks); i++ {
+		if onlineDisks[i] == nil || storageDisks[i] == nil {
+			xl.addPartialUpload(bucket, object)
+			break
+		}
+	}
+
 	// Object info is the same in all disks, so we can pick the first meta
 	// of the first disk
 	xlMeta = partsMetadata[0]
@@ -960,3 +968,12 @@ func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	}
 	return listObjectsV2Info, err
 }
+
+// Send the successul but partial upload, however ignore
+// if the channel is blocked by other items.
+func (xl xlObjects) addPartialUpload(bucket, key string) {
+	select {
+	case xl.mrfUploadCh <- partialUpload{bucket: bucket, object: key}:
+	default:
+	}
+}

cmd/xl-v1-utils.go

@@ -18,11 +18,11 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"hash/crc32"
 	"path"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -117,6 +117,7 @@ func hashOrder(key string, cardinality int) []int {
 
 // Constructs xlMetaV1 using `jsoniter` lib.
 func xlMetaV1UnmarshalJSON(ctx context.Context, xlMetaBuf []byte) (xlMeta xlMetaV1, err error) {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	err = json.Unmarshal(xlMetaBuf, &xlMeta)
 	return xlMeta, err
 }

cmd/xl-v1.go

@@ -39,6 +39,14 @@ const (
 // OfflineDisk represents an unavailable disk.
 var OfflineDisk StorageAPI // zero value is nil
 
+// partialUpload is a successful upload of an object
+// but not written in all disks (having quorum)
+type partialUpload struct {
+	bucket    string
+	object    string
+	failedSet int
+}
+
 // xlObjects - Implements XL object layer.
 type xlObjects struct {
 	// getDisks returns list of storageAPIs.
@@ -55,6 +63,8 @@ type xlObjects struct {
 
 	// TODO: ListObjects pool management, should be removed in future.
 	listPool *TreeWalkPool
+
+	mrfUploadCh chan partialUpload
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
@@ -123,7 +133,6 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin
 	// Wait for the routines.
 	for i, err := range g.Wait() {
 		peerAddr, pErr := getPeerAddress(disksInfo[i].RelativePath)
-
 		if pErr != nil {
 			continue
 		}
@@ -144,39 +153,20 @@ func getDisksInfo(disks []StorageAPI) (disksInfo []DiskInfo, onlineDisks, offlin
 	return disksInfo, onlineDisks, offlineDisks
 }
 
-// returns sorted disksInfo slice which has only valid entries.
-// i.e the entries where the total size of the disk is not stated
-// as 0Bytes, this means that the disk is not online or ignored.
-func sortValidDisksInfo(disksInfo []DiskInfo) []DiskInfo {
-	var validDisksInfo []DiskInfo
-	for _, diskInfo := range disksInfo {
-		if diskInfo.Total == 0 {
-			continue
-		}
-		validDisksInfo = append(validDisksInfo, diskInfo)
-	}
-	sort.Sort(byDiskTotal(validDisksInfo))
-	return validDisksInfo
-}
-
 // Get an aggregated storage info across all disks.
 func getStorageInfo(disks []StorageAPI) StorageInfo {
 	disksInfo, onlineDisks, offlineDisks := getDisksInfo(disks)
 
 	// Sort so that the first element is the smallest.
-	validDisksInfo := sortValidDisksInfo(disksInfo)
-	// If there are no valid disks, set total and free disks to 0
-	if len(validDisksInfo) == 0 {
-		return StorageInfo{}
-	}
+	sort.Sort(byDiskTotal(disksInfo))
 
 	// Combine all disks to get total usage
-	usedList := make([]uint64, len(validDisksInfo))
-	totalList := make([]uint64, len(validDisksInfo))
-	availableList := make([]uint64, len(validDisksInfo))
-	mountPaths := make([]string, len(validDisksInfo))
+	usedList := make([]uint64, len(disksInfo))
+	totalList := make([]uint64, len(disksInfo))
+	availableList := make([]uint64, len(disksInfo))
+	mountPaths := make([]string, len(disksInfo))
 
-	for i, di := range validDisksInfo {
+	for i, di := range disksInfo {
 		usedList[i] = di.Used
 		totalList[i] = di.Total
 		availableList[i] = di.Free