fix: make sure to lock CopyObject for in-place updates (#10492)

Fixes #10453

.nancy-ignore

@@ -0,0 +1,6 @@
+CVE-2018-17075
+CVE-2018-17848
+CVE-2018-17846
+CVE-2018-17847
+CVE-2018-17143
+CVE-2018-17142

README.md

@@ -177,7 +177,7 @@ mc admin update <minio alias, e.g., myminio>
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
 - `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
 - In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
-- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)x
+- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
 - If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
 - If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 

buildscripts/verify-build.sh

@@ -45,88 +45,63 @@ FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
 function start_minio_fs()
 {
     "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
-    minio_pid=$!
     sleep 10
-
-    echo "$minio_pid"
 }
 
 function start_minio_erasure()
 {
     "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
-    minio_pid=$!
     sleep 15
-
-    echo "$minio_pid"
 }
 
 function start_minio_erasure_sets()
 {
     "${MINIO[@]}" server "${WORK_DIR}/erasure-disk-sets{1...32}" >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
-    minio_pid=$!
     sleep 15
-
-    echo "$minio_pid"
 }
 
 function start_minio_zone_erasure_sets()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
 
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function start_minio_zone_erasure_sets_ipv6()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
 
     "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-
     "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function start_minio_dist_erasure()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
     "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
-    minio_pids[2]=$!
     "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
-    minio_pids[3]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function run_test_fs()
 {
-    minio_pid="$(start_minio_fs)"
+    start_minio_fs
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -138,12 +113,12 @@ function run_test_fs()
 }
 
 function run_test_erasure_sets() {
-    minio_pid="$(start_minio_erasure_sets)"
+    start_minio_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -156,14 +131,12 @@ function run_test_erasure_sets() {
 
 function run_test_zone_erasure_sets()
 {
-    minio_pids=( $(start_minio_zone_erasure_sets) )
+    start_minio_zone_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -182,16 +155,14 @@ function run_test_zone_erasure_sets()
 
 function run_test_zone_erasure_sets_ipv6()
 {
-    minio_pids=( $(start_minio_zone_erasure_sets_ipv6) )
+    start_minio_zone_erasure_sets_ipv6
 
     export SERVER_ENDPOINT="[::1]:9000"
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -210,12 +181,12 @@ function run_test_zone_erasure_sets_ipv6()
 
 function run_test_erasure()
 {
-    minio_pid="$(start_minio_erasure)"
+    start_minio_erasure
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -228,14 +199,12 @@ function run_test_erasure()
 
 function run_test_dist_erasure()
 {
-    minio_pids=( $(start_minio_dist_erasure) )
+    start_minio_dist_erasure
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then

cmd/admin-handlers.go

@@ -1241,7 +1241,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	vars := mux.Vars(r)
+	query := r.URL.Query()
 	obdInfo := madmin.OBDInfo{}
 	obdInfoCh := make(chan madmin.OBDInfo)
 
@@ -1287,7 +1287,13 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	go func() {
 		defer close(obdInfoCh)
 
-		if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
+		if log := query.Get("log"); log == "true" {
+			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r))
+			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if cpu := query.Get("syscpu"); cpu == "true" {
 			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
 
 			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
@@ -1295,7 +1301,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
+		if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
 			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
@@ -1303,7 +1309,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
+		if osInfo := query.Get("sysosinfo"); osInfo == "true" {
 			osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
@@ -1311,7 +1317,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if mem, ok := vars["sysmem"]; ok && mem == "true" {
+		if mem := query.Get("sysmem"); mem == "true" {
 			memInfo := getLocalMemOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
@@ -1319,7 +1325,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if proc, ok := vars["sysprocess"]; ok && proc == "true" {
+		if proc := query.Get("sysprocess"); proc == "true" {
 			procInfo := getLocalProcOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
@@ -1327,14 +1333,14 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if config, ok := vars["minioconfig"]; ok && config == "true" {
+		if config := query.Get("minioconfig"); config == "true" {
 			cfg, err := readServerConfig(ctx, objectAPI)
 			logger.LogIf(ctx, err)
 			obdInfo.Minio.Config = cfg
 			partialWrite(obdInfo)
 		}
 
-		if drive, ok := vars["perfdrive"]; ok && drive == "true" {
+		if drive := query.Get("perfdrive"); drive == "true" {
 			// Get drive obd details from local server's drive(s)
 			driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
 			driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
@@ -1365,7 +1371,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure {
+		if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
 			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
 			partialWrite(obdInfo)
 
@@ -1379,6 +1385,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
 			partialWrite(obdInfo)
 		}
+
 	}()
 
 	ticker := time.NewTicker(30 * time.Second)

cmd/admin-router.go

@@ -212,18 +212,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		if !globalIsGateway {
 			// -- OBD API --
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").
-				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).
-				Queries("perfdrive", "{perfdrive:true|false}",
-					"perfnet", "{perfnet:true|false}",
-					"minioinfo", "{minioinfo:true|false}",
-					"minioconfig", "{minioconfig:true|false}",
-					"syscpu", "{syscpu:true|false}",
-					"sysdiskhw", "{sysdiskhw:true|false}",
-					"sysosinfo", "{sysosinfo:true|false}",
-					"sysmem", "{sysmem:true|false}",
-					"sysprocess", "{sysprocess:true|false}",
-				)
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
+				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
 		}
 	}
 

cmd/api-errors.go

@@ -971,7 +971,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMetadataTooLarge: {
-		Code:           "InvalidArgument",
+		Code:           "MetadataTooLarge",
 		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},

cmd/api-response.go

@@ -408,7 +408,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 // generates ListBucketsResponse from array of BucketInfo which can be
 // serialized to match XML and JSON API spec output.
 func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
-	var listbuckets []Bucket
+	listbuckets := make([]Bucket, 0, len(buckets))
 	var data = ListBucketsResponse{}
 	var owner = Owner{}
 
@@ -428,8 +428,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
-	var versions []ObjectVersion
-	var prefixes []CommonPrefix
+	versions := make([]ObjectVersion, 0, len(resp.Objects))
 	var owner = Owner{}
 	var data = ListVersionsResponse{}
 
@@ -473,6 +472,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 	data.VersionIDMarker = versionIDMarker
 	data.IsTruncated = resp.IsTruncated
 
+	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
@@ -484,8 +484,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
-	var contents []Object
-	var prefixes []CommonPrefix
+	contents := make([]Object, 0, len(resp.Objects))
 	var owner = Owner{}
 	var data = ListObjectsResponse{}
 
@@ -517,9 +516,10 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 	data.Marker = s3EncodeName(marker, encodingType)
 	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.MaxKeys = maxKeys
-
 	data.NextMarker = s3EncodeName(resp.NextMarker, encodingType)
 	data.IsTruncated = resp.IsTruncated
+
+	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
@@ -531,8 +531,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
 func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
-	var contents []Object
-	var commonPrefixes []CommonPrefix
+	contents := make([]Object, 0, len(objects))
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
@@ -585,6 +584,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token))
 	data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken))
 	data.IsTruncated = isTruncated
+
+	commonPrefixes := make([]CommonPrefix, 0, len(prefixes))
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)

cmd/api-router.go

@@ -61,10 +61,6 @@ func newCachedObjectLayerFn() CacheObjectLayer {
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
 	CacheAPI  func() CacheObjectLayer
-	// Returns true of handlers should interpret encryption.
-	EncryptionEnabled func() bool
-	// Returns true if handlers allow SSE-KMS encryption headers.
-	AllowSSEKMS func() bool
 }
 
 // getHost tries its best to return the request host.
@@ -78,17 +74,11 @@ func getHost(r *http.Request) string {
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
+func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
 		CacheAPI:  newCachedObjectLayerFn,
-		EncryptionEnabled: func() bool {
-			return encryptionEnabled
-		},
-		AllowSSEKMS: func() bool {
-			return allowSSEKMS
-		},
 	}
 
 	// API Router

cmd/bucket-handlers.go

@@ -659,16 +659,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucket := mux.Vars(r)["bucket"]
 
-	// To detect if the client has disconnected.
-	r.Body = &contextReader{r.Body, r.Context()}
-
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {

cmd/bucket-replication.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strings"
 	"time"
@@ -179,20 +180,24 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 	if tgt == nil {
 		return
 	}
-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+		VersionID: versionID,
+	})
 	if err != nil {
 		return
 	}
-	defer gr.Close()
+
 	objInfo := gr.ObjInfo
 	size, err := objInfo.GetActualSize()
 	if err != nil {
 		logger.LogIf(ctx, err)
+		gr.Close()
 		return
 	}
 
 	dest := cfg.GetDestination()
 	if dest.Bucket == "" {
+		gr.Close()
 		return
 	}
 	// In the rare event that replication is in pending state either due to
@@ -201,6 +206,7 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 	if healPending {
 		_, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
 		if err == nil {
+			gr.Close()
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed.
 			return
@@ -210,6 +216,7 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 
 	replicationStatus := replication.Complete
 	_, err = tgt.PutObject(ctx, dest.Bucket, object, gr, size, "", "", putOpts)
+	gr.Close()
 	if err != nil {
 		replicationStatus = replication.Failed
 		// Notify replication failure  event.
@@ -231,8 +238,10 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 	objInfo.metadataOnly = true // Perform only metadata updates.
 	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
 		VersionID: objInfo.VersionID,
-	}, ObjectOptions{VersionID: objInfo.VersionID}); err != nil {
-		logger.LogIf(ctx, err)
+	}, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	}); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
 	}
 }
 

cmd/common-main.go

@@ -43,10 +43,6 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	// Initialize globalConsoleSys system
-	globalConsoleSys = NewConsoleLogger(GlobalContext)
-	logger.AddTarget(globalConsoleSys)
-
 	gob.Register(StorageErr(""))
 }
 

cmd/config-current.go

@@ -25,6 +25,7 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
@@ -55,6 +56,7 @@ func initHelp() {
 		config.KmsKesSubSys:         crypto.DefaultKesKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
+		config.CrawlerSubSys:        crawler.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -106,6 +108,10 @@ func initHelp() {
 			Key:         config.APISubSys,
 			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
 		},
+		config.HelpKV{
+			Key:         config.CrawlerSubSys,
+			Description: "manage continuous disk crawling for bucket disk usage, lifecycle, quota and data integrity checks",
+		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
 			Description:     "send server logs to webhook endpoints",
@@ -185,6 +191,7 @@ func initHelp() {
 		config.EtcdSubSys:           etcd.Help,
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
+		config.CrawlerSubSys:        crawler.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -246,6 +253,10 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
+	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+		return err
+	}
+
 	{
 		etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 		if err != nil {
@@ -398,6 +409,11 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 
 	globalAPIConfig.init(apiConfig, setDriveCount)
 
+	// Initialize remote instance transport once.
+	getRemoteInstanceTransportOnce.Do(func() {
+		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
+	})
+
 	if globalIsErasure {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
 		if err != nil {
@@ -422,6 +438,10 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 	}
+	globalCrawlerConfig, err = crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to read crawler config: %w", err))
+	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 	if err != nil {
@@ -435,7 +455,7 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 
 	// Enable auto-encryption if enabled
 	globalAutoEncryption = kmsCfg.AutoEncryption
-	if globalAutoEncryption {
+	if globalAutoEncryption && !globalIsGateway {
 		logger.LogIf(ctx, fmt.Errorf("%s env is deprecated please migrate to using `mc encrypt` at bucket level", crypto.EnvKMSAutoEncryption))
 	}
 

cmd/config/api/api.go

@@ -29,15 +29,17 @@ import (
 
 // API sub-system constants
 const (
-	apiRequestsMax      = "requests_max"
-	apiRequestsDeadline = "requests_deadline"
-	apiReadyDeadline    = "ready_deadline"
-	apiCorsAllowOrigin  = "cors_allow_origin"
+	apiRequestsMax             = "requests_max"
+	apiRequestsDeadline        = "requests_deadline"
+	apiReadyDeadline           = "ready_deadline"
+	apiCorsAllowOrigin         = "cors_allow_origin"
+	apiRemoteTransportDeadline = "remote_transport_deadline"
 
-	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
-	EnvAPICorsAllowOrigin  = "MINIO_API_CORS_ALLOW_ORIGIN"
+	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIReadyDeadline           = "MINIO_API_READY_DEADLINE"
+	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
+	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
 )
 
 // DefaultKVS - default storage class config
@@ -59,15 +61,20 @@ var (
 			Key:   apiCorsAllowOrigin,
 			Value: "*",
 		},
+		config.KV{
+			Key:   apiRemoteTransportDeadline,
+			Value: "2h",
+		},
 	}
 )
 
 // Config storage class configuration
 type Config struct {
-	APIRequestsMax      int           `json:"requests_max"`
-	APIRequestsDeadline time.Duration `json:"requests_deadline"`
-	APIReadyDeadline    time.Duration `json:"ready_deadline"`
-	APICorsAllowOrigin  []string      `json:"cors_allow_origin"`
+	RequestsMax             int           `json:"requests_max"`
+	RequestsDeadline        time.Duration `json:"requests_deadline"`
+	ReadyDeadline           time.Duration `json:"ready_deadline"`
+	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
+	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -108,10 +115,17 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	}
 
 	corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
+
+	remoteTransportDeadline, err := time.ParseDuration(env.Get(EnvAPIRemoteTransportDeadline, kvs.Get(apiRemoteTransportDeadline)))
+	if err != nil {
+		return cfg, err
+	}
+
 	return Config{
-		APIRequestsMax:      requestsMax,
-		APIRequestsDeadline: requestsDeadline,
-		APIReadyDeadline:    readyDeadline,
-		APICorsAllowOrigin:  corsAllowOrigin,
+		RequestsMax:             requestsMax,
+		RequestsDeadline:        requestsDeadline,
+		ReadyDeadline:           readyDeadline,
+		CorsAllowOrigin:         corsAllowOrigin,
+		RemoteTransportDeadline: remoteTransportDeadline,
 	}, nil
 }

cmd/config/api/help.go

@@ -39,5 +39,11 @@ var (
 			Optional:    true,
 			Type:        "csv",
 		},
+		config.HelpKV{
+			Key:         apiRemoteTransportDeadline,
+			Description: `set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"`,
+			Optional:    true,
+			Type:        "duration",
+		},
 	}
 )

cmd/config/config.go

@@ -76,6 +76,7 @@ const (
 	KmsKesSubSys         = "kms_kes"
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
+	CrawlerSubSys        = "crawler"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -112,6 +113,7 @@ var SubSystems = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	CrawlerSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
 	NotifyKafkaSubSys,
@@ -138,6 +140,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	CrawlerSubSys,
 }...)
 
 // Constant separators

cmd/config/crawler/crawler.go

@@ -0,0 +1,67 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawler
+
+import (
+	"errors"
+
+	"github.com/minio/minio/cmd/config"
+)
+
+// Compression environment variables
+const (
+	BitrotScan = "bitrotscan"
+)
+
+// Config represents the crawler settings.
+type Config struct {
+	// Bitrot will perform bitrot scan on local disk when checking objects.
+	Bitrot bool `json:"bitrotscan"`
+}
+
+var (
+	// DefaultKVS - default KV config for crawler settings
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   BitrotScan,
+			Value: config.EnableOff,
+		},
+	}
+
+	// Help provides help for config values
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         BitrotScan,
+			Description: `perform bitrot scan on disks when checking objects during crawl`,
+			Optional:    true,
+			Type:        "on|off",
+		},
+	}
+)
+
+// LookupConfig - lookup config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+	bitrot := kvs.Get(BitrotScan)
+	if bitrot != config.EnableOn && bitrot != config.EnableOff {
+		return cfg, errors.New(BitrotScan + ": must be 'on' or 'off'")
+	}
+	cfg.Bitrot = bitrot == config.EnableOn
+	return cfg, nil
+}

cmd/consolelogger.go

@@ -122,6 +122,24 @@ func (sys *HTTPConsoleLoggerSys) Validate() error {
 	return nil
 }
 
+// Content returns the console stdout log
+func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
+	sys.RLock()
+	sys.logBuf.Do(func(p interface{}) {
+		if p != nil {
+			lg, ok := p.(log.Info)
+			if ok {
+				if (lg.Entry != log.Entry{}) {
+					logs = append(logs, lg.Entry)
+				}
+			}
+		}
+	})
+	sys.RUnlock()
+
+	return
+}
+
 // Send log message 'e' to console and publish to console
 // log pubsub system
 func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {

cmd/data-crawler.go

@@ -27,9 +27,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio/pkg/madmin"
-
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
@@ -37,6 +36,7 @@ import (
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/madmin"
 	"github.com/willf/bloom"
 )
 
@@ -46,7 +46,14 @@ const (
 	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
 	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
 
-	healDeleteDangling = true
+	healDeleteDangling    = true
+	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
+	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
+
+)
+
+var (
+	globalCrawlerConfig crawler.Config
 )
 
 // initDataCrawler will start the crawler unless disabled.
@@ -174,9 +181,9 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	// Enable healing in XL mode.
 	if globalIsErasure {
 		// Include a clean folder one in n cycles.
-		s.healFolderInclude = 32
+		s.healFolderInclude = healFolderIncludeProb
 		// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
-		s.healObjectSelect = 512
+		s.healObjectSelect = healObjectSelectProb
 	}
 	if len(cache.Info.BloomFilter) > 0 {
 		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
@@ -624,7 +631,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return 0
 		}
-		if !errors.Is(err, NotImplemented{}) {
+		if err != nil && !errors.Is(err, NotImplemented{}) {
 			logger.LogIf(ctx, err)
 			return 0
 		}

cmd/data-update-tracker.go

@@ -426,6 +426,8 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 	}
 	// Ignore what remains on the stream.
 	// Update d:
+	d.mu.Lock()
+	defer d.mu.Unlock()
 	d.Current = dst.Current
 	d.History = dst.History
 	d.Saved = dst.Saved

cmd/erasure-bucket.go

@@ -243,7 +243,10 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 	// If we reduce quorum to nil, means we have deleted buckets properly
 	// on some servers in quorum, we should look for volumeNotEmpty errors
 	// and delete those buckets as well.
-	deleteDanglingBucket(ctx, storageDisks, dErrs, bucket)
+	//
+	// let this call succeed, even if client cancels the context
+	// this is to ensure that we don't leave any stale content
+	deleteDanglingBucket(context.Background(), storageDisks, dErrs, bucket)
 
 	return nil
 }

cmd/erasure-metadata-utils.go

@@ -119,7 +119,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 	metadataArray := make([]FileInfo, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
-	// Read `xl.meta` parallelly across disks.
+	// Read `xl.meta` in parallel across disks.
 	for index := range disks {
 		index := index
 		g.Go(func() (err error) {

cmd/erasure-multipart.go

@@ -346,7 +346,18 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 // of the multipart transaction.
 //
 // Implements S3 compatible Upload Part API.
-func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, e error) {
+func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return PartInfo{}, err
+	}
+	readLocked := true
+	defer func() {
+		if readLocked {
+			uploadIDLock.RUnlock()
+		}
+	}()
+
 	data := r.Reader
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
@@ -359,7 +370,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 
 	// Validates if upload ID exists.
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -446,8 +457,17 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		}
 	}
 
+	// Unlock here before acquiring write locks all concurrent
+	// PutObjectParts would serialize here updating `xl.meta`
+	uploadIDLock.RUnlock()
+	readLocked = false
+	if err = uploadIDLock.GetLock(globalOperationTimeout); err != nil {
+		return PartInfo{}, err
+	}
+	defer uploadIDLock.Unlock()
+
 	// Validates if upload ID exists.
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -522,6 +542,12 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return MultipartInfo{}, err
+	}
+	defer uploadIDLock.RUnlock()
+
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
@@ -564,6 +590,12 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return ListPartsInfo{}, err
+	}
+	defer uploadIDLock.RUnlock()
+
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
@@ -648,8 +680,16 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 // md5sums of all the parts.
 //
 // Implements S3 compatible Complete multipart API.
-func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, e error) {
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
+	// Hold read-locks to verify uploaded parts, also disallows
+	// parallel part uploads as well.
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return oi, err
+	}
+	defer uploadIDLock.RUnlock()
+
+	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -797,6 +837,13 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	// Hold namespace to complete the transaction
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err = lk.GetLock(globalOperationTimeout); err != nil {
+		return oi, err
+	}
+	defer lk.Unlock()
+
 	// Rename the multipart object to final location.
 	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
 		fi.DataDir, bucket, object, writeQuorum, nil); err != nil {
@@ -832,11 +879,13 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // All parts are purged from all disks and reference to the uploadID
 // would be removed from the system, rollback is not possible on this
 // operation.
-//
-// Implements S3 compatible Abort multipart API, slight difference is
-// that this is an atomic idempotent operation. Subsequent calls have
-// no affect and further requests to the same uploadID would not be honored.
-func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
+func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
+	lk := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer lk.Unlock()
+
 	// Validates if upload ID exists.
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return toObjectErr(err, bucket, object, uploadID)

cmd/erasure-object.go

@@ -25,6 +25,7 @@ import (
 	"net/http"
 	"path"
 	"sync"
+	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -71,6 +72,11 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, NotImplemented{}
 	}
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
+	lk := er.NewNSLock(ctx, dstBucket, dstObject)
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
+		return oi, err
+	}
+	defer lk.Unlock()
 
 	// Read metadata associated with the object from all disks.
 	storageDisks := er.getDisks()
@@ -111,18 +117,19 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	fi.VersionID = versionID // set any new versionID we might have created
 	fi.ModTime = modTime     // set modTime for the new versionID
 
+	srcInfo.UserDefined["etag"] = srcInfo.ETag
+
 	// Update `xl.meta` content on each disks.
 	for index := range metaArr {
 		metaArr[index].ModTime = modTime
 		metaArr[index].VersionID = versionID
 		metaArr[index].Metadata = srcInfo.UserDefined
-		metaArr[index].Metadata["etag"] = srcInfo.ETag
 	}
 
 	tempObj := mustGetUUID()
 
 	// Cleanup in case of xl.meta writing failure
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tempObj, writeQuorum)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
 
 	// Write unique `xl.meta` for each disk.
 	if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
@@ -144,6 +151,32 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return nil, err
 	}
 
+	var unlockOnDefer bool
+	var nsUnlocker = func() {}
+	defer func() {
+		if unlockOnDefer {
+			nsUnlocker()
+		}
+	}()
+
+	// Acquire lock
+	if lockType != noLock {
+		lock := er.NewNSLock(ctx, bucket, object)
+		switch lockType {
+		case writeLock:
+			if err = lock.GetLock(globalOperationTimeout); err != nil {
+				return nil, err
+			}
+			nsUnlocker = lock.Unlock
+		case readLock:
+			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+				return nil, err
+			}
+			nsUnlocker = lock.RUnlock
+		}
+		unlockOnDefer = true
+	}
+
 	// Handler directory request by returning a reader that
 	// returns no bytes.
 	if HasSuffix(object, SlashSeparator) {
@@ -151,7 +184,8 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		if objInfo, err = er.getObjectInfoDir(ctx, bucket, object); err != nil {
 			return nil, toObjectErr(err, bucket, object)
 		}
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts)
+		unlockOnDefer = false
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts, nsUnlocker)
 	}
 
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
@@ -172,7 +206,8 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		}, toObjectErr(errMethodNotAllowed, bucket, object)
 	}
 
-	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts)
+	unlockOnDefer = false
+	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts, nsUnlocker)
 	if nErr != nil {
 		return nil, nErr
 	}
@@ -201,6 +236,13 @@ func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, s
 		return err
 	}
 
+	// Lock the object before reading.
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer lk.RUnlock()
+
 	// Start offset cannot be negative.
 	if startOffset < 0 {
 		logger.LogIf(ctx, errUnexpected, logger.Application)
@@ -385,6 +427,13 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 		return info, err
 	}
 
+	// Lock the object before reading.
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.RUnlock()
+
 	if HasSuffix(object, SlashSeparator) {
 		info, err = er.getObjectInfoDir(ctx, bucket, object)
 		if err != nil {
@@ -599,7 +648,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// Delete temporary object in the event of failure.
 	// If PutObject succeeded there would be no temporary
 	// object to delete.
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tempObj, writeQuorum)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
 
 	// This is a special case with size as '0' and object ends with
 	// a slash separator, we treat it like a valid operation and
@@ -698,6 +747,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.Unlock()
+
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -818,7 +873,9 @@ func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			err := cleanupDir(ctx, disks[index], minioMetaTmpBucket, tmpObj)
+			tctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			defer cancel()
+			err := cleanupDir(tctx, disks[index], minioMetaTmpBucket, tmpObj)
 			if err != nil && err != errVolumeNotFound {
 				return err
 			}
@@ -967,6 +1024,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return objInfo, err
 	}
 
+	// Acquire a write lock before deleting the object.
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err = lk.GetLock(globalOperationTimeout); err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.Unlock()
+
 	storageDisks := er.getDisks()
 	writeQuorum := len(storageDisks)/2 + 1
 

cmd/erasure-sets.go

@@ -762,9 +762,13 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 
 	// Check if this request is only metadata update.
 	if cpSrcDstSame && srcInfo.metadataOnly {
+		// Version ID is set for the destination and source == destination version ID.
+		// perform an in-place update.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		// Destination is not versioned and source version ID is empty
+		// perform an in-place update.
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
@@ -1070,8 +1074,8 @@ func (s *erasureSets) ListObjectParts(ctx context.Context, bucket, object, uploa
 }
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
-func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
-	return s.getHashedSet(object).AbortMultipartUpload(ctx, bucket, object, uploadID)
+func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
+	return s.getHashedSet(object).AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 }
 
 // CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name.

cmd/erasure-zones.go

@@ -459,38 +459,16 @@ func (z *erasureZones) MakeBucketWithLocation(ctx context.Context, bucket string
 }
 
 func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	var nsUnlocker = func() {}
-
-	// Acquire lock
-	if lockType != noLock {
-		lock := z.NewNSLock(ctx, bucket, object)
-		switch lockType {
-		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
-				return nil, err
-			}
-			nsUnlocker = lock.Unlock
-		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
-				return nil, err
-			}
-			nsUnlocker = lock.RUnlock
-		}
-	}
-
 	for _, zone := range z.zones {
 		gr, err = zone.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
 			}
-			nsUnlocker()
 			return gr, err
 		}
-		gr.cleanUpFns = append(gr.cleanUpFns, nsUnlocker)
 		return gr, nil
 	}
-	nsUnlocker()
 	if opts.VersionID != "" {
 		return gr, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}
@@ -498,17 +476,6 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
 }
 
 func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
-	// Lock the object before reading.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer lk.RUnlock()
-
-	if z.SingleZone() {
-		return z.zones[0].GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
-	}
-
 	for _, zone := range z.zones {
 		if err := zone.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
@@ -518,20 +485,13 @@ func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, sta
 		}
 		return nil
 	}
+	if opts.VersionID != "" {
+		return VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
+	}
 	return ObjectNotFound{Bucket: bucket, Object: object}
 }
 
 func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Lock the object before reading.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
-	}
-	defer lk.RUnlock()
-
-	if z.SingleZone() {
-		return z.zones[0].GetObjectInfo(ctx, bucket, object, opts)
-	}
 	for _, zone := range z.zones {
 		objInfo, err = zone.GetObjectInfo(ctx, bucket, object, opts)
 		if err != nil {
@@ -550,13 +510,6 @@ func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // PutObject - writes an object to least used erasure zone.
 func (z *erasureZones) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
-	// Lock the object.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
-	}
-	defer lk.Unlock()
-
 	if z.SingleZone() {
 		return z.zones[0].PutObject(ctx, bucket, object, data, opts)
 	}
@@ -571,13 +524,6 @@ func (z *erasureZones) PutObject(ctx context.Context, bucket string, object stri
 }
 
 func (z *erasureZones) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Acquire a write lock before deleting the object.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
-	}
-	defer lk.Unlock()
-
 	if z.SingleZone() {
 		return z.zones[0].DeleteObject(ctx, bucket, object, opts)
 	}
@@ -629,15 +575,7 @@ func (z *erasureZones) DeleteObjects(ctx context.Context, bucket string, objects
 }
 
 func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Check if this request is only metadata update.
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
-	if !cpSrcDstSame {
-		lk := z.NewNSLock(ctx, dstBucket, dstObject)
-		if err := lk.GetLock(globalOperationTimeout); err != nil {
-			return objInfo, err
-		}
-		defer lk.Unlock()
-	}
 
 	zoneIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
 	if err != nil {
@@ -645,12 +583,19 @@ func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dst
 	}
 
 	if cpSrcDstSame && srcInfo.metadataOnly {
+		// Version ID is set for the destination and source == destination version ID.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
 			return z.zones[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		// Destination is not versioned and source version ID is empty
+		// perform an in-place update.
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
 			return z.zones[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		// Destination is versioned, source is not destination version,
+		// as a special case look for if the source object is not legacy
+		// from older format, for older format we will rewrite them as
+		// newer using PutObject() - this is an optimization to save space
 		if dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID && !srcInfo.Legacy {
 			// CopyObject optimization where we don't create an entire copy
 			// of the content, instead we add a reference.
@@ -692,8 +637,8 @@ func (z *erasureZones) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 
 func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
 
-	var zonesEntryChs [][]FileInfoCh
-	var zonesListTolerancePerSet []int
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 
 	endWalkCh := make(chan struct{})
 	defer close(endWalkCh)
@@ -708,8 +653,8 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix,
 	var eof bool
 	var prevPrefix string
 
-	var zonesEntriesInfos [][]FileInfo
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfo, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
@@ -811,9 +756,9 @@ func (z *erasureZones) listObjectsSplunk(ctx context.Context, bucket, prefix, ma
 
 	recursive := true
 
-	var zonesEntryChs [][]FileInfoCh
-	var zonesEndWalkCh []chan struct{}
-	var zonesListTolerancePerSet []int
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zonesEndWalkCh := make([]chan struct{}, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.poolSplunk.Release(listParams{bucket, recursive, marker, prefix})
@@ -903,9 +848,9 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 		recursive = false
 	}
 
-	var zonesEntryChs [][]FileInfoCh
-	var zonesEndWalkCh []chan struct{}
-	var zonesListTolerancePerSet []int
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zonesEndWalkCh := make([]chan struct{}, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.pool.Release(listParams{bucket, recursive, marker, prefix})
@@ -1106,8 +1051,8 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 // mergeZonesEntriesVersionsCh - merges FileInfoVersions channel to entries upto maxKeys.
 func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys int, zonesListTolerancePerSet []int) (entries FilesInfoVersions) {
 	var i = 0
-	var zonesEntriesInfos [][]FileInfoVersions
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfoVersions, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfoVersions, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
@@ -1137,8 +1082,8 @@ func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys i
 // mergeZonesEntriesCh - merges FileInfo channel to entries upto maxKeys.
 func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, zonesListTolerancePerSet []int) (entries FilesInfo) {
 	var i = 0
-	var zonesEntriesInfos [][]FileInfo
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfo, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
@@ -1273,9 +1218,9 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m
 		recursive = false
 	}
 
-	var zonesEntryChs [][]FileInfoVersionsCh
-	var zonesEndWalkCh []chan struct{}
-	var zonesListTolerancePerSet []int
+	zonesEntryChs := make([][]FileInfoVersionsCh, 0, len(z.zones))
+	zonesEndWalkCh := make([]chan struct{}, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.poolVersions.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
@@ -1384,12 +1329,6 @@ func (z *erasureZones) PutObjectPart(ctx context.Context, bucket, object, upload
 		return PartInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
-		return PartInfo{}, err
-	}
-	defer uploadIDLock.Unlock()
-
 	if z.SingleZone() {
 		return z.zones[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}
@@ -1420,12 +1359,6 @@ func (z *erasureZones) GetMultipartInfo(ctx context.Context, bucket, object, upl
 		return MultipartInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
-		return MultipartInfo{}, err
-	}
-	defer uploadIDLock.RUnlock()
-
 	if z.SingleZone() {
 		return z.zones[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
@@ -1456,12 +1389,6 @@ func (z *erasureZones) ListObjectParts(ctx context.Context, bucket, object, uplo
 		return ListPartsInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
-		return ListPartsInfo{}, err
-	}
-	defer uploadIDLock.RUnlock()
-
 	if z.SingleZone() {
 		return z.zones[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
@@ -1484,25 +1411,19 @@ func (z *erasureZones) ListObjectParts(ctx context.Context, bucket, object, uplo
 }
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
-func (z *erasureZones) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
+func (z *erasureZones) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
 	if err := checkAbortMultipartArgs(ctx, bucket, object, z); err != nil {
 		return err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer uploadIDLock.Unlock()
-
 	if z.SingleZone() {
-		return z.zones[0].AbortMultipartUpload(ctx, bucket, object, uploadID)
+		return z.zones[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
 
 	for _, zone := range z.zones {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, ObjectOptions{})
+		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.AbortMultipartUpload(ctx, bucket, object, uploadID)
+			return zone.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
@@ -1524,22 +1445,6 @@ func (z *erasureZones) CompleteMultipartUpload(ctx context.Context, bucket, obje
 		return objInfo, err
 	}
 
-	// Hold read-locks to verify uploaded parts, also disallows
-	// parallel part uploads as well.
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
-		return objInfo, err
-	}
-	defer uploadIDLock.RUnlock()
-
-	// Hold namespace to complete the transaction, only hold
-	// if uploadID can be held exclusively.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
-		return objInfo, err
-	}
-	defer lk.Unlock()
-
 	if z.SingleZone() {
 		return z.zones[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
@@ -1832,18 +1737,15 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 		return nil
 	}
 
-	var zonesEntryChs [][]FileInfoCh
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zoneDrivesPerSet := make([]int, 0, len(z.zones))
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs, zone.startMergeWalks(ctx, bucket, prefix, "", true, ctx.Done()))
-	}
-
-	var zoneDrivesPerSet []int
-	for _, zone := range z.zones {
 		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 	}
 
-	var zonesEntriesInfos [][]FileInfo
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfo, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
@@ -1874,23 +1776,20 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 type HealObjectFn func(bucket, object, versionID string) error
 
 func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject HealObjectFn) error {
-	var zonesEntryChs [][]FileInfoVersionsCh
-
 	endWalkCh := make(chan struct{})
 	defer close(endWalkCh)
 
+	zonesEntryChs := make([][]FileInfoVersionsCh, 0, len(z.zones))
+	zoneDrivesPerSet := make([]int, 0, len(z.zones))
+
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs,
 			zone.startMergeWalksVersions(ctx, bucket, prefix, "", true, endWalkCh))
-	}
-
-	var zoneDrivesPerSet []int
-	for _, zone := range z.zones {
 		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 	}
 
-	var zonesEntriesInfos [][]FileInfoVersions
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfoVersions, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfoVersions, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))

cmd/fs-v1-metadata.go

@@ -225,7 +225,6 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
 	}
 
 	if len(fsMetaBuf) == 0 {
-		logger.LogIf(ctx, io.EOF)
 		return 0, io.EOF
 	}
 

cmd/fs-v1-multipart.go

@@ -779,7 +779,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 // that this is an atomic idempotent operation. Subsequent calls have
 // no affect and further requests to the same uploadID would not be
 // honored.
-func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
+func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
 	if err := checkAbortMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return err
 	}

cmd/fs-v1-multipart_test.go

@@ -61,7 +61,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	cleanupWg.Wait()
 
 	// Check if upload id was already purged.
-	if err = obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID); err != nil {
+	if err = obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, ObjectOptions{}); err != nil {
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
@@ -215,11 +215,12 @@ func TestAbortMultipartUpload(t *testing.T) {
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	opts := ObjectOptions{}
+	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), opts); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
-	if err := obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID); err != nil {
+	if err := obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, opts); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }

cmd/fs-v1.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -502,13 +503,13 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 		atomic.AddInt64(&fs.activeIOCount, -1)
 	}()
 
-	var bucketInfos []BucketInfo
 	entries, err := readDir(fs.fsPath)
 	if err != nil {
 		logger.LogIf(ctx, errDiskNotFound)
 		return nil, toObjectErr(errDiskNotFound)
 	}
 
+	bucketInfos := make([]BucketInfo, 0, len(entries))
 	for _, entry := range entries {
 		// Ignore all reserved bucket names and invalid bucket names.
 		if isReservedOrInvalidBucket(entry, false) {
@@ -622,8 +623,15 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, srcObject, fs.metaJSONFile)
 		wlk, err := fs.rwPool.Write(fsMetaPath)
 		if err != nil {
-			logger.LogIf(ctx, err)
-			return oi, toObjectErr(err, srcBucket, srcObject)
+			if !errors.Is(err, errFileNotFound) {
+				logger.LogIf(ctx, err)
+				return oi, toObjectErr(err, srcBucket, srcObject)
+			}
+			wlk, err = fs.rwPool.Create(fsMetaPath)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return oi, toObjectErr(err, srcBucket, srcObject)
+			}
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
@@ -1491,8 +1499,15 @@ func (fs *FSObjects) PutObjectTags(ctx context.Context, bucket, object string, t
 	fsMeta := fsMetaV1{}
 	wlk, err := fs.rwPool.Write(fsMetaPath)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return toObjectErr(err, bucket, object)
+		if !errors.Is(err, errFileNotFound) {
+			logger.LogIf(ctx, err)
+			return toObjectErr(err, bucket, object)
+		}
+		wlk, err = fs.rwPool.Create(fsMetaPath)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return toObjectErr(err, bucket, object)
+		}
 	}
 	// This close will allow for locks to be synchronized on `fs.json`.
 	defer wlk.Close()

cmd/gateway-common.go

@@ -349,16 +349,22 @@ func ComputeCompleteMultipartMD5(parts []CompletePart) string {
 // parse gateway sse env variable
 func parseGatewaySSE(s string) (gatewaySSE, error) {
 	l := strings.Split(s, ";")
-	var gwSlice = make([]string, 0)
+	var gwSlice gatewaySSE
 	for _, val := range l {
 		v := strings.ToUpper(val)
-		if v == gatewaySSES3 || v == gatewaySSEC {
+		switch v {
+		case "":
+			continue
+		case gatewaySSES3:
+			fallthrough
+		case gatewaySSEC:
 			gwSlice = append(gwSlice, v)
 			continue
+		default:
+			return nil, config.ErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
 		}
-		return nil, config.ErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
 	}
-	return gatewaySSE(gwSlice), nil
+	return gwSlice, nil
 }
 
 // handle gateway env vars
@@ -372,7 +378,7 @@ func gatewayHandleEnvVars() {
 	}
 
 	gwsseVal := env.Get("MINIO_GATEWAY_SSE", "")
-	if len(gwsseVal) != 0 {
+	if gwsseVal != "" {
 		var err error
 		GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal)
 		if err != nil {

cmd/gateway-main.go

@@ -168,6 +168,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
 	}
 
+	// Initialize globalConsoleSys system
+	globalConsoleSys = NewConsoleLogger(GlobalContext)
+	logger.AddTarget(globalConsoleSys)
+
 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
 
@@ -261,12 +265,8 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(registerWebRouter(router), "Unable to configure web browser")
 	}
 
-	// Currently only NAS and S3 gateway support encryption headers.
-	encryptionEnabled := gatewayName == S3BackendGateway || gatewayName == NASBackendGateway
-	allowSSEKMS := gatewayName == S3BackendGateway // Only S3 can support SSE-KMS (as pass-through)
-
 	// Add API router.
-	registerAPIRouter(router, encryptionEnabled, allowSSEKMS)
+	registerAPIRouter(router)
 
 	// Use all the middlewares
 	router.Use(registerMiddlewares)

cmd/gateway-unsupported.go

@@ -91,7 +91,7 @@ func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string,
 }
 
 // AbortMultipartUpload aborts a ongoing multipart upload
-func (a GatewayUnsupported) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
+func (a GatewayUnsupported) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, opts ObjectOptions) error {
 	return NotImplemented{}
 }
 

cmd/gateway/azure/gateway-azure.go

@@ -70,7 +70,6 @@ const (
 	azureChunkSizeEnvVar        = "MINIO_AZURE_CHUNK_SIZE_MB"
 
 	azureDownloadRetryAttempts = 5
-	azureBlockSize             = 100 * humanize.MiByte
 	azureS3MinPartSize         = 5 * humanize.MiByte
 	metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
 	azureMarkerPrefix          = "{minio}"
@@ -880,20 +879,15 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 // uses Azure equivalent `UploadStreamToBlockBlob`.
 func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
-
-	if data.Size() > azureBlockSize/2 {
-		if opts.UserDefined == nil {
-			opts.UserDefined = map[string]string{}
-		}
-
-		// Save md5sum for future processing on the object.
-		opts.UserDefined["x-amz-meta-md5sum"] = r.MD5CurrentHexString()
+	if opts.UserDefined == nil {
+		opts.UserDefined = map[string]string{}
 	}
 
 	metadata, properties, err := s3MetaToAzureProperties(ctx, opts.UserDefined)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
+
 	blobURL := a.client.NewContainerURL(bucket).NewBlockBlobURL(object)
 
 	_, err = azblob.UploadStreamToBlockBlob(ctx, data, blobURL, azblob.UploadStreamToBlockBlobOptions{
@@ -905,7 +899,18 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-
+	// Query the blob's properties and metadata
+	get, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
+	if err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+	// Update the blob's metadata with Content-MD5 after the upload
+	metadata = get.NewMetadata()
+	metadata["md5sum"] = r.MD5CurrentHexString()
+	_, err = blobURL.SetMetadata(ctx, metadata, azblob.BlobAccessConditions{})
+	if err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
 	return a.GetObjectInfo(ctx, bucket, object, opts)
 }
 
@@ -929,6 +934,7 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des
 		return objInfo, azureToObjectError(err, srcBucket, srcObject)
 	}
 	props.ContentMD5 = srcProps.ContentMD5()
+	azureMeta["md5sum"] = srcInfo.ETag
 	res, err := destBlob.StartCopyFromURL(ctx, srcBlobURL, azureMeta, azblob.ModifiedAccessConditions{}, azblob.BlobAccessConditions{})
 	if err != nil {
 		return objInfo, azureToObjectError(err, srcBucket, srcObject)
@@ -1238,7 +1244,7 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo
 // AbortMultipartUpload - Not Implemented.
 // There is no corresponding API in azure to abort an incomplete upload. The uncommmitted blocks
 // gets deleted after one week.
-func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) {
+func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts minio.ObjectOptions) (err error) {
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return err
 	}

cmd/gateway/gcs/gateway-gcs.go

@@ -1253,7 +1253,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
 }
 
 // AbortMultipartUpload aborts a ongoing multipart upload
-func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, key string, uploadID string) error {
+func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, opts minio.ObjectOptions) error {
 	if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
 		return err
 	}

cmd/gateway/hdfs/gateway-hdfs.go

@@ -859,7 +859,7 @@ func (n *hdfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, objec
 	}, nil
 }
 
-func (n *hdfsObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) {
+func (n *hdfsObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts minio.ObjectOptions) (err error) {
 	_, err = n.clnt.Stat(n.hdfsPathJoin(bucket))
 	if err != nil {
 		return hdfsToObjectErr(ctx, err, bucket)

cmd/gateway/s3/gateway-s3-sse.go

@@ -316,7 +316,7 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	}
 	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, o)
 	if err != nil {
-		return nil, minio.ErrorRespToObjectError(err)
+		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 	if l.isGWEncrypted(ctx, bucket, object) {
 		object = getGWContentPath(object)
@@ -569,12 +569,12 @@ func (l *s3EncObjects) ListObjectParts(ctx context.Context, bucket string, objec
 }
 
 // AbortMultipartUpload aborts a ongoing multipart upload
-func (l *s3EncObjects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
+func (l *s3EncObjects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, opts minio.ObjectOptions) error {
 	if _, err := l.getGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID)); err != nil {
-		return l.s3Objects.AbortMultipartUpload(ctx, bucket, object, uploadID)
+		return l.s3Objects.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
 
-	if err := l.s3Objects.AbortMultipartUpload(ctx, bucket, getGWContentPath(object), uploadID); err != nil {
+	if err := l.s3Objects.AbortMultipartUpload(ctx, bucket, getGWContentPath(object), uploadID, opts); err != nil {
 		return err
 	}
 

cmd/gateway/s3/gateway-s3.go

@@ -391,21 +391,21 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
-	var startOffset, length int64
-	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, opts)
 	if err != nil {
 		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	pr, pw := io.Pipe()
 	go func() {
-		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		err := l.GetObject(ctx, bucket, object, off, length, pw, objInfo.ETag, opts)
 		pw.CloseWithError(err)
 	}()
+
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts, pipeCloser)
+	return fn(pr, h, opts.CheckPrecondFn, pipeCloser)
 }
 
 // GetObject reads an object from S3. Supports additional
@@ -653,7 +653,7 @@ func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object s
 }
 
 // AbortMultipartUpload aborts a ongoing multipart upload
-func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
+func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, opts minio.ObjectOptions) error {
 	err := l.Client.AbortMultipartUpload(ctx, bucket, object, uploadID)
 	return minio.ErrorRespToObjectError(err, bucket, object)
 }
@@ -745,7 +745,7 @@ func (l *s3Objects) IsCompressionSupported() bool {
 
 // IsEncryptionSupported returns whether server side encryption is implemented for this layer.
 func (l *s3Objects) IsEncryptionSupported() bool {
-	return minio.GlobalKMS != nil || len(minio.GlobalGatewaySSE) > 0
+	return minio.GlobalKMS != nil || minio.GlobalGatewaySSE.IsSet()
 }
 
 func (l *s3Objects) IsTaggingSupported() bool {

cmd/handler-api.go

@@ -38,10 +38,11 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	t.readyDeadline = cfg.APIReadyDeadline
-	t.corsAllowOrigins = cfg.APICorsAllowOrigin
+	t.readyDeadline = cfg.ReadyDeadline
+	t.corsAllowOrigins = cfg.CorsAllowOrigin
+
 	var apiRequestsMaxPerNode int
-	if cfg.APIRequestsMax <= 0 {
+	if cfg.RequestsMax <= 0 {
 		stats, err := sys.GetStats()
 		if err != nil {
 			return
@@ -51,21 +52,23 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
 		// ram_per_request is 4MiB * setDriveCount + 2 * 10MiB (default erasure block size)
 		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*readBlockSize+blockSizeV1*2))
 	} else {
-		apiRequestsMaxPerNode = cfg.APIRequestsMax
+		apiRequestsMaxPerNode = cfg.RequestsMax
 		if len(globalEndpoints.Hostnames()) > 0 {
 			apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames())
 		}
 	}
 
 	t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
-	t.requestsDeadline = cfg.APIRequestsDeadline
+	t.requestsDeadline = cfg.RequestsDeadline
 }
 
 func (t *apiConfig) getCorsAllowOrigins() []string {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
-	return t.corsAllowOrigins
+	corsAllowOrigins := make([]string, len(t.corsAllowOrigins))
+	copy(corsAllowOrigins, t.corsAllowOrigins)
+	return corsAllowOrigins
 }
 
 func (t *apiConfig) getReadyDeadline() time.Duration {

cmd/notification.go

@@ -1118,6 +1118,36 @@ func (sys *NotificationSys) ProcOBDInfo(ctx context.Context) []madmin.ServerProc
 	return reply
 }
 
+// LogOBDInfo - Logs OBD information
+func (sys *NotificationSys) LogOBDInfo(ctx context.Context) []madmin.ServerLogOBDInfo {
+	reply := make([]madmin.ServerLogOBDInfo, len(sys.peerClients))
+
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			var err error
+			reply[index], err = sys.peerClients[index].LogOBDInfo(ctx)
+			return err
+		}, index)
+	}
+
+	for index, err := range g.Wait() {
+		if err != nil {
+			addr := sys.peerClients[index].host.String()
+			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
+			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+			logger.LogIf(ctx, err)
+			reply[index].Addr = addr
+			reply[index].Error = err.Error()
+		}
+	}
+	return reply
+}
+
 // ServerInfo - calls ServerInfo RPC call on all peers.
 func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
 	reply := make([]madmin.ServerProperties, len(sys.peerClients))
@@ -1232,7 +1262,7 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 		},
 	}
 
-	if args.EventName != event.ObjectRemovedDelete {
+	if args.EventName != event.ObjectRemovedDelete && args.EventName != event.ObjectRemovedDeleteMarkerCreated {
 		newEvent.S3.Object.ETag = args.Object.ETag
 		newEvent.S3.Object.Size = args.Object.Size
 		newEvent.S3.Object.ContentType = args.Object.ContentType

cmd/obdinfo.go

@@ -148,6 +148,19 @@ func getLocalMemOBD(ctx context.Context, r *http.Request) madmin.ServerMemOBDInf
 	}
 }
 
+func getLocalLogOBD(ctx context.Context, r *http.Request) madmin.ServerLogOBDInfo {
+	addr := r.Host
+	if globalIsDistErasure {
+		addr = GetLocalPeer(globalEndpoints)
+	}
+
+	log := globalConsoleSys.Content()
+	return madmin.ServerLogOBDInfo{
+		Addr:    addr,
+		Entries: log,
+	}
+}
+
 func getLocalProcOBD(ctx context.Context, r *http.Request) madmin.ServerProcOBDInfo {
 	addr := r.Host
 	if globalIsDistErasure {

cmd/object-api-interface.go

@@ -110,7 +110,7 @@ type ObjectLayer interface {
 	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (info MultipartInfo, err error)
 	ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error)
-	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
+	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 
 	// Healing operations.

cmd/object-api-multipart_test.go

@@ -66,7 +66,7 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	err = obj.AbortMultipartUpload(context.Background(), bucket, "\\", uploadID)
+	err = obj.AbortMultipartUpload(context.Background(), bucket, "\\", uploadID, opts)
 	if err != nil {
 		switch err.(type) {
 		case InvalidUploadID:
@@ -114,7 +114,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 	}
 	// Iterating over creatPartCases to generate multipart chunks.
 	for i, testCase := range abortTestCases {
-		err = obj.AbortMultipartUpload(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID)
+		err = obj.AbortMultipartUpload(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, opts)
 		if testCase.expectedErrType == nil && err != nil {
 			t.Errorf("Test %d, unexpected err is received: %v, expected:%v\n", i+1, err, testCase.expectedErrType)
 		}
@@ -146,7 +146,8 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	err = obj.AbortMultipartUpload(context.Background(), bucket, object, "abc")
+	opts := ObjectOptions{}
+	err = obj.AbortMultipartUpload(context.Background(), bucket, object, "abc", opts)
 	switch err.(type) {
 	case InvalidUploadID:
 	default:
@@ -1565,7 +1566,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 	}
 
 	for i, testCase := range testCases {
-		actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, ObjectOptions{})
+		actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, opts)
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, actualErr.Error())
 		}
@@ -1801,6 +1802,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.(*testing.T).Run("", func(t *testing.T) {
+			opts = ObjectOptions{}
 			actualResult, actualErr := obj.CompleteMultipartUpload(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.parts, ObjectOptions{})
 			if actualErr != nil && testCase.shouldPass {
 				t.Errorf("%s: Expected to pass, but failed with: <ERROR> %s", instanceType, actualErr)

cmd/object-api-utils.go

@@ -877,19 +877,3 @@ func newS2CompressReader(r io.Reader) io.ReadCloser {
 	}()
 	return pr
 }
-
-// Returns error if the context is canceled, indicating
-// either client has disconnected
-type contextReader struct {
-	io.ReadCloser
-	ctx context.Context
-}
-
-func (d *contextReader) Read(p []byte) (int, error) {
-	select {
-	case <-d.ctx.Done():
-		return 0, d.ctx.Err()
-	default:
-		return d.ReadCloser.Read(p)
-	}
-}

cmd/object-handlers-common.go

@@ -287,31 +287,30 @@ func deleteObject(ctx context.Context, obj ObjectLayer, cache CacheObjectLayer,
 		deleteObject = cache.DeleteObject
 	}
 	// Proceed to delete the object.
-	if objInfo, err = deleteObject(ctx, bucket, object, opts); err != nil {
-		return objInfo, err
+	objInfo, err = deleteObject(ctx, bucket, object, opts)
+	if objInfo.Name != "" {
+		// Requesting only a delete marker which was successfully attempted.
+		if objInfo.DeleteMarker {
+			// Notify object deleted marker event.
+			sendEvent(eventArgs{
+				EventName:  event.ObjectRemovedDeleteMarkerCreated,
+				BucketName: bucket,
+				Object:     objInfo,
+				ReqParams:  extractReqParams(r),
+				UserAgent:  r.UserAgent(),
+				Host:       handlers.GetSourceIP(r),
+			})
+		} else {
+			// Notify object deleted event.
+			sendEvent(eventArgs{
+				EventName:  event.ObjectRemovedDelete,
+				BucketName: bucket,
+				Object:     objInfo,
+				ReqParams:  extractReqParams(r),
+				UserAgent:  r.UserAgent(),
+				Host:       handlers.GetSourceIP(r),
+			})
+		}
 	}
-
-	// Requesting only a delete marker which was successfully attempted.
-	if objInfo.DeleteMarker {
-		// Notify object deleted marker event.
-		sendEvent(eventArgs{
-			EventName:  event.ObjectRemovedDeleteMarkerCreated,
-			BucketName: bucket,
-			Object:     objInfo,
-			ReqParams:  extractReqParams(r),
-			UserAgent:  r.UserAgent(),
-			Host:       handlers.GetSourceIP(r),
-		})
-	} else {
-		// Notify object deleted event.
-		sendEvent(eventArgs{
-			EventName:  event.ObjectRemovedDelete,
-			BucketName: bucket,
-			Object:     objInfo,
-			ReqParams:  extractReqParams(r),
-			UserAgent:  r.UserAgent(),
-			Host:       handlers.GetSourceIP(r),
-		})
-	}
-	return objInfo, nil
+	return objInfo, err
 }

cmd/object-handlers.go

@@ -99,11 +99,13 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+
+	if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -189,7 +191,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 			End:            offset + length,
 		}
 
-		return getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, ObjectOptions{})
+		return getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	}
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
@@ -308,7 +310,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -507,7 +509,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -711,17 +713,17 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 }
 
 // getRemoteInstanceTransport contains a singleton roundtripper.
-var getRemoteInstanceTransport *http.Transport
-var getRemoteInstanceTransportLongTO *http.Transport
-var getRemoteInstanceTransportOnce sync.Once
+var (
+	getRemoteInstanceTransport     *http.Transport
+	getRemoteInstanceTransportOnce sync.Once
+)
 
 // Returns a minio-go Client configured to access remote host described by destDNSRecord
 // Applicable only in a federated deployment
 var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) {
-	getRemoteInstanceTransportOnce.Do(func() {
-		getRemoteInstanceTransport = NewGatewayHTTPTransport()
-		getRemoteInstanceTransportLongTO = newGatewayHTTPTransport(time.Hour)
-	})
+	if newObjectLayerFn() == nil {
+		return nil, errServerNotInitialized
+	}
 
 	cred := getReqAccessCred(r, globalServerRegion)
 	// In a federated deployment, all the instances share config files
@@ -737,29 +739,6 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core,
 	return core, nil
 }
 
-// Returns a minio-go Client configured to access remote host described by destDNSRecord
-// Applicable only in a federated deployment.
-// The transport does not contain any timeout except for dialing.
-func getRemoteInstanceClientLongTimeout(r *http.Request, host string) (*miniogo.Core, error) {
-	getRemoteInstanceTransportOnce.Do(func() {
-		getRemoteInstanceTransport = NewGatewayHTTPTransport()
-		getRemoteInstanceTransportLongTO = newGatewayHTTPTransport(time.Hour)
-	})
-
-	cred := getReqAccessCred(r, globalServerRegion)
-	// In a federated deployment, all the instances share config files
-	// and hence expected to have same credentials.
-	core, err := miniogo.NewCore(host, &miniogo.Options{
-		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, ""),
-		Secure:    globalIsSSL,
-		Transport: getRemoteInstanceTransportLongTO,
-	})
-	if err != nil {
-		return nil, err
-	}
-	return core, nil
-}
-
 // Check if the destination bucket is on a remote site, this code only gets executed
 // when federation is enabled, ie when globalDNSConfig is non 'nil'.
 //
@@ -804,11 +783,13 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
+
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -914,10 +895,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		getObjectNInfo = api.CacheAPI().GetObjectNInfo
 	}
 
-	var lock = noLock
-	if !cpSrcDstSame {
-		lock = readLock
-	}
 	checkCopyPrecondFn := func(o ObjectInfo) bool {
 		if objectAPI.IsEncryptionSupported() {
 			if _, err := DecryptObjectInfo(&o, r); err != nil {
@@ -929,6 +906,16 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 	getOpts.CheckPrecondFn = checkCopyPrecondFn
 
+	// FIXME: a possible race exists between a parallel
+	// GetObject v/s CopyObject with metadata updates, ideally
+	// we should be holding write lock here but it is not
+	// possible due to other constraints such as knowing
+	// the type of source content etc.
+	lock := noLock
+	if !cpSrcDstSame {
+		lock = readLock
+	}
+
 	var rs *HTTPRangeSpec
 	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, lock, getOpts)
 	if err != nil {
@@ -1222,7 +1209,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 
 		// Send PutObject request to appropriate instance (in federated deployment)
-		core, rerr := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(dstRecords))
+		core, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords))
 		if rerr != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL, guessIsBrowserReq(r))
 			return
@@ -1250,6 +1237,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if api.CacheAPI() != nil {
 			copyObjectFn = api.CacheAPI().CopyObject
 		}
+
 		// Copy source object to destination, if source and destination
 		// object is same then only metadata is updated.
 		objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
@@ -1263,7 +1251,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
 	if mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		defer replicateObject(GlobalContext, dstBucket, dstObject, objInfo.VersionID, objectAPI, &eventArgs{
+		go replicateObject(GlobalContext, dstBucket, dstObject, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedCopy,
 			BucketName:   dstBucket,
 			Object:       objInfo,
@@ -1313,14 +1301,17 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) && !api.AllowSSEKMS() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
-		return
-	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object, err := url.PathUnescape(vars["object"])
@@ -1329,9 +1320,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// To detect if the client has disconnected.
-	r.Body = &contextReader{r.Body, r.Context()}
-
 	// X-Amz-Copy-Source shouldn't be set for this call.
 	if _, ok := r.Header[xhttp.AmzCopySource]; ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
@@ -1448,7 +1436,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	// Check if bucket encryption is enabled
 	_, err = globalBucketSSEConfigSys.Get(bucket)
 	// This request header needs to be set prior to setting ObjectOptions
-	if (globalAutoEncryption || err == nil) && !crypto.SSEC.IsRequested(r.Header) && !crypto.S3KMS.IsRequested(r.Header) {
+	if (globalAutoEncryption || err == nil) && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
 
@@ -1579,7 +1567,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 	if mustReplicate(ctx, r, bucket, object, metadata, "") {
-		defer replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+		go replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedPut,
 			BucketName:   bucket,
 			Object:       objInfo,
@@ -1623,14 +1611,17 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) && !api.AllowSSEKMS() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
-		return
-	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object, err := url.PathUnescape(vars["object"])
@@ -1647,7 +1638,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	// Check if bucket encryption is enabled
 	_, err = globalBucketSSEConfigSys.Get(bucket)
 	// This request header needs to be set prior to setting ObjectOptions
-	if (globalAutoEncryption || err == nil) && !crypto.SSEC.IsRequested(r.Header) && !crypto.S3KMS.IsRequested(r.Header) {
+	if (globalAutoEncryption || err == nil) && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
 
@@ -1748,11 +1739,13 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
+
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1870,7 +1863,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return false
 	}
 	getOpts.CheckPrecondFn = checkCopyPartPrecondFn
-
 	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, readLock, getOpts)
 	if err != nil {
 		if isErrPreconditionFailed(err) {
@@ -1932,7 +1924,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		}
 
 		// Send PutObject request to appropriate instance (in federated deployment)
-		core, rerr := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(dstRecords))
+		core, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords))
 		if rerr != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL, guessIsBrowserReq(r))
 			return
@@ -2063,11 +2055,13 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
+
+	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -2079,9 +2073,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// To detect if the client has disconnected.
-	r.Body = &contextReader{r.Body, r.Context()}
-
 	// X-Amz-Copy-Source shouldn't be set for this call.
 	if _, ok := r.Header[xhttp.AmzCopySource]; ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
@@ -2331,7 +2322,8 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if err := abortMultipartUpload(ctx, bucket, object, uploadID); err != nil {
+	opts := ObjectOptions{}
+	if err := abortMultipartUpload(ctx, bucket, object, uploadID, opts); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -2378,7 +2370,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	var opts ObjectOptions
+	opts := ObjectOptions{}
 	listPartsInfo, err := objectAPI.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -2544,9 +2536,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	var objectEncryptionKey []byte
 	var isEncrypted, ssec bool
-	var opts ObjectOptions
 	if objectAPI.IsEncryptionSupported() {
-		mi, err := objectAPI.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+		mi, err := objectAPI.GetMultipartInfo(ctx, bucket, object, uploadID, ObjectOptions{})
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -2569,7 +2560,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	partsMap := make(map[string]PartInfo)
 	if isEncrypted {
 		maxParts := 10000
-		listPartsInfo, err := objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, maxParts, opts)
+		listPartsInfo, err := objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, maxParts, ObjectOptions{})
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
@@ -2580,7 +2571,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	// Complete parts.
-	var completeParts []CompletePart
+	completeParts := make([]CompletePart, 0, len(complMultipartUpload.Parts))
 	for _, part := range complMultipartUpload.Parts {
 		part.ETag = canonicalizeETag(part.ETag)
 		if isEncrypted {
@@ -2624,7 +2615,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	w = &whiteSpaceWriter{ResponseWriter: w, Flusher: w.(http.Flusher)}
 	completeDoneCh := sendWhiteSpace(w)
-	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, opts)
+	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, ObjectOptions{})
 	// Stop writing white spaces to the client. Note that close(doneCh) style is not used as it
 	// can cause white space to be written after we send XML response in a race condition.
 	headerWritten := <-completeDoneCh
@@ -2654,7 +2645,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	setPutObjHeaders(w, objInfo, false)
 	if mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		defer replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+		go replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedCompleteMultipartUpload,
 			BucketName:   bucket,
 			Object:       objInfo,
@@ -2758,16 +2749,6 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	setPutObjHeaders(w, objInfo, true)
 
 	writeSuccessNoContent(w)
-
-	sendEvent(eventArgs{
-		EventName:    event.ObjectRemovedDelete,
-		BucketName:   bucket,
-		Object:       objInfo,
-		ReqParams:    extractReqParams(r),
-		RespElements: extractRespElements(w),
-		UserAgent:    r.UserAgent(),
-		Host:         handlers.GetSourceIP(r),
-	})
 }
 
 // PutObjectLegalHoldHandler - set legal hold configuration to object,

cmd/object_api_suite_test.go

@@ -166,7 +166,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
 		}
 		parts[i] = expectedETaghex
 	}
-	err = obj.AbortMultipartUpload(context.Background(), "bucket", "key", uploadID)
+	err = obj.AbortMultipartUpload(context.Background(), "bucket", "key", uploadID, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}

cmd/os-readdir_unix.go

@@ -19,6 +19,7 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"sync"
@@ -42,14 +43,14 @@ var direntPool = sync.Pool{
 // value used to represent a syscall.DT_UNKNOWN Dirent.Type.
 const unexpectedFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice
 
-func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode, err error) {
+func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err error) {
 	// golang.org/issue/15653
 	dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0]))
 	if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v {
-		return consumed, name, typ, fmt.Errorf("buf size of %d smaller than dirent header size %d", len(buf), v)
+		return consumed, nil, typ, fmt.Errorf("buf size of %d smaller than dirent header size %d", len(buf), v)
 	}
 	if len(buf) < int(dirent.Reclen) {
-		return consumed, name, typ, fmt.Errorf("buf size %d < record length %d", len(buf), dirent.Reclen)
+		return consumed, nil, typ, fmt.Errorf("buf size %d < record length %d", len(buf), dirent.Reclen)
 	}
 	consumed = int(dirent.Reclen)
 	if direntInode(dirent) == 0 { // File absent in directory.
@@ -72,11 +73,10 @@ func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode, err er
 	nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))
 	nameLen, err := direntNamlen(dirent)
 	if err != nil {
-		return consumed, name, typ, err
+		return consumed, nil, typ, err
 	}
 
-	name = string(nameBuf[:nameLen])
-	return consumed, name, typ, nil
+	return consumed, nameBuf[:nameLen], typ, nil
 }
 
 // Return all the entries at the directory dirPath.
@@ -116,13 +116,13 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
 			return err
 		}
 		boff += consumed
-		if name == "" || name == "." || name == ".." {
+		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
 			continue
 		}
 		if typ&os.ModeSymlink == os.ModeSymlink {
 			continue
 		}
-		if err = filter(name, typ); err == errDoneForNow {
+		if err = filter(string(name), typ); err == errDoneForNow {
 			// filtering requested to return by caller.
 			return nil
 		}
@@ -143,6 +143,10 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	bufp := direntPool.Get().(*[]byte)
 	defer direntPool.Put(bufp)
 
+	nameTmp := direntPool.Get().(*[]byte)
+	defer direntPool.Put(nameTmp)
+	tmp := *nameTmp
+
 	boff := 0 // starting read position in buf
 	nbuf := 0 // end valid data in buf
 
@@ -165,14 +169,14 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			return nil, err
 		}
 		boff += consumed
-		if name == "" || name == "." || name == ".." {
+		if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) {
 			continue
 		}
 		// Fallback for filesystems (like old XFS) that don't
 		// support Dirent.Type and have DT_UNKNOWN (0) there
 		// instead.
 		if typ == unexpectedFileMode {
-			fi, err := os.Lstat(pathJoin(dirPath, name))
+			fi, err := os.Lstat(pathJoin(dirPath, string(name)))
 			if err != nil {
 				// It got deleted in the meantime, not found
 				// or returns too many symlinks ignore this
@@ -189,9 +193,13 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			continue
 		}
 		if typ.IsRegular() {
-			entries = append(entries, name)
+			entries = append(entries, string(name))
 		} else if typ.IsDir() {
-			entries = append(entries, name+SlashSeparator)
+			// Use temp buffer to append a slash to avoid string concat.
+			tmp = tmp[:len(name)+1]
+			copy(tmp, name)
+			tmp[len(tmp)-1] = '/' // SlashSeparator
+			entries = append(entries, string(tmp))
 		}
 		count--
 	}

cmd/peer-rest-client.go

@@ -420,6 +420,17 @@ func (client *peerRESTClient) ProcOBDInfo(ctx context.Context) (info madmin.Serv
 	return info, err
 }
 
+// LogOBDInfo - fetch Log OBD information for a remote node.
+func (client *peerRESTClient) LogOBDInfo(ctx context.Context) (info madmin.ServerLogOBDInfo, err error) {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodLogOBDInfo, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
 // StartProfiling - Issues profiling command on the peer node.
 func (client *peerRESTClient) StartProfiling(profiler string) error {
 	values := make(url.Values)
@@ -829,8 +840,9 @@ func (client *peerRESTClient) ConsoleLog(logCh chan interface{}, doneCh <-chan s
 }
 
 func getRemoteHosts(endpointZones EndpointZones) []*xnet.Host {
-	var remoteHosts []*xnet.Host
-	for _, hostStr := range GetRemotePeers(endpointZones) {
+	peers := GetRemotePeers(endpointZones)
+	remoteHosts := make([]*xnet.Host, 0, len(peers))
+	for _, hostStr := range peers {
 		host, err := xnet.ParseHost(hostStr)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)

cmd/peer-rest-common.go

@@ -33,6 +33,7 @@ const (
 	peerRESTMethodOsInfoOBDInfo         = "/osinfoobdinfo"
 	peerRESTMethodMemOBDInfo            = "/memobdinfo"
 	peerRESTMethodProcOBDInfo           = "/procobdinfo"
+	peerRESTMethodLogOBDInfo            = "/logobdinfo"
 	peerRESTMethodDispatchNetOBDInfo    = "/dispatchnetobdinfo"
 	peerRESTMethodDeleteBucketMetadata  = "/deletebucketmetadata"
 	peerRESTMethodLoadBucketMetadata    = "/loadbucketmetadata"

cmd/peer-rest-server.go

@@ -47,7 +47,7 @@ func (s *peerRESTServer) GetLocksHandler(w http.ResponseWriter, r *http.Request)
 
 	ctx := newContext(r, w, "GetLocks")
 
-	var llockers []map[string][]lockRequesterInfo
+	llockers := make([]map[string][]lockRequesterInfo, 0, len(globalLockServers))
 	for _, llocker := range globalLockServers {
 		llockers = append(llockers, llocker.DupLockMap())
 	}
@@ -545,6 +545,22 @@ func (s *peerRESTServer) ProcOBDInfoHandler(w http.ResponseWriter, r *http.Reque
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
+// LogOBDInfoHandler - returns Log OBD info.
+func (s *peerRESTServer) LogOBDInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+
+	info := getLocalLogOBD(ctx, r)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
 // MemOBDInfoHandler - returns Mem OBD info.
 func (s *peerRESTServer) MemOBDInfoHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -1039,6 +1055,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcOBDInfo).HandlerFunc(httpTraceHdrs(server.ProcOBDInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLogOBDInfo).HandlerFunc(httpTraceHdrs(server.LogOBDInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemOBDInfo).HandlerFunc(httpTraceHdrs(server.MemOBDInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfoOBDInfo).HandlerFunc(httpTraceHdrs(server.OsOBDInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwOBDInfo).HandlerFunc(httpTraceHdrs(server.DiskHwOBDInfoHandler))

cmd/routers.go

@@ -108,9 +108,8 @@ func configureServerHandler(endpointZones EndpointZones) (http.Handler, error) {
 		}
 	}
 
-	// Add API router, additionally all server mode support encryption
-	// but don't allow SSE-KMS.
-	registerAPIRouter(router, true, false)
+	// Add API router
+	registerAPIRouter(router)
 
 	router.Use(registerMiddlewares)
 

cmd/server-main.go

@@ -17,12 +17,9 @@
 package cmd
 
 import (
-	"bufio"
 	"context"
 	"errors"
 	"fmt"
-	"io"
-	"log"
 	"net"
 	"os"
 	"os/signal"
@@ -247,7 +244,7 @@ func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
 	//    version is needed, migration is needed etc.
 	rquorum := InsufficientReadQuorum{}
 	wquorum := InsufficientWriteQuorum{}
-	for range retry.NewTimer(retryCtx) {
+	for range retry.NewTimerWithJitter(retryCtx, 250*time.Millisecond, 500*time.Millisecond, retry.MaxJitter) {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
 		if err = txnLk.GetLock(configLockTimeout); err != nil {
@@ -394,6 +391,7 @@ func serverMain(ctx *cli.Context) {
 
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(GlobalContext)
+	logger.AddTarget(globalConsoleSys)
 
 	// Handle all server command args.
 	serverHandleCmdArgs(ctx)
@@ -462,27 +460,7 @@ func serverMain(ctx *cli.Context) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	// Annonying hack to ensure that Go doesn't write its own logging,
-	// interleaved with our formatted logging, this allows us to
-	// honor --json and --quiet flag properly.
-	//
-	// Unfortunately we have to resort to this sort of hacky approach
-	// because, Go automatically initializes ErrorLog on its own
-	// and can log without application control.
-	//
-	// This is an implementation issue in Go and should be fixed, but
-	// until then this hack is okay and works for our needs.
-	pr, pw := io.Pipe()
-	go func() {
-		defer pr.Close()
-		scanner := bufio.NewScanner(&contextReader{pr, GlobalContext})
-		for scanner.Scan() {
-			logger.LogIf(GlobalContext, errors.New(scanner.Text()))
-		}
-	}()
-
 	httpServer := xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{corsHandler(handler)}, getCert)
-	httpServer.ErrorLog = log.New(pw, "", 0)
 	httpServer.BaseContext = func(listener net.Listener) context.Context {
 		return GlobalContext
 	}

cmd/signals.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"net/http"
 	"os"
 	"strings"
 
@@ -53,7 +55,9 @@ func handleSignals() {
 
 		if httpServer := newHTTPServerFn(); httpServer != nil {
 			err = httpServer.Shutdown()
-			logger.LogIf(context.Background(), err)
+			if !errors.Is(err, http.ErrServerClosed) {
+				logger.LogIf(context.Background(), err)
+			}
 		}
 
 		if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {

cmd/sts-handlers.go

@@ -88,7 +88,7 @@ func registerSTSRouter(router *mux.Router) {
 		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType))
 		noQueries := len(r.URL.Query()) == 0
 		return ctypeOk && noQueries
-	}).HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
+	}).HandlerFunc(httpTraceAll(sts.AssumeRoleWithSSO))
 
 	// AssumeRoleWithClientGrants
 	stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).
@@ -258,8 +258,8 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseXML(w, encodeResponse(assumeRoleResponse))
 }
 
-func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AssumeRoleJWTCommon")
+func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AssumeRoleSSOCommon")
 
 	// Parse the incoming form data.
 	if err := r.ParseForm(); err != nil {
@@ -274,6 +274,9 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 
 	action := r.Form.Get(stsAction)
 	switch action {
+	case ldapIdentity:
+		sts.AssumeRoleWithLDAPIdentity(w, r)
+		return
 	case clientGrants, webIdentity:
 	default:
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
@@ -417,7 +420,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 // Eg:-
 //    $ curl https://minio:9000/?Action=AssumeRoleWithWebIdentity&WebIdentityToken=<jwt>
 func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *http.Request) {
-	sts.AssumeRoleWithJWT(w, r)
+	sts.AssumeRoleWithSSO(w, r)
 }
 
 // AssumeRoleWithClientGrants - implementation of AWS STS extension API supporting
@@ -426,7 +429,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *h
 // Eg:-
 //    $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token=<jwt>
 func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
-	sts.AssumeRoleWithJWT(w, r)
+	sts.AssumeRoleWithSSO(w, r)
 }
 
 // AssumeRoleWithLDAPIdentity - implements user auth against LDAP server

cmd/test-utils_test.go

@@ -2058,7 +2058,7 @@ func registerBucketLevelFunc(bucket *mux.Router, api objectAPIHandlers, apiFunct
 func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFunctions ...string) {
 	if len(apiFunctions) == 0 {
 		// Register all api endpoints by default.
-		registerAPIRouter(muxRouter, true, false)
+		registerAPIRouter(muxRouter)
 		return
 	}
 	// API Router.
@@ -2088,7 +2088,6 @@ func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFuncti
 			}
 			return nil
 		},
-		EncryptionEnabled: func() bool { return true },
 	}
 
 	// Register ListBuckets	handler.
@@ -2109,7 +2108,7 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
 		registerAPIFunctions(muxRouter, objLayer, apiFunctions...)
 		return muxRouter
 	}
-	registerAPIRouter(muxRouter, true, false)
+	registerAPIRouter(muxRouter)
 	return muxRouter
 }
 

cmd/tree-walk.go

@@ -29,29 +29,20 @@ type TreeWalkResult struct {
 }
 
 // Return entries that have prefix prefixEntry.
-// Note: input entries are expected to be sorted.
+// The supplied entries are modified and the returned string is a subslice of entries.
 func filterMatchingPrefix(entries []string, prefixEntry string) []string {
-	start := 0
-	end := len(entries)
-	for {
-		if start == end {
-			break
-		}
-		if HasPrefix(entries[start], prefixEntry) {
-			break
-		}
-		start++
+	if len(entries) == 0 || prefixEntry == "" {
+		return entries
 	}
-	for {
-		if start == end {
-			break
+	// Write to the beginning of entries.
+	dst := entries[:0]
+	for _, s := range entries {
+		if !HasPrefix(s, prefixEntry) {
+			continue
 		}
-		if HasPrefix(entries[end-1], prefixEntry) {
-			break
-		}
-		end--
+		dst = append(dst, s)
 	}
-	return entries[start:end]
+	return dst
 }
 
 // xl.ListDir returns entries with trailing "/" for directories. At the object layer
@@ -101,12 +92,12 @@ type IsLeafFunc func(string, string) bool
 type IsLeafDirFunc func(string, string) bool
 
 func filterListEntries(bucket, prefixDir string, entries []string, prefixEntry string, isLeaf IsLeafFunc) ([]string, bool) {
-	// Listing needs to be sorted.
-	sort.Strings(entries)
-
 	// Filter entries that have the prefix prefixEntry.
 	entries = filterMatchingPrefix(entries, prefixEntry)
 
+	// Listing needs to be sorted.
+	sort.Strings(entries)
+
 	// Can isLeaf() check be delayed till when it has to be sent down the
 	// TreeWalkResult channel?
 	delayIsLeaf := delayIsLeafCheck(entries)
@@ -124,6 +115,7 @@ func filterListEntries(bucket, prefixDir string, entries []string, prefixEntry s
 	// Sort again after removing trailing "/" for objects as the previous sort
 	// does not hold good anymore.
 	sort.Strings(entries)
+
 	return entries, false
 }
 

cmd/web-handlers.go

@@ -427,7 +427,7 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 			}
 			return toJSONError(ctx, err, args.BucketName)
 		}
-		core, err := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(sr))
+		core, err := getRemoteInstanceClient(r, getHostFromSrv(sr))
 		if err != nil {
 			return toJSONError(ctx, err, args.BucketName)
 		}
@@ -653,7 +653,7 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
 			}
 			return toJSONError(ctx, err, args.BucketName)
 		}
-		core, err := getRemoteInstanceClientLongTimeout(r, getHostFromSrv(sr))
+		core, err := getRemoteInstanceClient(r, getHostFromSrv(sr))
 		if err != nil {
 			return toJSONError(ctx, err, args.BucketName)
 		}
@@ -1166,7 +1166,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	if mustReplicate {
-		defer replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+		go replicateObject(GlobalContext, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedPut,
 			BucketName:   bucket,
 			Object:       objInfo,

cmd/xl-storage.go

@@ -38,7 +38,7 @@ import (
 	"syscall"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio/cmd/config"
@@ -46,6 +46,7 @@ import (
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/env"
 	xioutil "github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 const (
@@ -357,6 +358,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
 	if objAPI == nil {
 		return cache, errServerNotInitialized
 	}
+	opts := globalCrawlerConfig
 
 	dataUsageInfo, err := crawlDataFolder(ctx, s.diskPath, cache, s.waitForLowActiveIO, func(item crawlItem) (int64, error) {
 		// Look for `xl.meta/xl.json' at the leaf.
@@ -394,6 +396,25 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
 				oi:               oi,
 			})
 			if !version.Deleted {
+				// Bitrot check local data
+				if size > 0 && item.heal && opts.Bitrot {
+					s.waitForLowActiveIO()
+					err := s.VerifyFile(ctx, item.bucket, item.objectPath(), version)
+					switch err {
+					case errFileCorrupt:
+						res, err := objAPI.HealObject(ctx, item.bucket, item.objectPath(), oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling, ScanMode: madmin.HealDeepScan})
+						if err != nil {
+							if !errors.Is(err, NotImplemented{}) {
+								logger.LogIf(ctx, err)
+							}
+							size = 0
+						} else {
+							size = res.ObjectSize
+						}
+					default:
+						// VerifyFile already logs errors
+					}
+				}
 				totalSize += size
 			}
 			item.healReplication(ctx, objAPI, actionMeta{oi: oi})
@@ -633,7 +654,7 @@ func listVols(dirPath string) ([]VolInfo, error) {
 	if err != nil {
 		return nil, errDiskNotFound
 	}
-	var volsInfo []VolInfo
+	volsInfo := make([]VolInfo, 0, len(entries))
 	for _, entry := range entries {
 		if !HasSuffix(entry, SlashSeparator) || !isValidVolname(slashpath.Clean(entry)) {
 			// Skip if entry is neither a directory not a valid volume name.

docs/kms/README.md

@@ -1,16 +1,10 @@
 # KMS Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
 
-MinIO uses a key-management-system (KMS) to support SSE-S3. If a client requests SSE-S3, or auto-encryption
-is enabled, the MinIO server encrypts each object with an unique object key which is protected by a master key
-managed by the KMS.
+MinIO uses a key-management-system (KMS) to support SSE-S3. If a client requests SSE-S3, or auto-encryption is enabled, the MinIO server encrypts each object with an unique object key which is protected by a master key managed by the KMS.
 
 ## Quick Start
 
-MinIO supports multiple KMS implementations via our [KES](https://github.com/minio/kes#kes) project. We run
-a KES instance at `https://play.min.io:7373` for you to experiment and quickly get started. To run MinIO with
-a KMS just fetch the root identity, set the following environment variables and then start your MinIO server.
-If you havn't installed MinIO, yet, then follow the MinIO [install instructions](https://docs.min.io/docs/minio-quickstart-guide)
-first.
+MinIO supports multiple KMS implementations via our [KES](https://github.com/minio/kes#kes) project. We run a KES instance at `https://play.min.io:7373` for you to experiment and quickly get started. To run MinIO with a KMS just fetch the root identity, set the following environment variables and then start your MinIO server. If you havn't installed MinIO, yet, then follow the MinIO [install instructions](https://docs.min.io/docs/minio-quickstart-guide) first.
 
 #### 1. Fetch the root identity
 As the initial step, fetch the private key and certificate of the root identity:
@@ -55,11 +49,9 @@ A typical MinIO deployment that uses a KMS for SSE-S3 looks like this:
                                                                 └─────────┘
 ```
 
-In a given setup, there are `n` MinIO instances talking to `m` KES servers but only `1` central KMS. The most simple
-setup consists of `1` MinIO server or cluster talking to `1` KMS via `1` KES server.
+In a given setup, there are `n` MinIO instances talking to `m` KES servers but only `1` central KMS. The most simple setup consists of `1` MinIO server or cluster talking to `1` KMS via `1` KES server.
 
-The main difference between various MinIO-KMS deployments is the KMS implementation. The following table
-helps you select the right option for your use case:
+The main difference between various MinIO-KMS deployments is the KMS implementation. The following table helps you select the right option for your use case:
 
 | KMS                                                                              | Purpose                                                           |
 |:---------------------------------------------------------------------------------|:------------------------------------------------------------------|
@@ -67,8 +59,7 @@ helps you select the right option for your use case:
 | [AWS-KMS + SecretsManager](https://github.com/minio/kes/wiki/AWS-SecretsManager) | Cloud KMS. MinIO in combination with a managed KMS installation   |
 | [FS](https://github.com/minio/kes/wiki/Filesystem-Keystore)                      | Local testing or development (**Not recommended for production**) |
 
-The MinIO-KES configuration is always the same - regardless of the underlying KMS implementation.
-Checkout the MinIO-KES [configuration example](https://github.com/minio/kes/wiki/MinIO-Object-Storage).
+The MinIO-KES configuration is always the same - regardless of the underlying KMS implementation. Checkout the MinIO-KES [configuration example](https://github.com/minio/kes/wiki/MinIO-Object-Storage).
 
 ### Further references
 
@@ -78,9 +69,9 @@ Checkout the MinIO-KES [configuration example](https://github.com/minio/kes/wiki
 - [Understand the KES server concepts](https://github.com/minio/kes/wiki/Concepts)
 
 ## Auto Encryption
+Auto-Encryption is useful when MinIO administrator wants to ensure that all data stored on MinIO is encrypted at rest.
 
-Auto-Encryption is useful when MinIO administrator wants to ensure that all data stored on MinIO is encrypted at rest. 
-
+### Using `mc encrypt` (recommended)
 MinIO automatically encrypts all objects on buckets if KMS is successfully configured and bucket encryption configuration is enabled for each bucket as shown below:
 ```
 mc encrypt sse-s3 myminio/bucket/
@@ -92,7 +83,15 @@ mc encrypt info myminio/bucket/
 Auto encryption 'sse-s3' is enabled
 ```
 
+### Using environment (deprecated)
+> NOTE: Following ENV might be removed in future, you are advised to move to previous recommeneded approach using `mc encrypt`. S3 gateway supports encryption at gateway layer which may thus be dropped in favor of simplicity, it is advised that S3 gateway users migrate to MinIO server mode or enable encryption at REST at the backend.
 
+MinIO automatically encrypts all objects on buckets if KMS is successfully configured and following ENV is enabled:
+```
+export MINIO_KMS_AUTO_ENCRYPTION=on
+```
+
+### Verify auto-encryption
 > Note that auto-encryption only affects requests without S3 encryption headers. So, if a S3 client sends
 > e.g. SSE-C headers, MinIO will encrypt the object with the key sent by the client and won't reach out to
 > the configured KMS.

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # it through port 9000.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
+    image: minio/minio:RELEASE.2020-09-10T22-02-45Z
     hostname: minio4
     volumes:
       - minio4-data:/export

go.mod

@@ -79,9 +79,9 @@ require (
 	github.com/willf/bloom v2.0.3+incompatible
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200707003333-58bb8ae09f8e
-	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381
-	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
+	golang.org/x/net v0.0.0-20200904194848-62affa334b73
+	golang.org/x/sys v0.0.0-20200915084602-288bc346aa39
 	golang.org/x/tools v0.0.0-20200814172026-c4923e618c08 // indirect
 	google.golang.org/api v0.5.0
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0

go.sum

@@ -501,6 +501,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
+golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -530,6 +532,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
+golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -564,6 +568,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20u
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200915084602-288bc346aa39 h1:356XA7ITklAU2//sYkjFeco+dH1bCRD8XCJ9FIEsvo4=
+golang.org/x/sys v0.0.0-20200915084602-288bc346aa39/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=

pkg/dsync/drwmutex.go

@@ -25,7 +25,6 @@ import (
 	"time"
 
 	"github.com/minio/minio/pkg/console"
-	"github.com/minio/minio/pkg/retry"
 )
 
 // Indicator if logging is enabled.
@@ -132,6 +131,10 @@ func (dm *DRWMutex) GetRLock(ctx context.Context, id, source string, opts Option
 	return dm.lockBlocking(ctx, id, source, isReadLock, opts)
 }
 
+const (
+	lockRetryInterval = 50 * time.Millisecond
+)
+
 // lockBlocking will try to acquire either a read or a write lock
 //
 // The function will loop using a built-in timing randomized back-off
@@ -140,40 +143,50 @@ func (dm *DRWMutex) GetRLock(ctx context.Context, id, source string, opts Option
 func (dm *DRWMutex) lockBlocking(ctx context.Context, id, source string, isReadLock bool, opts Options) (locked bool) {
 	restClnts := dm.clnt.GetLockersFn()
 
-	retryCtx, cancel := context.WithTimeout(ctx, opts.Timeout)
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	defer cancel()
+	// Create lock array to capture the successful lockers
+	locks := make([]string, len(restClnts))
 
-	// Use incremental back-off algorithm for repeated attempts to acquire the lock
-	for range retry.NewTimer(retryCtx) {
-		// Create temp array on stack.
-		locks := make([]string, len(restClnts))
-
-		// Try to acquire the lock.
-		locked = lock(retryCtx, dm.clnt, &locks, id, source, isReadLock, opts.Tolerance, dm.Names...)
-		if !locked {
-			continue
+	cleanLocks := func(locks []string) {
+		for i := range locks {
+			locks[i] = ""
 		}
-
-		dm.m.Lock()
-
-		// If success, copy array to object
-		if isReadLock {
-			// Append new array of strings at the end
-			dm.readersLocks = append(dm.readersLocks, make([]string, len(restClnts)))
-			// and copy stack array into last spot
-			copy(dm.readersLocks[len(dm.readersLocks)-1], locks[:])
-		} else {
-			copy(dm.writeLocks, locks[:])
-		}
-
-		dm.m.Unlock()
-		return locked
 	}
 
-	// Failed to acquire the lock on this attempt, incrementally wait
-	// for a longer back-off time and try again afterwards.
-	return locked
+	retryCtx, cancel := context.WithTimeout(ctx, opts.Timeout)
+	defer cancel()
+
+	for {
+		// cleanup any older state, re-use the lock slice.
+		cleanLocks(locks)
+
+		select {
+		case <-retryCtx.Done():
+			// Caller context canceled or we timedout,
+			// return false anyways for both situations.
+			return false
+		default:
+			// Try to acquire the lock.
+			if locked = lock(retryCtx, dm.clnt, &locks, id, source, isReadLock, opts.Tolerance, dm.Names...); locked {
+				dm.m.Lock()
+
+				// If success, copy array to object
+				if isReadLock {
+					// Append new array of strings at the end
+					dm.readersLocks = append(dm.readersLocks, make([]string, len(restClnts)))
+					// and copy stack array into last spot
+					copy(dm.readersLocks[len(dm.readersLocks)-1], locks[:])
+				} else {
+					copy(dm.writeLocks, locks[:])
+				}
+
+				dm.m.Unlock()
+				return locked
+			}
+			time.Sleep(time.Duration(r.Float64() * float64(lockRetryInterval)))
+		}
+	}
 }
 
 // lock tries to acquire the distributed lock, returning true or false.

pkg/event/name.go

@@ -56,12 +56,21 @@ func (name Name) Expand() []Name {
 	case BucketRemoved:
 		return []Name{BucketRemoved}
 	case ObjectAccessedAll:
-		return []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}
+		return []Name{
+			ObjectAccessedGet, ObjectAccessedHead,
+			ObjectAccessedGetRetention, ObjectAccessedGetLegalHold,
+		}
 	case ObjectCreatedAll:
-		return []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy,
-			ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention, ObjectCreatedPutLegalHold}
+		return []Name{
+			ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy,
+			ObjectCreatedPost, ObjectCreatedPut,
+			ObjectCreatedPutRetention, ObjectCreatedPutLegalHold,
+		}
 	case ObjectRemovedAll:
-		return []Name{ObjectRemovedDelete}
+		return []Name{
+			ObjectRemovedDelete,
+			ObjectRemovedDeleteMarkerCreated,
+		}
 	default:
 		return []Name{name}
 	}

pkg/event/name_test.go

@@ -33,7 +33,7 @@ func TestNameExpand(t *testing.T) {
 		{ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}},
 		{ObjectCreatedAll, []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy,
 			ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention, ObjectCreatedPutLegalHold}},
-		{ObjectRemovedAll, []Name{ObjectRemovedDelete}},
+		{ObjectRemovedAll, []Name{ObjectRemovedDelete, ObjectRemovedDeleteMarkerCreated}},
 		{ObjectAccessedHead, []Name{ObjectAccessedHead}},
 	}
 

pkg/event/target/nats_tls_test.go

@@ -19,7 +19,6 @@ package target
 import (
 	"path"
 	"path/filepath"
-	"runtime"
 	"testing"
 
 	xnet "github.com/minio/minio/pkg/net"
@@ -42,9 +41,6 @@ func TestNatsConnTLSCustomCA(t *testing.T) {
 
 	con, err := clientConfig.connectNats()
 	if err != nil {
-		if runtime.Version() == "go1.15" {
-			t.Skip()
-		}
 		t.Errorf("Could not connect to nats: %v", err)
 	}
 	defer con.Close()
@@ -68,9 +64,6 @@ func TestNatsConnTLSClientAuthorization(t *testing.T) {
 
 	con, err := clientConfig.connectNats()
 	if err != nil {
-		if runtime.Version() == "go1.15" {
-			t.Skip()
-		}
 		t.Errorf("Could not connect to nats: %v", err)
 	}
 	defer con.Close()

pkg/event/target/testdata/certs/nats_client_cert.pem

@@ -1,19 +1,13 @@
 -----BEGIN CERTIFICATE-----
-MIIDFTCCAf0CFHzIPe2CTpos6fYYVcZD4xAMYD0FMA0GCSqGSIb3DQEBCwUAMEox
-CzAJBgNVBAYTAkNBMRMwEQYDVQQIDApTb21lLVN0YXRlMQ4wDAYDVQQKDAVNaW5p
-bzEWMBQGA1UECwwNTWluaW8gUm9vdCBDQTAgFw0xOTEwMTEyMDE4NTJaGA8yMTE5
-MDkxNzIwMTg1MlowQjELMAkGA1UEBhMCQ0ExDjAMBgNVBAoMBU1pbmlvMQ8wDQYD
-VQQLDAZDbGllbnQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
-BQADggEPADCCAQoCggEBANsLgS7C0BDsfhWQVyuFGrpl9IWGyAQvSECzepdvxcUz
-CJ5QTSjtMq+2XbkozsmJ0WKtXY+c2vYuNMcthfNzLmn8KMxDoCqMwLXNMO8lLuF5
-94pH0DmTvEngjEDIOPM9aNbfmLU9Lbeiplkjt0izYObg+MzsTS0lIR2yKtSoVFOV
-zzSGRvMfUO/rOlRgBio9Hhx8fL0cckr7SzwJcYtttr+1cySNkGVI8/E5Pe0T3fVW
-7kiCHtUbbST0sQWorDDMkwTYyd2TvB8B8BzfSRNUavnb37bp8woqJottYXXEcLff
-HL7kOhnmqvOSLSnG9ynF+WlE7ZykHw+VsIrRsl/5NwkCAwEAATANBgkqhkiG9w0B
-AQsFAAOCAQEASfRN6esWr68ieeODFuiyhMaZ3kHKWiyvT3MDrFPcBgTIbi4CvFmb
-+ynT7aDKxBpO8Ttl208QyLZxn9YeiFCZVVy8+o1JaQd30cjewX6A9dCd16yi3lva
-tbVLVl+pJqKwr/wdEwrJ23bUYpFNrex38dxaRUOQ4Zaj6QzCXvB+IR/RvYZ2U23o
-pW0BEYdi+V3M0tuAvmX4NOrLvGUF1D93WRQuiueaMF8yNhiwuT/eCcpct3FLqacC
-+NvdK0aiBdUgDvdA7OroXy0Ow768wHpLvfsxqepiXXK/9748hMQxgeaUzDE6SRFF
-FqgnBuFgIu4MH7ki1EnH+KekFepsPF7BIA==
+MIICCjCCAbGgAwIBAgIUKLFyLD0Ze9gR3A2aBxgEiT6MgZUwCgYIKoZIzj0EAwIw
+GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMzI0MzNaFw0zMDA5
+MTIxMzI0MzNaMEIxCzAJBgNVBAYTAkNBMQ4wDAYDVQQKDAVNaW5JTzEPMA0GA1UE
+CwwGQ2xpZW50MRIwEAYDVQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjO
+PQMBBwNCAARAhYrQXYbzeKyVSw8nf57gBphwFP1o5S7CjxoGKCfghzdhExKiEmbi
+sK+FSS2YtltU7cM7L7AduLIbuEnGHHYQo4GuMIGrMAkGA1UdEwQCMAAwUwYDVR0j
+BEwwSoAUWN6Fr30E5vvvNOBkuGGkqGzA3SihHKQaMBgxFjAUBgNVBAMMDU1pbmlv
+IFJvb3QgQ0GCFHiTsAON45VvwFb0MxHEdLPeWi95MA4GA1UdDwEB/wQEAwIFoDAd
+BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGgYDVR0RBBMwEYcEfwAAAYIJ
+bG9jYWxob3N0MAoGCCqGSM49BAMCA0cAMEQCIC7MHOEf0C/zqw/ZOaCffeJIMeFm
+iT8ugBfhFbgGkd5YAiBz9FEfV4JMZQ4N29WLmvxxDSxkL8g5e3fnIK8Aa4excw==
 -----END CERTIFICATE-----

pkg/event/target/testdata/certs/nats_client_key.pem

@@ -1,27 +1,5 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEA2wuBLsLQEOx+FZBXK4UaumX0hYbIBC9IQLN6l2/FxTMInlBN
-KO0yr7ZduSjOyYnRYq1dj5za9i40xy2F83MuafwozEOgKozAtc0w7yUu4Xn3ikfQ
-OZO8SeCMQMg48z1o1t+YtT0tt6KmWSO3SLNg5uD4zOxNLSUhHbIq1KhUU5XPNIZG
-8x9Q7+s6VGAGKj0eHHx8vRxySvtLPAlxi222v7VzJI2QZUjz8Tk97RPd9VbuSIIe
-1RttJPSxBaisMMyTBNjJ3ZO8HwHwHN9JE1Rq+dvftunzCiomi21hdcRwt98cvuQ6
-Geaq85ItKcb3KcX5aUTtnKQfD5WwitGyX/k3CQIDAQABAoIBACrO7Mg+unsUPO/p
-7Z9LvBWBp3ARDzYCJ5S9fs/pwDTx7FVETFAbSzSb52UwXHl2vb0TNJ3EgeZq0VW7
-x9n0QLXl2fNRpBOsvlzJZS7XjXnzZDVaI4+dF8c4YzCl8LtY3QxhVm7VLgIdf3Uc
-Tc2fgOiePwGNjOetwfMTxtsYqqJK4Ang/nop6WIZtJxlq/H4D/OueDScv8yM96WP
-Lb+5YMrlTtA79TTn44hdd/IVJBV74UK335bWaV2gqop/r/kymYcmIS6eQ8Fx8Au8
-FhqjL38r6H5mnyH/euBJDf+B9aPej44m093j/gy8C6PFdVHXOw4h3su+RJkV7J+x
-JzSCYiECgYEA/0e1h7XVHVjRI/hzlpZlHomw4SYXc1R5DBljsD+0baNWdgwShdiA
-qZyhVl1/WJEK6r+tT/a0rGcIz1CkjPvjOf66+FVwukqp4UDdeZA28CC6GvbAlBW7
-Gz3wWjbC3quR8OCcGbNrRIUegeN3KxIpyzTXKv2PRUnkkj2z2+clf1sCgYEA26mj
-AiynCFZH2DIk36y1Qvn2Yh/LgO+JrZNmh0YlDENstLXEU5tkLyRxhbYDsDFKHr7a
-1pHZZa2SLKMQEKNZ3OgP5osvl4BxSt5SrusGQHACb1FeWN3BdIWI5S0ZIgBih46G
-IKA1ce4CnLo3oWm1L9tEFnTtMEqEtkECMnYTtGsCgYAsDgYH65thuygsmv3nqQC+
-ami6EkbNwnA5ZFBN5FCQ8zVCnga8Toa1vrAhJXWKpU1LAdU5DYxUjFt626HqKrYm
-Fg3SOXyAyc3Tb8xI5Fh2zE8RxC+r3qwxoVjPWM/8eYNwHHMUBGCorIh9RfIU3seT
-qATSCHwnKv9lNXzKoHNaLQKBgGURdkkn1mrFmCTnXYP06Sm57R1U18Opc0WEDqar
-JZyw4TF8eKqnUr9GG12UU7ob06i10+bqEIbyB9G17UxafJxhwf8nh2xD0tzJ0m1d
-AfFgGB6z558n2T0Nu+EGkQvN9Ye9kgUs8apMArOuEq6X/p/YWUmj3wZbIxjgbGxf
-W82lAoGATTwJCXE6y+D5Vc5I9BTPgJd6Wg/6n+BAmadt3lG/VbPxndQBudCNii0Z
-C7uyMrbK0RVY+DZAFuZWLskIoU+t82HgIsQNAH5KPz27PCWq+DznrARp+maKXj5f
-f5Vsvz6KBzM8Lnts3m20AYcRP4meBa6HNHLnYGedTbRLKYobOgg=
------END RSA PRIVATE KEY-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIBluB2BuspJcz1e58rnXpQEx48/ZwNmygNw06NbdTZDroAoGCCqGSM49
+AwEHoUQDQgAEQIWK0F2G83islUsPJ3+e4AaYcBT9aOUuwo8aBign4Ic3YRMSohJm
+4rCvhUktmLZbVO3DOy+wHbiyG7hJxhx2EA==
+-----END EC PRIVATE KEY-----

pkg/event/target/testdata/certs/nats_server_cert.pem

@@ -1,19 +1,12 @@
 -----BEGIN CERTIFICATE-----
-MIIDFTCCAf0CFHzIPe2CTpos6fYYVcZD4xAMYD0EMA0GCSqGSIb3DQEBCwUAMEox
-CzAJBgNVBAYTAkNBMRMwEQYDVQQIDApTb21lLVN0YXRlMQ4wDAYDVQQKDAVNaW5p
-bzEWMBQGA1UECwwNTWluaW8gUm9vdCBDQTAgFw0xOTEwMTEyMDE1NTRaGA8yMTE5
-MDkxNzIwMTU1NFowQjELMAkGA1UEBhMCQ0ExDjAMBgNVBAoMBU1pbmlvMQ8wDQYD
-VQQLDAZTZXJ2ZXIxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
-BQADggEPADCCAQoCggEBAMxbKHUPlW4wIX0G45jRG84o5viXtu7E044QpTGwDS+u
-KCG3w4pWUwnor/uCjkzKSmg+6Sx/889de7QySPwt4TA+F7qy7TrUnIUiaVFcPmeQ
-YsI68JM857nk4ScxK4C4NzJ78DlQreyPwIMx4rAGfEZpdQbAE0LWyOJ9rfMK1uF/
-PFdyNKOUw/fy5iA/DOUeoUqFATIjj5KUWojnt9QZ+M1pGbQsLWc7fzDuavVrAvR7
-wt386bayzLftzHr6rLWWzPoA70ltzvu6JxKU5xZSY5nA2Pip+le6q997+Sm5Dt5L
-VZuSTzjKVNDV9l329eZJnozROdlSURVvg6fK5J8vUIsCAwEAATANBgkqhkiG9w0B
-AQsFAAOCAQEAg6kLnBrjT4agClF94Rc2y6fOTWWzz7C4Ao6iQp3Tcr/YraY7FEEM
-kkZT0osn1JSvut9f2gy2py3+FLAk4OdAt9NBIpjIMDmbIyiQYw+2w3IHJYwdHBXj
-HapyjU2zFQvD6PNUUV4YukR28MmMk0UEizXxkEDm+Qf5t4Bpnv7tFiysMWmA6cCJ
-1fvGOtCPazi0hZc0VkfTnl5OY/Msi+sYCh4gnCqYWICszMm37QKOJrJbY8mTSzAk
-p/0dxAUDDKTuf+UPDqJ2I6o9/XEEwmPGOFovoS9iQhdyWWonRDL3+gCeE2cxEeu5
-QEOwxMQz08XRSsgxag9R3tcywXbaamgQYw==
+MIIByTCCAW+gAwIBAgIUdAg80BTm1El7s5ZZezgjsls9BwkwCgYIKoZIzj0EAwIw
+GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMjQzMjNaFw0zMDA5
+MTIxMjQzMjNaMAAwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASolKUI7FVSA2Ts
++GSW/DHDKNczDNjfccI2GLETso6ie8buveOODj1JIL9ff5pRDN+U6QvwwlDmXEqh
+1a6XBI4Ho4GuMIGrMAkGA1UdEwQCMAAwUwYDVR0jBEwwSoAUWN6Fr30E5vvvNOBk
+uGGkqGzA3SihHKQaMBgxFjAUBgNVBAMMDU1pbmlvIFJvb3QgQ0GCFHiTsAON45Vv
+wFb0MxHEdLPeWi95MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcD
+AQYIKwYBBQUHAwIwGgYDVR0RBBMwEYcEfwAAAYIJbG9jYWxob3N0MAoGCCqGSM49
+BAMCA0gAMEUCIB7WXnQAkmjw2QE6A3uOscOIctJnlVNREfm4V9CrF6UGAiEA734B
+vKlhMk8H459BRoIp8GpOuUWqLqocSmMM1febvcg=
 -----END CERTIFICATE-----

pkg/event/target/testdata/certs/nats_server_key.pem

@@ -1,27 +1,5 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAzFsodQ+VbjAhfQbjmNEbzijm+Je27sTTjhClMbANL64oIbfD
-ilZTCeiv+4KOTMpKaD7pLH/zz117tDJI/C3hMD4XurLtOtSchSJpUVw+Z5Biwjrw
-kzznueThJzErgLg3MnvwOVCt7I/AgzHisAZ8Rml1BsATQtbI4n2t8wrW4X88V3I0
-o5TD9/LmID8M5R6hSoUBMiOPkpRaiOe31Bn4zWkZtCwtZzt/MO5q9WsC9HvC3fzp
-trLMt+3MevqstZbM+gDvSW3O+7onEpTnFlJjmcDY+Kn6V7qr33v5KbkO3ktVm5JP
-OMpU0NX2Xfb15kmejNE52VJRFW+Dp8rkny9QiwIDAQABAoIBABPVsTqTdaSJRWbW
-OVcGzNUYwTpVt2q7bfE3CmGlPdJn6/tB09fkgxDOJe6agGdRpyExIf6wuKBi6XPX
-AaCAb3/4NuNnJIF2S27cpS5BbsksiXlisSEJY1B2t6fPLPLEbo8W2n0lqIvyc/QH
-7oG5T2yiJbqu/++X202ody7E2ZBvVXgwjBJuzxeQ9PPiPyjH2lAgciA0HtmXiHSj
-pIz6V6XmN5CkyawvExA3F1wxIxf2CNRHLpls/DFGkVIz6pUlleKe7fBKVqeLuwnd
-Lnjt9lrCrEozXcwUQnw5KVccB2+0xU8pIl78tPf9EHOHf5Bx1WCFl/YL5qEmfSsL
-x3WM/WkCgYEA+7wdzK2M3LSJ6tat+eM9DT8bqSFhFAA41Jv1UdhTJZb8AKPs+6wX
-13rUcjTnSI75KqApGaIbe3dF7640VmD9eyLNkc2SHr/yNPKV96sEiWk7TVgIWa0J
-vBqPB9UWrkzqBov5VJEpeKlKuCCZGOxN7nkMtH93XmX9Nm3BxmMCQ8cCgYEAz9GK
-FS1pwjj8DpyLW5GEFbcwGugkb/4ebc/OK4W93YvAYwoyJF7oSI9WzKG/R5Ko/0G7
-ESHt0d2pfRDR5XCkoJee4MFFIheLeVM4HDUoIY42I7gl/op+H9fGSNplBYDpVk7O
-FNl9NFgUDytMbuh14SmAU3Yc7EjGruFgoZUPRR0CgYEAimwNos/HxlDMCcMUlXT7
-zD3oct7056+bkGVVxzSBvAjC94MsO9OMpKNZEJfAmehsYKEDGKJIJGMYpMwQ4XKh
-z8T6bvMwJxJ7F9xQ1IhIjVq9DjGbHdyFntan0bG9sAiBIypy5qqPuFa2zHq4VLkT
-vU74yoPQ2qqQSw6dX/5vb18CgYEAnLlZeT8WUvLGo/5K2nOTOQ09qg0H7a2nJQli
-YlAqL7oFDKvTxLoOUypGO2x/5GomKNpZSUJdJ7gS6c6VfILGpJWzq6wVhvBartSj
-rCIqcaPeEHH/tUacd0cysh6BsPTXA8/Kes7KLX9/ITF2Iu4MeBHkRQz+IvN/YsN1
-LGZNbcECgYAqWDnPa2GwFOgTrVwB1AfsCD1PVhSM+AnWnL7avHwD6WIDKB5qb9Ip
-cF83o5bhLrLYVlpZNI1ESKgONyWlYOssBNb0KP51kgkBbtr+5qcCFoKzf5inyve2
-MTE1g+maOLtfRaVrlsGcirWSZdIk8dQmDX8nNM+Rg5hahF2l0/zopg==
------END RSA PRIVATE KEY-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEILFuMS2xvsc/CsuqtSv3S2iSCcc28rZsg1wpR2kirXFloAoGCCqGSM49
+AwEHoUQDQgAEqJSlCOxVUgNk7PhklvwxwyjXMwzY33HCNhixE7KOonvG7r3jjg49
+SSC/X3+aUQzflOkL8MJQ5lxKodWulwSOBw==
+-----END EC PRIVATE KEY-----

pkg/event/target/testdata/certs/root_ca_cert.pem

@@ -1,21 +1,11 @@
 -----BEGIN CERTIFICATE-----
-MIIDdzCCAl+gAwIBAgIUTuCjkiDWaSa3X747ybs/SnZ6aHQwDQYJKoZIhvcNAQEL
-BQAwSjELMAkGA1UEBhMCQ0ExEzARBgNVBAgMClNvbWUtU3RhdGUxDjAMBgNVBAoM
-BU1pbmlvMRYwFAYDVQQLDA1NaW5pbyBSb290IENBMCAXDTE5MTAxMTE5NTIxOFoY
-DzIxMTkwOTE3MTk1MjE4WjBKMQswCQYDVQQGEwJDQTETMBEGA1UECAwKU29tZS1T
-dGF0ZTEOMAwGA1UECgwFTWluaW8xFjAUBgNVBAsMDU1pbmlvIFJvb3QgQ0EwggEi
-MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDDAOyo76TMpltNUXiRAGaxaVMG
-E2cf/JudJ+2ve7lxXo1Zz9PaJUOa5wl2b5ebwhXYP7lxdRRz488TfM4YmAm1fkgS
-fhBHP337aeUCeW4mybv7P0jBDHYasiDy92YqX5tv2zPifRaoQ/jkUmEyY29VdTQq
-LyuUCo4hmY3tO5daXN+x9DdnZj8VDkC235UpEe9vgvtY62m7w+guV/XdwKBvQKpa
-248qaqUxvJ8l9sXCnLjNMWwm/6ZpPPcQvYW95P2bhwGH8KrdKeSutW8vVeUa2DN+
-E7QsEopVvg48t/dSVol9xXVQ2QMlNYBsrPQS04GX3aakVREI0r/krRl11Kf7AgMB
-AAGjUzBRMB0GA1UdDgQWBBQtvCicVaNfZaLEXW7sgZYlTyjuuzAfBgNVHSMEGDAW
-gBQtvCicVaNfZaLEXW7sgZYlTyjuuzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3
-DQEBCwUAA4IBAQC9LflSKc6l8j280aTQyjjTdMysmKqb8Rs7dtSSaFXQK9nuzOhD
-FGtahhsJFv/GnUu0weZoEMYySeNzaUoW0ICc3+iI2KMWcUlF/K3/P47tYrVMGJKW
-YbuUS7lTxtra9xbFhxGCF8NwM6JG9TrP7JT9b/tn2Xb+UNsxWTH8pf1plusiOvQl
-2JFc+BJzhooLP9sK0YGrb7i3CdQj5QcZjxVCDKAuosvMY/shoW4aEIEECSEtJ9mY
-Rrfft0bOQ7ZKhDgu2ou+lyeyyWFS/19wr5LMvy+lUYG9H100peCpv5tHdGUF1lLR
-m4g6eQK57EI6BoPpNomYtXkoRKP9MXiZ7InC
+MIIBlTCCATygAwIBAgIUeJOwA43jlW/AVvQzEcR0s95aL3kwCgYIKoZIzj0EAwIw
+GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMjMwMDJaFw0zMDA5
+MTIxMjMwMDJaMBgxFjAUBgNVBAMMDU1pbmlvIFJvb3QgQ0EwWTATBgcqhkjOPQIB
+BggqhkjOPQMBBwNCAARK9fVNGHc1h5B5fpOMyEdyhh18xNNcNUGQ5iGLO97Z0KtK
+5vRlDeeE1I0SaJgqppm9OEHw32JU0HMi4FBZi2Rso2QwYjAdBgNVHQ4EFgQUWN6F
+r30E5vvvNOBkuGGkqGzA3SgwHwYDVR0jBBgwFoAUWN6Fr30E5vvvNOBkuGGkqGzA
+3SgwDwYDVR0TAQH/BAUwAwEB/zAPBgNVHREECDAGhwR/AAABMAoGCCqGSM49BAMC
+A0cAMEQCIDPOiks2Vs3RmuJZl5HHjuqaFSOAp1g7pZpMb3Qrh9YDAiAtjO2xOpkS
+WynK8P7EfyQP/IUa7GxJIoHk6/H/TCsYvQ==
 -----END CERTIFICATE-----

pkg/event/target/testdata/certs/root_ca_key.pem

@@ -1,27 +1,5 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEAwwDsqO+kzKZbTVF4kQBmsWlTBhNnH/ybnSftr3u5cV6NWc/T
-2iVDmucJdm+Xm8IV2D+5cXUUc+PPE3zOGJgJtX5IEn4QRz99+2nlAnluJsm7+z9I
-wQx2GrIg8vdmKl+bb9sz4n0WqEP45FJhMmNvVXU0Ki8rlAqOIZmN7TuXWlzfsfQ3
-Z2Y/FQ5Att+VKRHvb4L7WOtpu8PoLlf13cCgb0CqWtuPKmqlMbyfJfbFwpy4zTFs
-Jv+maTz3EL2FveT9m4cBh/Cq3SnkrrVvL1XlGtgzfhO0LBKKVb4OPLf3UlaJfcV1
-UNkDJTWAbKz0EtOBl92mpFURCNK/5K0ZddSn+wIDAQABAoIBABcVbu54n9uR/dDj
-ShBwKbfqredUOKryrkEmTt6tGMCw3q65CW0TaDNYEiixARNxIEDfGkayA1/MoeC+
-r2794HhZoJJ/1dF5VKKEYJWwZje1Pcl6LlSb8wcp+viIDNILS00sd7Hh+OKmWTo9
-/j+GwdEt9KThvJjjvvt/P+vLWBlcHD2zKFJnq826jKFESkUkY8P6e0cRzOuwGY+E
-LYXAwZC+j7wS3c7tW7PyhGAXkTxohtNNYZ6vOGnGetnTxQDICZ9hCIZMA+7hrtK8
-TAGWtWvwx/5SgEl2+b7M0q70I/2LQSB4ACgOfkhm38z7FwqkvTMl9YhR8E4qvZsZ
-2fv4hoECgYEA5GG8W1H82Z8fxY4AMKwbvlZ5pZGuIwwm+ABsg5BlSMfXZS2APJuP
-wUUOW2gcjDXq/m+7HphgJADpkf+j6NhTks2gWF32VPCfp1EnxOrjOkrZqtBcud+A
-1d0HeWvUMYl4iq9+3haILyfN4nQkFeYO1X0wD2kgrEeIkgipjG//tlsCgYEA2pXd
-PvXugREkt+HladpCQXPnxHgS+QZuPNBYcDFMlOpC6RdcLzDWcqGzmYROM9/sbI+c
-0H3l9ROOkqkA1jq8K2apOKmH9NBOZAaIfi+A450rnJK0trhMsuUgY/xrj+BBO1Vm
-LrAIJ6TMpp8DntO/EVitLBwjR6WrM4pUB9kRxuECgYEA3vEp1luq4SYc9dUxClJ4
-os334lDcFQp/4AlJ5QTIWsv60KIiVQfmxVyML17qv1TDGa4olC1bbMoXOJa9g0fq
-DZz9skXHehOLRuJKWEiTmQwIgF72pdwxAJTL/xPsCI+SRJAc4OBOAPpyWWXW9Cmo
-wW97ww90/bi28RfTq2yJy5ECgYARPXq6yYjrMx/zROTkSWuqX+rqyxGsBH7TWxdu
-meTRZfyrB8WkjzSKzAgvVokYfFPYaCdVJmjpwIYhOSUwwGcxASLdrjlj7L4SE5XW
-ZgbDbRUQf12zf6vE/F9mo3UUXvqmJGEv04CBJ/VgOvB9KXRLePQHo5yAvSdYpFNm
-Xw+Q4QKBgHbV9YV+b1D+8SKNgpCZcrgTmvXIS4ibHdIYT2cxqGbkYS9T1UuJ1kWd
-PBeeYsF3CeCBi1PGKI9eYO85W0aS3fZyrekBOyOVDUqQNpXQPQVdCZtB1l6dtRbD
-3hARd9eQ1Gq1XofKX6Lbc+7tQcmZbAKJbr3JjHJMDUORYk+o8hdK
------END RSA PRIVATE KEY-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIB8tAGuc9FP4XbYqMP67TKgjL7OTrACGgEmTf+zMvYRhoAoGCCqGSM49
+AwEHoUQDQgAESvX1TRh3NYeQeX6TjMhHcoYdfMTTXDVBkOYhizve2dCrSub0ZQ3n
+hNSNEmiYKqaZvThB8N9iVNBzIuBQWYtkbA==
+-----END EC PRIVATE KEY-----

pkg/event/target/testdata/nats_tls_client_cert.conf

@@ -13,6 +13,6 @@ authorization {
         subscribe = ">"
     }
     users = [
-        {user: "CN=localhost,OU=Client,O=Minio,C=CA", permissions: $ADMIN}
+        {user: "CN=localhost,OU=Client,O=MinIO,C=CA", permissions: $ADMIN}
     ]
 }

pkg/lsync/lrwmutex.go

@@ -19,10 +19,9 @@ package lsync
 import (
 	"context"
 	"math"
+	"math/rand"
 	"sync"
 	"time"
-
-	"github.com/minio/minio/pkg/retry"
 )
 
 // A LRWMutex is a mutual exclusion lock with timeouts.
@@ -94,25 +93,33 @@ func (lm *LRWMutex) lock(id, source string, isWriteLock bool) (locked bool) {
 	return locked
 }
 
+const (
+	lockRetryInterval = 50 * time.Millisecond
+)
+
 // lockLoop will acquire either a read or a write lock
 //
 // The call will block until the lock is granted using a built-in
 // timing randomized back-off algorithm to try again until successful
 func (lm *LRWMutex) lockLoop(ctx context.Context, id, source string, timeout time.Duration, isWriteLock bool) (locked bool) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	retryCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
-	// We timed out on the previous lock, incrementally wait
-	// for a longer back-off time and try again afterwards.
-	for range retry.NewTimer(retryCtx) {
-		if lm.lock(id, source, isWriteLock) {
-			return true
+	for {
+		select {
+		case <-retryCtx.Done():
+			// Caller context canceled or we timedout,
+			// return false anyways for both situations.
+			return false
+		default:
+			if lm.lock(id, source, isWriteLock) {
+				return true
+			}
+			time.Sleep(time.Duration(r.Float64() * float64(lockRetryInterval)))
 		}
 	}
-
-	// We timed out on the previous lock, incrementally wait
-	// for a longer back-off time and try again afterwards.
-	return false
 }
 
 // Unlock unlocks the write lock.

pkg/madmin/obd.go

@@ -25,6 +25,7 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/net"
 
@@ -42,6 +43,19 @@ type OBDInfo struct {
 	Perf      PerfOBDInfo  `json:"perf,omitempty"`
 	Minio     MinioOBDInfo `json:"minio,omitempty"`
 	Sys       SysOBDInfo   `json:"sys,omitempty"`
+	Logging   LogOBDInfo   `json:"logging,omitempty"`
+}
+
+// LogOBDInfo holds different type of logs of all clusters
+type LogOBDInfo struct {
+	ServersLog []ServerLogOBDInfo `json:"serverLogs"`
+}
+
+// ServerLogOBDInfo holds server's stdout log
+type ServerLogOBDInfo struct {
+	Addr    string      `json:"nodeName"`
+	Entries []log.Entry `json:"entries"`
+	Error   string      `json:"error,omitempty"`
 }
 
 // SysOBDInfo - Includes hardware and system information of the MinIO cluster
@@ -185,6 +199,7 @@ const (
 	OBDDataTypeSysMem      OBDDataType = "sysmem"
 	OBDDataTypeSysNet      OBDDataType = "sysnet"
 	OBDDataTypeSysProcess  OBDDataType = "sysprocess"
+	OBDDataTypeLog         OBDDataType = "log"
 )
 
 // OBDDataTypesMap - Map of OBD datatypes
@@ -201,6 +216,7 @@ var OBDDataTypesMap = map[string]OBDDataType{
 	"sysmem":      OBDDataTypeSysMem,
 	"sysnet":      OBDDataTypeSysNet,
 	"sysprocess":  OBDDataTypeSysProcess,
+	"log":         OBDDataTypeLog,
 }
 
 // OBDDataTypesList - List of OBD datatypes
@@ -217,6 +233,7 @@ var OBDDataTypesList = []OBDDataType{
 	OBDDataTypeSysMem,
 	OBDDataTypeSysNet,
 	OBDDataTypeSysProcess,
+	OBDDataTypeLog,
 }
 
 // ServerOBDInfo - Connect to a minio server and call OBD Info Management API

pkg/retry/retry.go

@@ -99,10 +99,10 @@ func NewTimerWithJitter(ctx context.Context, unit time.Duration, cap time.Durati
 		// Channel used to signal after the expiry of backoff wait seconds.
 		for {
 			select {
-			case attemptCh <- nextBackoff:
-				nextBackoff++
 			case <-ctx.Done():
 				return
+			case attemptCh <- nextBackoff:
+				nextBackoff++
 			}
 
 			t.Start(exponentialBackoffWait(nextBackoff))