fix: Incorrect ServersCount in cluster.info (#15431)

The `ServersCount` field in cluster.info is expected to contain the
number of nodes, and not number of endpoints.

.github/workflows/depsreview.yaml

@@ -0,0 +1,14 @@
+name: 'Dependency Review'
+on: [pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v1

.github/workflows/go-cross.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2

.github/workflows/go-healing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/go-lint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]

.github/workflows/replication.yaml

@@ -21,7 +21,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/upgrade-ci-cd.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x]
         os: [ubuntu-latest]
 
     steps:

.golangci.yml

@@ -14,13 +14,9 @@ linters:
     - govet
     - revive
     - ineffassign
-    - gosimple
     - deadcode
-    - structcheck
     - gomodguard
     - gofmt
-    - unused
-    - structcheck
     - unconvert
     - varcheck
     - gocritic

README.md

@@ -125,7 +125,7 @@ You can also connect using any S3-compatible tool, such as the MinIO Client `mc`
 
 ## Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.17](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go install github.com/minio/minio@latest

buildscripts/resolve-right-versions.sh

@@ -39,7 +39,7 @@ function start_minio_5drive() {
     "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
     pid=$!
     disown $pid
-    sleep 30
+    sleep 5
 
     if ! ps -p ${pid} 1>&2 >/dev/null; then
 	echo "server1 log:"

cmd/acl-handlers.go

@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/admin-bucket-handlers.go

@@ -69,7 +69,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -131,7 +131,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -173,7 +173,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -301,7 +301,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	}
 	if bucket != "" {
 		// Check if bucket exists.
-		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -340,7 +340,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -390,13 +390,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	)
 	if bucket != "" {
 		// Check if bucket exists.
-		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
 		buckets = append(buckets, BucketInfo{Name: bucket})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx)
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -684,7 +684,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				opts := BucketOptions{
+				opts := MakeBucketOptions{
 					LockEnabled: config.ObjectLockEnabled == "Enabled",
 				}
 				err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
@@ -750,7 +750,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				return
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+				err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 				if err != nil {
 					if _, ok := err.(BucketExists); !ok {
 						writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
@@ -817,7 +817,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
-			err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				if _, ok := err.(BucketExists); !ok {
 					writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
@@ -1089,7 +1089,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-site-replication.go

@@ -24,6 +24,8 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
@@ -116,19 +118,38 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
 		_, isForceCreate := r.Form["forceCreate"]
-		opts := BucketOptions{
+		createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
+		createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
+		if cerr != nil {
+			createdAt = timeSentinel
+		}
+
+		opts := MakeBucketOptions{
 			Location:          r.Form.Get("location"),
 			LockEnabled:       isLockEnabled,
 			VersioningEnabled: isVersioningEnabled,
 			ForceCreate:       isForceCreate,
+			CreatedAt:         createdAt,
 		}
 		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 	case madmin.ConfigureReplBktOp:
 		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
 	case madmin.DeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      false,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
 	case madmin.ForceDeleteBucketBktOp:
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
+		_, noRecreate := r.Form["noRecreate"]
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+			Force:      true,
+			NoRecreate: noRecreate,
+			SRDeleteOp: getSRBucketDeleteOp(true),
+		})
+	case madmin.PurgeDeletedBucketOp:
+		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -436,6 +457,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 	opts.Users = q.Get("users") == "true"
 	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
 	opts.EntityValue = q.Get("entityvalue")
+	opts.ShowDeleted = q.Get("showDeleted") == "true"
 	return
 }
 

cmd/admin-handlers-users.go

@@ -1177,7 +1177,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx)
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1189,17 +1189,32 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		// For derived credentials, check the parent user's permissions.
 		accountName = cred.ParentUser
 	}
-	policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 
-	buf, err := json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+	var buf []byte
+	if accountName == globalActiveCred.AccessKey {
+		for _, policy := range iampolicy.DefaultPolicies {
+			if policy.Name == "consoleAdmin" {
+				buf, err = json.MarshalIndent(policy.Definition, "", " ")
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				break
+			}
+		}
+	} else {
+		policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		buf, err = json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	acctInfo := madmin.AccountInfo{

cmd/admin-handlers.go

@@ -71,16 +71,6 @@ const (
 	mgmtForceStop   = "forceStop"
 )
 
-func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string, mode string) (us madmin.ServerUpdateStatus, err error) {
-	if err = doUpdate(u, lrTime, sha256Sum, releaseInfo, mode); err != nil {
-		return us, err
-	}
-
-	us.CurrentVersion = Version
-	us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout)
-	return us, nil
-}
-
 // ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
@@ -152,7 +142,16 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
+	// Download Binary Once
+	reader, err := downloadBinary(u, mode)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Push binary to other servers
+	for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader) {
 		if nerr.Err != nil {
 			err := AdminError{
 				Code:       AdminUpdateApplyFailure,
@@ -166,13 +165,39 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
+	err = verifyBinary(u, sha256Sum, releaseInfo, mode, reader)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
+	for _, nerr := range globalNotificationSys.CommitBinary(ctx) {
+		if nerr.Err != nil {
+			err := AdminError{
+				Code:       AdminUpdateApplyFailure,
+				Message:    nerr.Err.Error(),
+				StatusCode: http.StatusInternalServerError,
+			}
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	err = commitBinary()
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	updateStatus := madmin.ServerUpdateStatus{
+		CurrentVersion: Version,
+		UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+	}
+
 	// Marshal API response
 	jsonBytes, err := json.Marshal(updateStatus)
 	if err != nil {
@@ -1190,16 +1215,14 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		concurrent = 32
 	}
 
-	if runtime.GOMAXPROCS(0) < concurrent {
-		concurrent = runtime.GOMAXPROCS(0)
-	}
-
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
 		duration = time.Second * 10
 	}
 
-	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
+	storageInfo, _ := objectAPI.StorageInfo(ctx)
+
+	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 	if !sufficientCapacity {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
 			Code:       "XMinioSpeedtestInsufficientCapacity",
@@ -1272,7 +1295,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 }
 
 func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
-	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
+	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
@@ -1286,11 +1309,11 @@ func deleteObjectPerfBucket(objectAPI ObjectLayer) {
 	objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
 		Force:      true,
 		NoRecreate: true,
+		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 	})
 }
 
-func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
-	storageInfo, _ := objectAPI.StorageInfo(ctx)
+func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo, concurrent int, size int, autotune bool) (bool, bool, string) {
 	capacityNeeded := uint64(concurrent * size)
 	capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)
 
@@ -2199,15 +2222,13 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	getAndWriteObjPerfInfo := func() {
 		if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
 			concurrent := 32
-			if runtime.GOMAXPROCS(0) < concurrent {
-				concurrent = runtime.GOMAXPROCS(0)
-			}
+
+			storageInfo, _ := objectAPI.StorageInfo(ctx)
 
 			size := 64 * humanize.MiByte
 			autotune := true
 
-			sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
-
+			sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 			if !sufficientCapacity {
 				healthInfo.Perf.Error = capacityErrMsg
 				partialWrite(healthInfo)
@@ -2636,6 +2657,100 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	return nil
 }
 
+type clusterInfo struct {
+	DeploymentID string `json:"deployment_id"`
+	ClusterName  string `json:"cluster_name"`
+	UsedCapacity uint64 `json:"used_capacity"`
+	Info         struct {
+		MinioVersion    string `json:"minio_version"`
+		PoolsCount      int    `json:"pools_count"`
+		ServersCount    int    `json:"servers_count"`
+		DrivesCount     int    `json:"drives_count"`
+		BucketsCount    uint64 `json:"buckets_count"`
+		ObjectsCount    uint64 `json:"objects_count"`
+		TotalDriveSpace uint64 `json:"total_drive_space"`
+		UsedDriveSpace  uint64 `json:"used_drive_space"`
+	} `json:"info"`
+}
+
+func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
+	// Send profiling data to zip as file
+	header, zerr := zip.FileInfoHeader(dummyFileInfo{
+		name:    name,
+		size:    int64(len(data)),
+		mode:    0o600,
+		modTime: UTCNow(),
+		isDir:   false,
+		sys:     nil,
+	})
+	if zerr != nil {
+		return zerr
+	}
+	header.Method = zip.Deflate
+	zwriter, zerr := zipWriter.CreateHeader(header)
+	if zerr != nil {
+		return zerr
+	}
+	_, err := io.Copy(zwriter, bytes.NewReader(data))
+	return err
+}
+
+// appendClusterMetaInfoToZip gets information of the current cluster and embedded
+// it in the passed zipwriter, This is not a critical function and it is allowed
+// to fail with a ten seconds timeout.
+func appendClusterMetaInfoToZip(ctx context.Context, zipWriter *zip.Writer) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return
+	}
+
+	// Add a ten seconds timeout because getting profiling data
+	// is critical for debugging, in contrary to getting cluster info
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	resultCh := make(chan clusterInfo)
+	go func() {
+		ci := clusterInfo{}
+		ci.Info.PoolsCount = len(globalEndpoints)
+		ci.Info.ServersCount = len(globalEndpoints.Hostnames())
+		ci.Info.MinioVersion = Version
+
+		si, _ := objectAPI.StorageInfo(ctx)
+
+		ci.Info.DrivesCount = len(si.Disks)
+		for _, disk := range si.Disks {
+			ci.Info.TotalDriveSpace += disk.TotalSpace
+			ci.Info.UsedDriveSpace += disk.UsedSpace
+		}
+
+		dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
+
+		ci.UsedCapacity = dataUsageInfo.ObjectsTotalSize
+		ci.Info.BucketsCount = dataUsageInfo.BucketsCount
+		ci.Info.ObjectsCount = dataUsageInfo.ObjectsTotalCount
+
+		ci.DeploymentID = globalDeploymentID
+		ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.ServersCount, ci.Info.DrivesCount, ci.Info.MinioVersion)
+		resultCh <- ci
+	}()
+
+	select {
+	case <-ctx.Done():
+		return
+	case ci := <-resultCh:
+		out, err := json.MarshalIndent(ci, "", "   ")
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return
+		}
+		err = embedFileInZip(zipWriter, "cluster.info", out)
+		if err != nil {
+			logger.LogIf(ctx, err)
+		}
+	}
+}
+
 // getRawDataer provides an interface for getting raw FS files.
 type getRawDataer interface {
 	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error
@@ -2655,7 +2770,8 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	o, ok := newObjectLayerFn().(getRawDataer)
+	objLayer := newObjectLayerFn()
+	o, ok := objLayer.(getRawDataer)
 	if !ok {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
@@ -2778,6 +2894,8 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}); err != nil {
 		logger.LogIf(ctx, err)
 	}
+
+	appendClusterMetaInfoToZip(ctx, zipWriter)
 }
 
 func createHostAnonymizerForFSMode() map[string]string {

cmd/admin-heal-ops.go

@@ -822,7 +822,7 @@ func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	buckets, err := objAPI.ListBuckets(h.ctx)
+	buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{})
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}

cmd/api-headers.go

@@ -199,5 +199,12 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 	}
 
+	if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok {
+		if i := strings.LastIndexByte(v, '/'); i >= 0 {
+			v = v[i+1:]
+		}
+		w.Header()[xhttp.MinIOCompressed] = []string{v}
+	}
+
 	return nil
 }

cmd/background-newdisks-heal-ops.go

@@ -318,8 +318,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	ctx = lkctx.Context()
 	defer locker.Unlock(lkctx.Cancel)
 
-	buckets, _ := z.ListBuckets(ctx)
-
+	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to heal all bucket metadata configuration.
 	buckets = append(buckets, BucketInfo{

cmd/benchmark-utils_test.go

@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/bitrot_test.go

@@ -20,17 +20,11 @@ package cmd
 import (
 	"context"
 	"io"
-	"io/ioutil"
-	"os"
 	"testing"
 )
 
 func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
-	tmpDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
+	tmpDir := t.TempDir()
 
 	volume := "testvol"
 	filePath := "testfile"
@@ -60,7 +54,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	writer.(io.Closer).Close()
+	if bw, ok := writer.(io.Closer); ok {
+		bw.Close()
+	}
 
 	reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
 	b := make([]byte, 10)
@@ -76,6 +72,9 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	if _, err = reader.ReadAt(b[:5], 30); err != nil {
 		t.Fatal(err)
 	}
+	if br, ok := reader.(io.Closer); ok {
+		br.Close()
+	}
 }
 
 func TestAllBitrotAlgorithms(t *testing.T) {

cmd/bucket-encryption-handlers.go

@@ -65,7 +65,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -155,7 +155,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -198,7 +198,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 
 	// Check if bucket exists
 	var err error
-	if _, err = objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-handlers.go

@@ -201,7 +201,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -332,7 +332,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	} else {
 		// Invoke the list buckets.
 		var err error
-		bucketsInfo, err = listBuckets(ctx)
+		bucketsInfo, err = listBuckets(ctx, BucketOptions{})
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -447,7 +447,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
 
 	// Before proceeding validate if bucket exists.
-	_, err := objectAPI.GetBucketInfo(ctx, bucket)
+	_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -746,7 +746,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	opts := BucketOptions{
+	opts := MakeBucketOptions{
 		Location:    location,
 		LockEnabled: objectLockEnabled,
 		ForceCreate: forceCreate,
@@ -765,7 +765,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
+					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+						Force:      false,
+						NoRecreate: true,
+						SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+					})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
@@ -1165,7 +1169,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1228,7 +1232,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 	getBucketInfo := objectAPI.GetBucketInfo
 
-	if _, err := getBucketInfo(ctx, bucket); err != nil {
+	if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
@@ -1305,7 +1309,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
+	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{
+		Force:      forceDelete,
+		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
+	}); err != nil {
 		apiErr := toAPIError(ctx, err)
 		if _, ok := err.(BucketNotEmpty); ok {
 			if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1472,7 +1479,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle-handlers.go

@@ -62,7 +62,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -163,7 +163,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-lifecycle.go

@@ -560,7 +560,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 	}
 	// Check if bucket exists.
 	if !r.OutputLocation.IsEmpty() {
-		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName); err != nil {
+		if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName, BucketOptions{}); err != nil {
 			return err
 		}
 		if r.OutputLocation.S3.Prefix == "" {

cmd/bucket-metadata.go

@@ -105,10 +105,8 @@ type BucketMetadata struct {
 
 // newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
 func newBucketMetadata(name string) BucketMetadata {
-	now := UTCNow()
 	return BucketMetadata{
-		Name:    name,
-		Created: now,
+		Name: name,
 		notificationConfig: &event.Config{
 			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
 		},
@@ -121,6 +119,17 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }
 
+// SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
+// creation time of bucket on this cluster in all other cases.
+func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
+	if b.Created.IsZero() {
+		b.Created = UTCNow()
+	}
+	if !createdAt.IsZero() {
+		b.Created = createdAt.UTC()
+	}
+}
+
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
@@ -160,17 +169,19 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
-
+	if err == nil {
+		b.defaultTimestamps()
+	}
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
 	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
 		return b, err
 	}
 
 	// migrate unencrypted remote targets
-	if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
+	if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
 		return b, err
 	}
-	b.defaultTimestamps()
+
 	return b, nil
 }
 
@@ -345,6 +356,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.BucketTargetsConfigJSON = configData
 		}
 	}
+	b.defaultTimestamps()
 
 	if err := b.Save(ctx, objectAPI); err != nil {
 		return err

cmd/bucket-notification-handlers.go

@@ -60,7 +60,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -132,7 +132,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
+	_, err := objectAPI.GetBucketInfo(ctx, bucketName, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-policy-handlers.go

@@ -61,7 +61,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -191,7 +191,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Check if bucket exists.
-	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-policy-handlers_test.go

@@ -128,7 +128,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
 			defer wg.Done()
 			// Sync start.
 			<-start
-			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 				if _, ok := err.(BucketExists); !ok {
 					t.Logf("unexpected error: %T: %v", err, err)
 					return
@@ -163,7 +163,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T,
 ) {
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 

cmd/bucket-replication-handlers.go

@@ -55,7 +55,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -118,7 +118,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -157,7 +157,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		return
 	}
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -198,7 +198,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -267,7 +267,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -356,7 +356,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-replication.go

@@ -1492,7 +1492,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), string(replicationSubsystem))
 	}
 }
 
@@ -1518,7 +1518,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		})
 	case ch <- ri:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
 	}
 }
 
@@ -1555,7 +1555,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing number of replicate workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
 	}
 }
 
@@ -1805,6 +1805,7 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli
 	objInfo.VersionPurgeStatusInternal = ""
 	objInfo.ReplicationStatus = ""
 	objInfo.VersionPurgeStatus = ""
+	delete(objInfo.UserDefined, xhttp.AmzBucketReplicationStatus)
 	resyncdsc := mustReplicate(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(objInfo, replication.ExistingObjectReplicationType, ObjectOptions{}))
 	dsc = &resyncdsc
 	return c.resync(oi, dsc, tgtStatuses)

cmd/bucket-versioning-handler.go

@@ -143,7 +143,7 @@ func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r
 	}
 
 	// Check if bucket exists.
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/common-main.go

@@ -264,7 +264,7 @@ func initConsoleServer() (*restapi.Server, error) {
 		return nil, err
 	}
 
-	api := operations.NewConsoleAPI(swaggerSpec, buildOpenIDConsoleConfig())
+	api := operations.NewConsoleAPI(swaggerSpec)
 
 	if !serverDebugLog {
 		// Disable console logging if server debug log is not enabled
@@ -275,6 +275,12 @@ func initConsoleServer() (*restapi.Server, error) {
 		api.Logger = noLog
 	}
 
+	// Pass in console application config. This needs to happen before the
+	// ConfigureAPI() call.
+	restapi.GlobalMinIOConfig = restapi.MinIOConfig{
+		OpenIDProviders: buildOpenIDConsoleConfig(),
+	}
+
 	server := restapi.NewServer(api)
 	// register all APIs
 	server.ConfigureAPI()

cmd/config-migrate_test.go

@@ -37,11 +37,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Init Test config failed")
 	}
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 	globalConfigDir = &ConfigDir{path: rootPath}
 
 	globalObjLayerMutex.Lock()
@@ -74,13 +70,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 // Test if all migrate code returns nil when config file does not
 // exist
 func TestServerConfigMigrateInexistentConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	globalConfigDir = &ConfigDir{path: rootPath}
+	globalConfigDir = &ConfigDir{path: t.TempDir()}
 
 	if err := migrateV2ToV3(); err != nil {
 		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
@@ -164,11 +154,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 
 // Test if a config migration from v2 to v33 is successfully done
 func TestServerConfigMigrateV2toV33(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 
@@ -234,11 +220,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateFaultyConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
@@ -331,35 +313,31 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 
 // Test if all migrate code returns error with corrupted config files
 func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
-	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	globalConfigDir = &ConfigDir{path: rootPath}
 	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	for i := 3; i <= 17; i++ {
 		// Create a corrupted config file
-		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
+		if err := ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
 			0o644); err != nil {
 			t.Fatal("Unexpected error: ", err)
 		}
 
 		// Test different migrate versions and be sure they are returning an error
-		if err = migrateConfig(); err == nil {
+		if err := migrateConfig(); err == nil {
 			t.Fatal("migrateConfig() should fail with a corrupted json")
 		}
 	}
 
 	// Create a corrupted config file for version '2'.
-	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test different migrate versions and be sure they are returning an error
-	if err = migrateConfig(); err == nil {
+	if err := migrateConfig(); err == nil {
 		t.Fatal("migrateConfig() should fail with a corrupted json")
 	}
 }

cmd/data-update-tracker_test.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path"
@@ -100,15 +99,11 @@ func TestDataUpdateTracker(t *testing.T) {
 
 	dut.Current.bf = dut.newBloomFilter()
 
-	tmpDir, err := ioutil.TempDir("", "TestDataUpdateTracker")
+	tmpDir := t.TempDir()
+	err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	dut.start(ctx, tmpDir)

cmd/data-usage_test.go

@@ -35,12 +35,8 @@ type usageTestFile struct {
 }
 
 func TestDataUsageUpdate(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUsageUpdate")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	const bucket = "bucket"
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},
@@ -251,12 +247,8 @@ func TestDataUsageUpdate(t *testing.T) {
 }
 
 func TestDataUsageUpdatePrefix(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUpdateUsagePrefix")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	scannerSleeper.Update(0, 0)
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "bucket/rootfile", size: 10000},
 		{name: "bucket/rootfile2", size: 10000},
@@ -537,12 +529,8 @@ func generateUsageTestFiles(t *testing.T, base, bucket string, nFolders, nFiles,
 }
 
 func TestDataUsageCacheSerialize(t *testing.T) {
-	base, err := ioutil.TempDir("", "TestDataUsageCacheSerialize")
-	if err != nil {
-		t.Skip(err)
-	}
+	base := t.TempDir()
 	const bucket = "abucket"
-	defer os.RemoveAll(base)
 	files := []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},

cmd/dummy-handlers.go

@@ -52,7 +52,7 @@ func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *h
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -84,7 +84,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -117,7 +117,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -151,7 +151,7 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -189,7 +189,7 @@ func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http
 	}
 
 	// Validate if bucket exists, before proceeding further...
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/endpoint.go

@@ -292,6 +292,19 @@ func (l EndpointServerPools) LocalDisksPaths() []string {
 	return disks
 }
 
+// NLocalDisksPathsPerPool returns the disk paths of the local disks per pool
+func (l EndpointServerPools) NLocalDisksPathsPerPool() []int {
+	localDisksCount := make([]int, len(l))
+	for i, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				localDisksCount[i]++
+			}
+		}
+	}
+	return localDisksCount
+}
+
 // FirstLocal returns true if the first endpoint is local.
 func (l EndpointServerPools) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
@@ -302,7 +315,7 @@ func (l EndpointServerPools) HTTPS() bool {
 	return l[0].Endpoints.HTTPS()
 }
 
-// NEndpoints - returns all nodes count
+// NEndpoints - returns number of endpoints
 func (l EndpointServerPools) NEndpoints() (count int) {
 	for _, ep := range l {
 		count += len(ep.Endpoints)

cmd/erasure-bucket.go

@@ -35,7 +35,7 @@ var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
 // Bucket operations
 
 // MakeBucket - make a bucket.
-func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Verify if bucket is valid.
@@ -94,7 +94,7 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
 }
 
 // getBucketInfo - returns the BucketInfo from one of the load balanced disks.
-func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) {
+func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -108,9 +108,17 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 			}
 			volInfo, err := storageDisks[index].StatVol(ctx, bucketName)
 			if err != nil {
+				if opts.Deleted {
+					dvi, derr := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucketName))
+					if derr != nil {
+						return err
+					}
+					bucketsInfo[index] = BucketInfo{Name: bucketName, Deleted: dvi.Created}
+					return nil
+				}
 				return err
 			}
-			bucketsInfo[index] = BucketInfo(volInfo)
+			bucketsInfo[index] = BucketInfo{Name: volInfo.Name, Created: volInfo.Created}
 			return nil
 		}, index)
 	}
@@ -131,8 +139,8 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
-func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketInfo, err := er.getBucketInfo(ctx, bucket)
+func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
+	bucketInfo, err := er.getBucketInfo(ctx, bucket, opts)
 	if err != nil {
 		return bi, toObjectErr(err, bucket)
 	}
@@ -191,12 +199,58 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts D
 		if purgedDangling {
 			err = nil
 		}
-
+		if opts.SRDeleteOp == MarkDelete {
+			er.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+		}
 	}
 
 	return toObjectErr(err, bucket)
 }
 
+// markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
+// syncs the delete to peers
+func (er erasureObjects) markDelete(ctx context.Context, bucket, prefix string) error {
+	storageDisks := er.getDisks()
+	g := errgroup.WithNErrs(len(storageDisks))
+	// Make a volume entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		if storageDisks[index] == nil {
+			continue
+		}
+		g.Go(func() error {
+			if err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, prefix)); err != nil {
+				if errors.Is(err, errVolumeExists) {
+					return nil
+				}
+				return err
+			}
+			return nil
+		}, index)
+	}
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
+	return toObjectErr(err, bucket)
+}
+
+// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
+// syncs the delete to peers OR on a new MakeBucket call.
+func (er erasureObjects) purgeDelete(ctx context.Context, bucket, prefix string) error {
+	storageDisks := er.getDisks()
+	g := errgroup.WithNErrs(len(storageDisks))
+	// Make a volume entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if storageDisks[index] != nil {
+				return storageDisks[index].DeleteVol(ctx, pathJoin(bucket, prefix), true)
+			}
+			return errDiskNotFound
+		}, index)
+	}
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
+	return toObjectErr(err, bucket)
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (er erasureObjects) IsNotificationSupported() bool {
 	return true

cmd/erasure-decode_test.go

@@ -85,19 +85,17 @@ var erasureDecodeTests = []struct {
 
 func TestErasureDecode(t *testing.T) {
 	for i, test := range erasureDecodeTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		disks := setup.disks
 		data := make([]byte, test.data)
 		if _, err = io.ReadFull(crand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
 
@@ -113,11 +111,9 @@ func TestErasureDecode(t *testing.T) {
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
 		}
 		if n != test.data {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file", i)
 		}
 		for i, w := range writers {
@@ -195,7 +191,6 @@ func TestErasureDecode(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }
 
@@ -210,12 +205,11 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 	dataBlocks := 7
 	parityBlocks := 7
 	blockSize := int64(1 * humanize.MiByte)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	setup, err := newErasureTestSetup(t, dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Error(err)
 		return
 	}
-	defer setup.Remove()
 	disks := setup.disks
 	erasure, err := NewErasure(context.Background(), dataBlocks, parityBlocks, blockSize)
 	if err != nil {
@@ -288,11 +282,10 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 // Benchmarks
 
 func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV2)
+	setup, err := newErasureTestSetup(b, data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
-	defer setup.Remove()
 	disks := setup.disks
 	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2)
 	if err != nil {

cmd/erasure-encode_test.go

@@ -87,21 +87,19 @@ var erasureEncodeTests = []struct {
 
 func TestErasureEncode(t *testing.T) {
 	for i, test := range erasureEncodeTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		disks := setup.disks
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 
 		data := make([]byte, test.data)
 		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
 		}
 		writers := make([]io.Writer, len(disks))
@@ -160,18 +158,16 @@ func TestErasureEncode(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }
 
 // Benchmarks
 
 func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV2)
+	setup, err := newErasureTestSetup(b, data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
-	defer setup.Remove()
 	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)

cmd/erasure-heal_test.go

@@ -70,19 +70,17 @@ func TestErasureHeal(t *testing.T) {
 		}
 
 		// create some test data
-		setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		setup, err := newErasureTestSetup(t, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to setup Erasure environment: %v", i, err)
 		}
 		disks := setup.disks
 		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
 		data := make([]byte, test.size)
 		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
@@ -93,7 +91,6 @@ func TestErasureHeal(t *testing.T) {
 		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
 		if err != nil {
-			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 
@@ -156,6 +153,5 @@ func TestErasureHeal(t *testing.T) {
 				}
 			}
 		}
-		setup.Remove()
 	}
 }

cmd/erasure-healing-common_test.go

@@ -176,7 +176,7 @@ func TestListOnlineDisks(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -351,7 +351,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -472,7 +472,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	z := obj.(*erasureServerPools)
 	s := z.serverPools[0].sets[0]
 	erasureDisks := s.getDisks()
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}

cmd/erasure-healing_test.go

@@ -236,7 +236,7 @@ func TestHealing(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -359,7 +359,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	object := getRandomObjectName()
 	data := bytes.Repeat([]byte("a"), 128*1024)
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -554,7 +554,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -685,7 +685,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -861,7 +861,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1004,7 +1004,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1161,7 +1161,7 @@ func TestHealObjectErasure(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1258,7 +1258,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	object := "empty-dir/"
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1359,7 +1359,7 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 			var opts ObjectOptions
 
-			err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				t.Fatalf("Failed to make a bucket - %v", err)
 			}

cmd/erasure-object.go

@@ -534,6 +534,11 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		return metaFileInfos, errs
 	}
 
+	versionID := lfi.VersionID
+	if versionID == "" {
+		versionID = nullVersionID
+	}
+
 	for index := range metadataArray {
 		if metadataArray[index] == nil {
 			continue
@@ -541,21 +546,15 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 
 		// make sure to preserve this for diskmtime based healing bugfix.
 		diskMTime := metaFileInfos[index].DiskMTime
-		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, "")
+		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID)
 		if errs[index] != nil {
 			continue
 		}
 
-		if metaFileInfos[index].IsValid() && metaFileInfos[index].ModTime.Equal(lfi.ModTime) {
-			versionID := metaFileInfos[index].VersionID
-			if versionID == "" {
-				versionID = nullVersionID
-			}
-			if readData {
-				metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
-			}
-			metaFileInfos[index].DiskMTime = diskMTime
+		if readData {
+			metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
 		}
+		metaFileInfos[index].DiskMTime = diskMTime
 	}
 
 	// Return all the metadata.

cmd/erasure-object_test.go

@@ -53,7 +53,7 @@ func TestRepeatPutObjectPart(t *testing.T) {
 	defer objLayer.Shutdown(context.Background())
 	defer removeRoots(disks)
 
-	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,7 +101,7 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	}
 	defer xl.Shutdown(context.Background())
 
-	err = xl.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = xl.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,7 +155,7 @@ func TestDeleteObjectsVersioned(t *testing.T) {
 		{bucketName, "dir/obj1"},
 	}
 
-	err = obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{
+	err = obj.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{
 		VersioningEnabled: true,
 	})
 	if err != nil {
@@ -244,7 +244,7 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 		{bucketName, "obj_4"},
 	}
 
-	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, BucketOptions{})
+	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -305,7 +305,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -374,7 +374,7 @@ func TestErasureDeleteObjectDiskNotFoundErasure4(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -434,7 +434,7 @@ func TestErasureDeleteObjectDiskNotFoundErr(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +505,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -614,7 +614,7 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -692,7 +692,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -755,7 +755,7 @@ func TestPutObjectNoQuorumSmall(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -824,7 +824,7 @@ func TestPutObjectSmallInlineData(t *testing.T) {
 	object := "object"
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -892,7 +892,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	ctx, cancel := context.WithCancel(GlobalContext)
 	defer cancel()
 
-	err := obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -1073,10 +1073,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	// Create a backend with 4 disks named disk{1...4}, this name convention
 	// because we will unzip some object data from a sample archive.
 	const numDisks = 4
-	path, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
+	path := t.TempDir()
 
 	var fsDirs []string
 	for i := 1; i <= numDisks; i++ {
@@ -1094,7 +1091,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Create a testbucket
-	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", BucketOptions{})
+	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1160,7 +1157,7 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
 
 	for i, testCase := range testCases {
 		// Step 1: create a bucket
-		err = z.MakeBucketWithLocation(ctx, testCase.bucket, BucketOptions{VersioningEnabled: testCase.versioned})
+		err = z.MakeBucketWithLocation(ctx, testCase.bucket, MakeBucketOptions{VersioningEnabled: testCase.versioned})
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create a bucket: %v", i+1, err)
 		}

cmd/erasure-server-pool-decom.go

@@ -1119,7 +1119,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		return errInvalidArgument
 	}
 
-	buckets, err := z.ListBuckets(ctx)
+	buckets, err := z.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}
@@ -1157,7 +1157,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		pathJoin(minioMetaBucket, bucketMetaPrefix),
 	} {
 		var bucketExists BucketExists
-		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
+		if err = z.MakeBucketWithLocation(ctx, metaBucket, MakeBucketOptions{}); err != nil {
 			if !errors.As(err, &bucketExists) {
 				return err
 			}

cmd/erasure-server-pool.go

@@ -533,9 +533,11 @@ func (z *erasureServerPools) BackendInfo() (b madmin.BackendInfo) {
 	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
 
 	// Data blocks can vary per pool, but parity is same.
-	for _, setDriveCount := range z.SetDriveCounts() {
+	for i, setDriveCount := range z.SetDriveCounts() {
 		b.StandardSCData = append(b.StandardSCData, setDriveCount-scParity)
 		b.RRSCData = append(b.RRSCData, setDriveCount-rrSCParity)
+		b.DrivesPerSet = append(b.DrivesPerSet, setDriveCount)
+		b.TotalSets = append(b.TotalSets, z.serverPools[i].setCount)
 	}
 
 	b.StandardSCParity = scParity
@@ -613,7 +615,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 	var results []dataUsageCache
 	var firstErr error
 
-	allBuckets, err := z.ListBuckets(ctx)
+	allBuckets, err := z.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}
@@ -718,7 +720,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 // MakeBucketWithLocation - creates a new bucket across all serverPools simultaneously
 // even if one of the sets fail to create buckets, we proceed all the successful
 // operations.
-func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	g := errgroup.WithNErrs(len(z.serverPools))
 
 	// Lock the bucket name before creating.
@@ -758,6 +760,7 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 
 	// If it doesn't exist we get a new, so ignore errors
 	meta := newBucketMetadata(bucket)
+	meta.SetCreatedAt(opts.CreatedAt)
 	if opts.LockEnabled {
 		meta.VersioningConfigXML = enabledBucketVersioningConfig
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
@@ -1539,9 +1542,9 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 }
 
 // GetBucketInfo - returns bucket info from one of the erasure coded serverPools.
-func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
+func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
 	if z.SinglePool() {
-		bucketInfo, err = z.serverPools[0].GetBucketInfo(ctx, bucket)
+		bucketInfo, err = z.serverPools[0].GetBucketInfo(ctx, bucket, opts)
 		if err != nil {
 			return bucketInfo, err
 		}
@@ -1552,7 +1555,7 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (
 		return bucketInfo, nil
 	}
 	for _, pool := range z.serverPools {
-		bucketInfo, err = pool.GetBucketInfo(ctx, bucket)
+		bucketInfo, err = pool.GetBucketInfo(ctx, bucket, opts)
 		if err != nil {
 			if isErrBucketNotFound(err) {
 				continue
@@ -1626,7 +1629,11 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 
 	// Purge the entire bucket metadata entirely.
 	z.renameAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
-
+	// If site replication is configured, hold on to deleted bucket state until sites sync
+	switch opts.SRDeleteOp {
+	case MarkDelete:
+		z.markDelete(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+	}
 	// Success.
 	return nil
 }
@@ -1644,6 +1651,27 @@ func (z *erasureServerPools) renameAll(ctx context.Context, bucket, prefix strin
 	}
 }
 
+// markDelete will create a directory of deleted bucket in .minio.sys/buckets/.deleted across all disks
+// in situations where the deleted bucket needs to be held on to until all sites are in sync for
+// site replication
+func (z *erasureServerPools) markDelete(ctx context.Context, bucket, prefix string) {
+	for _, servers := range z.serverPools {
+		for _, set := range servers.sets {
+			set.markDelete(ctx, bucket, prefix)
+		}
+	}
+}
+
+// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
+// syncs the delete to peers.
+func (z *erasureServerPools) purgeDelete(ctx context.Context, bucket, prefix string) {
+	for _, servers := range z.serverPools {
+		for _, set := range servers.sets {
+			set.purgeDelete(ctx, bucket, prefix)
+		}
+	}
+}
+
 // This function is used to undo a successful DeleteBucket operation.
 func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools []*erasureSets, errs []error) {
 	g := errgroup.WithNErrs(len(serverPools))
@@ -1653,7 +1681,7 @@ func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools
 		index := index
 		g.Go(func() error {
 			if errs[index] == nil {
-				return serverPools[index].MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+				return serverPools[index].MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			}
 			return nil
 		}, index)
@@ -1665,15 +1693,15 @@ func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools
 // List all buckets from one of the serverPools, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all serverPools.
-func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	if z.SinglePool() {
-		buckets, err = z.serverPools[0].ListBuckets(ctx)
+		buckets, err = z.serverPools[0].ListBuckets(ctx, opts)
 	} else {
 		for idx, pool := range z.serverPools {
 			if z.IsSuspended(idx) {
 				continue
 			}
-			buckets, err = pool.ListBuckets(ctx)
+			buckets, err = pool.ListBuckets(ctx, opts)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				continue
@@ -1685,9 +1713,9 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketI
 		return nil, err
 	}
 	for i := range buckets {
-		meta, err := globalBucketMetadataSys.Get(buckets[i].Name)
+		createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
 		if err == nil {
-			buckets[i].Created = meta.Created
+			buckets[i].Created = createdAt
 		}
 	}
 	return buckets, nil

cmd/erasure-sets.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"reflect"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -689,7 +690,7 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 
 // MakeBucketLocation - creates a new bucket across all sets simultaneously,
 // then return the first encountered error
-func (s *erasureSets) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (s *erasureSets) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	g := errgroup.WithNErrs(len(s.sets))
 
 	// Create buckets in parallel across all sets.
@@ -760,8 +761,8 @@ func (s *erasureSets) getHashedSet(input string) (set *erasureObjects) {
 }
 
 // GetBucketInfo - returns bucket info from one of the erasure coded set.
-func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
-	return s.getHashedSet("").GetBucketInfo(ctx, bucket)
+func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
+	return s.getHashedSet("").GetBucketInfo(ctx, bucket, opts)
 }
 
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
@@ -825,7 +826,7 @@ func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObj
 		index := index
 		g.Go(func() error {
 			if errs[index] == nil {
-				return sets[index].MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+				return sets[index].MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
 			}
 			return nil
 		}, index)
@@ -837,7 +838,7 @@ func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObj
 // List all buckets from one of the set, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all sets.
-func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+func (s *erasureSets) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	var listBuckets []BucketInfo
 	healBuckets := map[string]VolInfo{}
 	for _, set := range s.sets {
@@ -847,8 +848,34 @@ func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, er
 		}
 	}
 
+	// include deleted buckets in listBuckets output
+	deletedBuckets := map[string]VolInfo{}
+
+	if opts.Deleted {
+		for _, set := range s.sets {
+			// lists all unique buckets across drives.
+			if err := listDeletedBuckets(ctx, set.getDisks(), deletedBuckets, s.defaultParityCount); err != nil {
+				return nil, err
+			}
+		}
+	}
 	for _, v := range healBuckets {
-		listBuckets = append(listBuckets, BucketInfo(v))
+		bi := BucketInfo{
+			Name:    v.Name,
+			Created: v.Created,
+		}
+		if vi, ok := deletedBuckets[v.Name]; ok {
+			bi.Deleted = vi.Created
+		}
+		listBuckets = append(listBuckets, bi)
+	}
+	for _, v := range deletedBuckets {
+		if _, ok := healBuckets[v.Name]; !ok {
+			listBuckets = append(listBuckets, BucketInfo{
+				Name:    v.Name,
+				Deleted: v.Created,
+			})
+		}
 	}
 
 	sort.Slice(listBuckets, func(i, j int) bool {
@@ -858,6 +885,45 @@ func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, er
 	return listBuckets, nil
 }
 
+// listDeletedBuckets lists deleted buckets from all disks.
+func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBuckets map[string]VolInfo, readQuorum int) error {
+	g := errgroup.WithNErrs(len(storageDisks))
+	var mu sync.Mutex
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if storageDisks[index] == nil {
+				// we ignore disk not found errors
+				return nil
+			}
+			volsInfo, err := storageDisks[index].ListDir(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix), -1)
+			if err != nil {
+				if err == errFileNotFound {
+					return nil
+				}
+				return err
+			}
+			for _, volName := range volsInfo {
+				mu.Lock()
+				if _, ok := delBuckets[volName]; !ok {
+					vi, err := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, volName))
+					if err != nil {
+						return err
+					}
+					bkt := strings.TrimPrefix(vi.Name, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix))
+					bkt = strings.TrimPrefix(bkt, slashSeparator)
+					bkt = strings.TrimSuffix(bkt, slashSeparator)
+					vi.Name = bkt
+					delBuckets[bkt] = vi
+				}
+				mu.Unlock()
+			}
+			return nil
+		}, index)
+	}
+	return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum)
+}
+
 // --- Object Operations ---
 
 // GetObjectNInfo - returns object info and locked object ReadCloser

cmd/erasure-single-drive.go

@@ -108,16 +108,40 @@ func newErasureSingle(ctx context.Context, storageDisk StorageAPI, format *forma
 // List all buckets from one of the set, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all sets.
-func (es *erasureSingle) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
+func (es *erasureSingle) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	var listBuckets []BucketInfo
 	healBuckets := map[string]VolInfo{}
 	// lists all unique buckets across drives.
 	if err := listAllBuckets(ctx, []StorageAPI{es.disk}, healBuckets, 0); err != nil {
 		return nil, err
 	}
+	// include deleted buckets in listBuckets output
+	deletedBuckets := map[string]VolInfo{}
 
+	if opts.Deleted {
+		// lists all deleted buckets across drives.
+		if err := listDeletedBuckets(ctx, []StorageAPI{es.disk}, deletedBuckets, 0); err != nil {
+			return nil, err
+		}
+	}
 	for _, v := range healBuckets {
-		listBuckets = append(listBuckets, BucketInfo(v))
+		bi := BucketInfo{
+			Name:    v.Name,
+			Created: v.Created,
+		}
+		if vi, ok := deletedBuckets[v.Name]; ok {
+			bi.Deleted = vi.Created
+		}
+		listBuckets = append(listBuckets, bi)
+	}
+
+	for _, v := range deletedBuckets {
+		if _, ok := healBuckets[v.Name]; !ok {
+			listBuckets = append(listBuckets, BucketInfo{
+				Name:    v.Name,
+				Deleted: v.Created,
+			})
+		}
 	}
 
 	sort.Slice(listBuckets, func(i, j int) bool {
@@ -256,7 +280,7 @@ type renameAllStorager interface {
 
 // Bucket operations
 // MakeBucket - make a bucket.
-func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Lock the bucket name before creating.
@@ -289,6 +313,7 @@ func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket stri
 
 	// If it doesn't exist we get a new, so ignore errors
 	meta := newBucketMetadata(bucket)
+	meta.SetCreatedAt(opts.CreatedAt)
 	if opts.LockEnabled {
 		meta.VersioningConfigXML = enabledBucketVersioningConfig
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
@@ -308,12 +333,17 @@ func (es *erasureSingle) MakeBucketWithLocation(ctx context.Context, bucket stri
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
-func (es *erasureSingle) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
+func (es *erasureSingle) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
 	volInfo, err := es.disk.StatVol(ctx, bucket)
 	if err != nil {
+		if opts.Deleted {
+			if dvi, derr := es.disk.StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket)); derr == nil {
+				return BucketInfo{Name: bucket, Deleted: dvi.Created}, nil
+			}
+		}
 		return bi, toObjectErr(err, bucket)
 	}
-	return BucketInfo(volInfo), nil
+	return BucketInfo{Name: volInfo.Name, Created: volInfo.Created}, nil
 }
 
 // DeleteBucket - deletes a bucket.
@@ -326,6 +356,28 @@ func (es *erasureSingle) DeleteBucket(ctx context.Context, bucket string, opts D
 	deleteBucketMetadata(ctx, es, bucket)
 	globalBucketMetadataSys.Remove(bucket)
 
+	if err == nil || errors.Is(err, errVolumeNotFound) {
+		if opts.SRDeleteOp == MarkDelete {
+			es.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+		}
+	}
+	return toObjectErr(err, bucket)
+}
+
+// markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
+// syncs the delete to peers
+func (es *erasureSingle) markDelete(ctx context.Context, bucket, prefix string) error {
+	err := es.disk.MakeVol(ctx, pathJoin(bucket, prefix))
+	if err != nil && errors.Is(err, errVolumeExists) {
+		return nil
+	}
+	return toObjectErr(err, bucket)
+}
+
+// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
+// syncs the delete to peers OR on a new MakeBucket call.
+func (es *erasureSingle) purgeDelete(ctx context.Context, bucket, prefix string) error {
+	err := es.disk.DeleteVol(ctx, pathJoin(bucket, prefix), true)
 	return toObjectErr(err, bucket)
 }
 
@@ -3040,6 +3092,33 @@ func (es *erasureSingle) Walk(ctx context.Context, bucket, prefix string, result
 	return nil
 }
 
+// Health - returns current status of the object layer health, for single drive
+// its as simple as returning healthy as long as drive is accessible.
+func (es *erasureSingle) Health(ctx context.Context, opts HealthOptions) HealthResult {
+	_, err := es.disk.DiskInfo(ctx)
+	if err != nil {
+		return HealthResult{}
+	}
+	if opts.Maintenance {
+		// Single drive cannot be put under maintenance.
+		return HealthResult{
+			Healthy:     false,
+			WriteQuorum: 1,
+		}
+	}
+	return HealthResult{
+		Healthy:     true,
+		WriteQuorum: 1,
+	}
+}
+
+// ReadHealth - returns current status of the object layer health for reads,
+// for single drive its as simple as returning healthy as long as drive is accessible.
+func (es *erasureSingle) ReadHealth(ctx context.Context) bool {
+	res := es.Health(ctx, HealthOptions{})
+	return res.Healthy
+}
+
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (es *erasureSingle) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
@@ -3235,7 +3314,7 @@ func (es *erasureSingle) NSScanner(ctx context.Context, bf *bloomFilter, updates
 	results := make([]dataUsageCache, 1)
 	var firstErr error
 
-	allBuckets, err := es.ListBuckets(ctx)
+	allBuckets, err := es.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}

cmd/erasure_test.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"crypto/rand"
 	"io"
-	"os"
 	"testing"
 )
 
@@ -118,20 +117,13 @@ type erasureTestSetup struct {
 	disks        []StorageAPI
 }
 
-// Removes the temporary disk directories.
-func (e erasureTestSetup) Remove() {
-	for _, path := range e.diskPaths {
-		os.RemoveAll(path)
-	}
-}
-
 // Returns an initialized setup for erasure tests.
-func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*erasureTestSetup, error) {
+func newErasureTestSetup(tb testing.TB, dataBlocks int, parityBlocks int, blockSize int64) (*erasureTestSetup, error) {
 	diskPaths := make([]string, dataBlocks+parityBlocks)
 	disks := make([]StorageAPI, len(diskPaths))
 	var err error
 	for i := range diskPaths {
-		disks[i], diskPaths[i], err = newXLStorageTestSetup()
+		disks[i], diskPaths[i], err = newXLStorageTestSetup(tb)
 		if err != nil {
 			return nil, err
 		}

cmd/format-erasure_test.go

@@ -104,11 +104,7 @@ func TestFormatErasureEmpty(t *testing.T) {
 // Tests xl format migration.
 func TestFormatErasureMigrate(t *testing.T) {
 	// Get test root.
-	rootPath, err := getTestRoot()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
+	rootPath := t.TempDir()
 
 	m := &formatErasureV1{}
 	m.Format = formatBackendErasure

cmd/fs-v1-helpers_test.go

@@ -30,11 +30,10 @@ import (
 
 func TestFSRenameFile(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	if err = fsMkdir(GlobalContext, pathJoin(path, "testvolume1")); err != nil {
 		t.Fatal(err)
@@ -55,11 +54,10 @@ func TestFSRenameFile(t *testing.T) {
 
 func TestFSStats(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 
@@ -183,11 +181,10 @@ func TestFSStats(t *testing.T) {
 
 func TestFSCreateAndOpen(t *testing.T) {
 	// Setup test environment.
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
 		t.Fatalf("Unable to create directory, %s", err)
@@ -248,11 +245,10 @@ func TestFSCreateAndOpen(t *testing.T) {
 
 func TestFSDeletes(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
@@ -350,11 +346,10 @@ func TestFSDeletes(t *testing.T) {
 
 func BenchmarkFSDeleteFile(b *testing.B) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(b)
 	if err != nil {
 		b.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(path, "benchmark")); err != nil {
@@ -384,11 +379,10 @@ func BenchmarkFSDeleteFile(b *testing.B) {
 // Tests fs removes.
 func TestFSRemoves(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
@@ -501,11 +495,10 @@ func TestFSRemoves(t *testing.T) {
 
 func TestFSRemoveMeta(t *testing.T) {
 	// create xlStorage test setup
-	_, fsPath, err := newXLStorageTestSetup()
+	_, fsPath, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(fsPath)
 
 	// Setup test environment.
 	if err = fsMkdir(GlobalContext, pathJoin(fsPath, "success-vol")); err != nil {
@@ -529,10 +522,7 @@ func TestFSRemoveMeta(t *testing.T) {
 
 	defer rwPool.Close(filePath)
 
-	tmpDir, tmpErr := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if tmpErr != nil {
-		t.Fatal(tmpErr)
-	}
+	tmpDir := t.TempDir()
 
 	if err := fsRemoveMeta(GlobalContext, fsPath, filePath, tmpDir); err != nil {
 		t.Fatalf("Unable to remove file, %s", err)
@@ -548,15 +538,9 @@ func TestFSRemoveMeta(t *testing.T) {
 }
 
 func TestFSIsFile(t *testing.T) {
-	dirPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Unable to create tmp directory %s", err)
-	}
-	defer os.RemoveAll(dirPath)
+	filePath := pathJoin(t.TempDir(), "tmpfile")
 
-	filePath := pathJoin(dirPath, "tmpfile")
-
-	if err = ioutil.WriteFile(filePath, nil, 0o777); err != nil {
+	if err := ioutil.WriteFile(filePath, nil, 0o777); err != nil {
 		t.Fatalf("Unable to create file %s", filePath)
 	}
 

cmd/fs-v1-metadata_test.go

@@ -56,7 +56,7 @@ func TestReadFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	if _, err := obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
@@ -92,7 +92,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 	if _, err := obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {

cmd/fs-v1-multipart_test.go

@@ -45,7 +45,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 
 	// Create a context we can cancel.
 	ctx, cancel := context.WithCancel(GlobalContext)
-	obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{})
+	obj.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{})
 
 	uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
 	if err != nil {
@@ -99,7 +99,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -124,7 +124,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	data := []byte("12345")
 	dataLen := int64(len(data))
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -157,7 +157,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -190,7 +190,7 @@ func TestCompleteMultipartUpload(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -227,7 +227,7 @@ func TestAbortMultipartUpload(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
@@ -258,7 +258,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 

cmd/fs-v1-rwpool.go

@@ -187,6 +187,8 @@ func (fsi *fsIOPool) Create(path string) (wlk *lock.LockedFile, err error) {
 			return nil, errFileAccessDenied
 		case isSysErrIsDir(err):
 			return nil, errIsNotRegular
+		case isSysErrNotDir(err):
+			return nil, errFileAccessDenied
 		case isSysErrPathNotFound(err):
 			return nil, errFileAccessDenied
 		default:

cmd/fs-v1-rwpool_test.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"os"
 	"runtime"
 	"testing"
 
@@ -48,11 +47,10 @@ func TestRWPoolLongPath(t *testing.T) {
 // Tests all RWPool methods.
 func TestRWPool(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	rwPool := &fsIOPool{
 		readersMap: make(map[string]*lock.RLockedFile),

cmd/fs-v1.go

@@ -254,7 +254,7 @@ func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates cha
 		return err
 	}
 	totalCache.Info.Name = dataUsageRoot
-	buckets, err := fs.ListBuckets(ctx)
+	buckets, err := fs.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
 		return err
 	}
@@ -453,7 +453,7 @@ func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileI
 }
 
 // MakeBucketWithLocation - create a new bucket, returns if it already exists.
-func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	if opts.LockEnabled || opts.VersioningEnabled {
 		return NotImplemented{}
 	}
@@ -524,7 +524,7 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 }
 
 // GetBucketInfo - fetch bucket metadata info.
-func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
+func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
 	st, err := fs.statBucketDir(ctx, bucket)
 	if err != nil {
 		return bi, toObjectErr(err, bucket)
@@ -543,7 +543,7 @@ func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi Bucke
 }
 
 // ListBuckets - list all s3 compatible buckets (directories) at fsPath.
-func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
+func (fs *FSObjects) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
 	if err := checkPathLength(fs.fsPath); err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err

cmd/fs-v1_test.go

@@ -62,7 +62,7 @@ func TestFSShutdown(t *testing.T) {
 		fs := obj.(*FSObjects)
 
 		objectContent := "12345"
-		obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+		obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 		obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 		return fs, disk
 	}
@@ -95,18 +95,18 @@ func TestFSGetBucketInfo(t *testing.T) {
 	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucketWithLocation(GlobalContext, "a", BucketOptions{})
+	err := obj.MakeBucketWithLocation(GlobalContext, "a", MakeBucketOptions{})
 	if !isSameType(err, BucketNameInvalid{}) {
 		t.Fatal("BucketNameInvalid error not returned")
 	}
 
-	err = obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	err = obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Test with valid parameters
-	info, err := fs.GetBucketInfo(GlobalContext, bucketName)
+	info, err := fs.GetBucketInfo(GlobalContext, bucketName, BucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -115,7 +115,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	}
 
 	// Test with non-existent bucket
-	_, err = fs.GetBucketInfo(GlobalContext, "a")
+	_, err = fs.GetBucketInfo(GlobalContext, "a", BucketOptions{})
 	if !isSameType(err, BucketNotFound{}) {
 		t.Fatal("BucketNotFound error not returned")
 	}
@@ -123,7 +123,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	// Check for buckets and should get disk not found.
 	os.RemoveAll(disk)
 
-	if _, err = fs.GetBucketInfo(GlobalContext, bucketName); err != nil {
+	if _, err = fs.GetBucketInfo(GlobalContext, bucketName, BucketOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("BucketNotFound error not returned")
 		}
@@ -139,7 +139,7 @@ func TestFSPutObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "1/2/3/4/object"
 
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
@@ -179,7 +179,7 @@ func TestFSDeleteObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 	obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 
 	// Test with invalid bucket name
@@ -223,7 +223,7 @@ func TestFSDeleteBucket(t *testing.T) {
 	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
@@ -242,7 +242,7 @@ func TestFSDeleteBucket(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{})
+	obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{})
 
 	// Delete bucket should get error disk not found.
 	os.RemoveAll(disk)
@@ -264,7 +264,7 @@ func TestFSListBuckets(t *testing.T) {
 	fs := obj.(*FSObjects)
 
 	bucketName := "bucket"
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -279,7 +279,7 @@ func TestFSListBuckets(t *testing.T) {
 	f.Close()
 
 	// Test list buckets to have only one entry.
-	buckets, err := fs.ListBuckets(GlobalContext)
+	buckets, err := fs.ListBuckets(GlobalContext, BucketOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
@@ -289,7 +289,7 @@ func TestFSListBuckets(t *testing.T) {
 
 	// Test ListBuckets with disk not found.
 	os.RemoveAll(disk)
-	if _, err := fs.ListBuckets(GlobalContext); err != nil {
+	if _, err := fs.ListBuckets(GlobalContext, BucketOptions{}); err != nil {
 		if err != errDiskNotFound {
 			t.Fatal("Unexpected error: ", err)
 		}

cmd/gateway-main.go

@@ -310,7 +310,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
 
 	if gatewayName == NASBackendGateway {
-		buckets, err := newObject.ListBuckets(GlobalContext)
+		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
 		if err != nil {
 			logger.Fatal(err, "Unable to list buckets")
 		}
@@ -332,7 +332,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Populate existing buckets to the etcd backend
 	if globalDNSConfig != nil {
-		buckets, err := newObject.ListBuckets(GlobalContext)
+		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
 		if err != nil {
 			logger.Fatal(err, "Unable to list buckets")
 		}

cmd/gateway/s3/gateway-s3-sse.go

@@ -734,7 +734,7 @@ func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, clea
 
 // cleanupStaleUploads removes old custom encryption multipart uploads on backend
 func (l *s3EncObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
-	buckets, err := l.s3Objects.ListBuckets(ctx)
+	buckets, err := l.s3Objects.ListBuckets(ctx, minio.BucketOptions{})
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return

cmd/gateway/s3/gateway-s3.go

@@ -325,7 +325,7 @@ func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []
 }
 
 // MakeBucket creates a new container on S3 backend.
-func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
+func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.MakeBucketOptions) error {
 	if opts.LockEnabled || opts.VersioningEnabled {
 		return minio.NotImplemented{}
 	}
@@ -348,7 +348,7 @@ func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket string, o
 }
 
 // GetBucketInfo gets bucket metadata..
-func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) {
+func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string, opts minio.BucketOptions) (bi minio.BucketInfo, e error) {
 	buckets, err := l.Client.ListBuckets(ctx)
 	if err != nil {
 		// Listbuckets may be disallowed, proceed to check if
@@ -381,7 +381,7 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.
 }
 
 // ListBuckets lists all S3 buckets
-func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) {
+func (l *s3Objects) ListBuckets(ctx context.Context, opts minio.BucketOptions) ([]minio.BucketInfo, error) {
 	buckets, err := l.Client.ListBuckets(ctx)
 	if err != nil {
 		return nil, minio.ErrorRespToObjectError(err)

cmd/iam-store.go

@@ -337,33 +337,16 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 		return c.iamGroupPolicyMap[name].toSlice(), c.iamGroupPolicyMap[name].UpdatedAt, nil
 	}
 
-	if name == globalActiveCred.AccessKey {
-		return []string{"consoleAdmin"}, time.Time{}, nil
-	}
-
 	// When looking for a user's policies, we also check if the user
 	// and the groups they are member of are enabled.
-	var parentName string
 	u, ok := c.iamUsersMap[name]
 	if ok {
 		if !u.Credentials.IsValid() {
 			return nil, time.Time{}, nil
 		}
-		parentName = u.Credentials.ParentUser
 	}
 
-	mp, ok := c.iamUserPolicyMap[name]
-	if !ok {
-		// Service accounts with root credentials, inherit parent permissions
-		if parentName == globalActiveCred.AccessKey && u.Credentials.IsServiceAccount() {
-			// even if this is set, the claims present in the service
-			// accounts apply the final permissions if any.
-			return []string{"consoleAdmin"}, mp.UpdatedAt, nil
-		}
-		if parentName != "" {
-			mp = c.iamUserPolicyMap[parentName]
-		}
-	}
+	mp := c.iamUserPolicyMap[name]
 
 	// returned policy could be empty
 	policies := mp.toSlice()

cmd/iam.go

@@ -1458,7 +1458,7 @@ const sessionPolicyNameExtracted = iampolicy.SessionPolicyName + "-extracted"
 // IsAllowedServiceAccount - checks if the given service account is allowed to perform
 // actions. The permission of the parent user is checked first
 func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser string) bool {
-	// Now check if we have a subject claim
+	// Verify if the parent claim matches the parentUser.
 	p, ok := args.Claims[parentClaim]
 	if ok {
 		parentInClaim, ok := p.(string)
@@ -1478,41 +1478,56 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 		return false
 	}
 
-	// Check policy for parent user of service account.
-	svcPolicies, err := sys.PolicyDBGet(parentUser, false, args.Groups...)
-	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return false
-	}
+	isOwnerDerived := parentUser == globalActiveCred.AccessKey
 
-	if len(svcPolicies) == 0 {
-		// If parent user has no policies, check for OpenID
-		// claims/RoleARN in case it exists.
-		roleArn := args.GetRoleArn()
-		if roleArn != "" {
-			arn, err := arn.Parse(roleArn)
-			if err != nil {
-				logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
-				return false
-			}
-			svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
-		} else {
-			// If there is no roleArn claim, check the OpenID
-			// provider's policy claim.
+	var err error
+	var svcPolicies []string
+	roleArn := args.GetRoleArn()
+
+	switch {
+	case isOwnerDerived:
+		// All actions are allowed by default and no policy evaluation is
+		// required.
+
+	case roleArn != "":
+		arn, err := arn.Parse(roleArn)
+		if err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
+			return false
+		}
+		svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
+
+	default:
+		// Check policy for parent user of service account.
+		svcPolicies, err = sys.PolicyDBGet(parentUser, false, args.Groups...)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			return false
+		}
+
+		// Finally, if there is no parent policy, check if a policy claim is
+		// present.
+		if len(svcPolicies) == 0 {
 			policySet, _ := iampolicy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID())
 			svcPolicies = policySet.ToSlice()
 		}
-		if len(svcPolicies) == 0 {
-			return false
-		}
 	}
 
-	// Policies were found, evaluate all of them.
-	availablePoliciesStr, combinedPolicy := sys.store.FilterPolicies(strings.Join(svcPolicies, ","), "")
-	if availablePoliciesStr == "" {
+	// Defensive code: Do not allow any operation if no policy is found.
+	if !isOwnerDerived && len(svcPolicies) == 0 {
 		return false
 	}
 
+	var combinedPolicy iampolicy.Policy
+	// Policies were found, evaluate all of them.
+	if !isOwnerDerived {
+		availablePoliciesStr, c := sys.store.FilterPolicies(strings.Join(svcPolicies, ","), "")
+		if availablePoliciesStr == "" {
+			return false
+		}
+		combinedPolicy = c
+	}
+
 	parentArgs := args
 	parentArgs.AccountName = parentUser
 	// These are dynamic values set them appropriately.
@@ -1532,7 +1547,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 	}
 
 	if saPolicyClaimStr == inheritedPolicyType {
-		return combinedPolicy.IsAllowed(parentArgs)
+		return isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)
 	}
 
 	// Now check if we have a sessionPolicy.
@@ -1558,37 +1573,51 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 
 	// This can only happen if policy was set but with an empty JSON.
 	if subPolicy.Version == "" && len(subPolicy.Statements) == 0 {
-		return combinedPolicy.IsAllowed(parentArgs)
+		return isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)
 	}
 
 	if subPolicy.Version == "" {
 		return false
 	}
 
-	return combinedPolicy.IsAllowed(parentArgs) && subPolicy.IsAllowed(parentArgs)
+	return subPolicy.IsAllowed(parentArgs) && (isOwnerDerived || combinedPolicy.IsAllowed(parentArgs))
 }
 
 // IsAllowedSTS is meant for STS based temporary credentials,
 // which implements claims validation and verification other than
 // applying policies.
 func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
+	// 1. Determine mapped policies
+
+	isOwnerDerived := parentUser == globalActiveCred.AccessKey
 	var policies []string
 	roleArn := args.GetRoleArn()
-	if roleArn != "" {
+
+	switch {
+	case isOwnerDerived:
+		// All actions are allowed by default and no policy evaluation is
+		// required.
+
+	case roleArn != "":
+		// If a roleARN is present, the role policy is applied.
 		arn, err := arn.Parse(roleArn)
 		if err != nil {
 			logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
 			return false
 		}
 		policies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
-	} else {
-		// Lookup the parent user's mapping if there's no role-ARN.
+
+	default:
+		// Otherwise, inherit parent user's policy
 		var err error
 		policies, err = sys.store.PolicyDBGet(parentUser, false, args.Groups...)
 		if err != nil {
 			logger.LogIf(GlobalContext, fmt.Errorf("error fetching policies on %s: %v", parentUser, err))
 			return false
 		}
+
+		// Finally, if there is no parent policy, check if a policy claim is
+		// present in the session token.
 		if len(policies) == 0 {
 			// If there is no parent policy mapping, we fall back to
 			// using policy claim from JWT.
@@ -1599,27 +1628,37 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 			}
 			policies = policySet.ToSlice()
 		}
+
 	}
 
 	// Defensive code: Do not allow any operation if no policy is found in the session token
-	if len(policies) == 0 {
+	if !isOwnerDerived && len(policies) == 0 {
 		return false
 	}
 
-	combinedPolicy, err := sys.store.GetPolicy(strings.Join(policies, ","))
-	if err == errNoSuchPolicy {
-		for _, pname := range policies {
-			_, err := sys.store.GetPolicy(pname)
-			if err == errNoSuchPolicy {
-				// all policies presented in the claim should exist
-				logger.LogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
-				return false
+	// 2. Combine the mapped policies into a single combined policy.
+
+	var combinedPolicy iampolicy.Policy
+	if !isOwnerDerived {
+		var err error
+		combinedPolicy, err = sys.store.GetPolicy(strings.Join(policies, ","))
+		if err == errNoSuchPolicy {
+			for _, pname := range policies {
+				_, err := sys.store.GetPolicy(pname)
+				if err == errNoSuchPolicy {
+					// all policies presented in the claim should exist
+					logger.LogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
+					return false
+				}
 			}
+			logger.LogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
+			return false
 		}
-		logger.LogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
-		return false
+
 	}
 
+	// 3. If an inline session-policy is present, evaluate it.
+
 	// These are dynamic values set them appropriately.
 	args.ConditionValues["username"] = []string{parentUser}
 	args.ConditionValues["userid"] = []string{parentUser}
@@ -1627,12 +1666,12 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 	// Now check if we have a sessionPolicy.
 	hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicy(args)
 	if hasSessionPolicy {
-		return isAllowedSP && combinedPolicy.IsAllowed(args)
+		return isAllowedSP && (isOwnerDerived || combinedPolicy.IsAllowed(args))
 	}
 
 	// Sub policy not set, this is most common since subPolicy
 	// is optional, use the inherited policies.
-	return combinedPolicy.IsAllowed(args)
+	return isOwnerDerived || combinedPolicy.IsAllowed(args)
 }
 
 func isAllowedBySessionPolicy(args iampolicy.Args) (hasSessionPolicy bool, isAllowed bool) {

cmd/listen-notification-handlers.go

@@ -113,7 +113,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 	}
 
 	if bucketName != "" {
-		if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
+		if _, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{}); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}

cmd/metrics-v2.go

@@ -996,7 +996,7 @@ func getMinioProcMetrics() *MetricsGroup {
 		metrics = make([]Metric, 0, 20)
 		p, err := procfs.Self()
 		if err != nil {
-			logger.LogOnceIf(ctx, err, nodeMetricNamespace)
+			logger.LogOnceIf(ctx, err, string(nodeMetricNamespace))
 			return
 		}
 

cmd/notification.go

@@ -297,9 +297,7 @@ func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerEr
 }
 
 // DownloadProfilingData - download profiling data from all remote peers.
-func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) bool {
-	profilingDataFound := false
-
+func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) (profilingDataFound bool) {
 	// Initialize a zip writer which will provide a zipped content
 	// of profiling data of all nodes
 	zipWriter := zip.NewWriter(writer)
@@ -320,34 +318,11 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		profilingDataFound = true
 
 		for typ, data := range data {
-			// Send profiling data to zip as file
-			header, zerr := zip.FileInfoHeader(dummyFileInfo{
-				name:    fmt.Sprintf("profile-%s-%s", client.host.String(), typ),
-				size:    int64(len(data)),
-				mode:    0o600,
-				modTime: UTCNow(),
-				isDir:   false,
-				sys:     nil,
-			})
-			if zerr != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-				ctx := logger.SetReqInfo(ctx, reqInfo)
-				logger.LogIf(ctx, zerr)
-				continue
-			}
-			header.Method = zip.Deflate
-			zwriter, zerr := zipWriter.CreateHeader(header)
-			if zerr != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-				ctx := logger.SetReqInfo(ctx, reqInfo)
-				logger.LogIf(ctx, zerr)
-				continue
-			}
-			if _, err = io.Copy(zwriter, bytes.NewReader(data)); err != nil {
+			err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data)
+			if err != nil {
 				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 				ctx := logger.SetReqInfo(ctx, reqInfo)
 				logger.LogIf(ctx, err)
-				continue
 			}
 		}
 	}
@@ -371,34 +346,18 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 
 	// Send profiling data to zip as file
 	for typ, data := range data {
-		header, zerr := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profile-%s-%s", thisAddr, typ),
-			size:    int64(len(data)),
-			mode:    0o600,
-			modTime: UTCNow(),
-			isDir:   false,
-			sys:     nil,
-		})
-		if zerr != nil {
-			return profilingDataFound
-		}
-		header.Method = zip.Deflate
-
-		zwriter, zerr := zipWriter.CreateHeader(header)
-		if zerr != nil {
-			return profilingDataFound
-		}
-
-		if _, err = io.Copy(zwriter, bytes.NewReader(data)); err != nil {
-			return profilingDataFound
+		err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", thisAddr, typ), data)
+		if err != nil {
+			logger.LogIf(ctx, err)
 		}
 	}
 
-	return profilingDataFound
+	appendClusterMetaInfoToZip(ctx, zipWriter)
+	return
 }
 
-// ServerUpdate - updates remote peers.
-func (sys *NotificationSys) ServerUpdate(ctx context.Context, u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string) []NotificationPeerErr {
+// VerifyBinary - asks remote peers to verify the checksum
+func (sys *NotificationSys) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, reader []byte) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
@@ -406,7 +365,22 @@ func (sys *NotificationSys) ServerUpdate(ctx context.Context, u *url.URL, sha256
 		}
 		client := client
 		ng.Go(ctx, func() error {
-			return client.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo)
+			return client.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
+// CommitBinary - asks remote peers to overwrite the old binary with the new one
+func (sys *NotificationSys) CommitBinary(ctx context.Context) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(ctx, func() error {
+			return client.CommitBinary(ctx)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -482,7 +456,7 @@ func (sys *NotificationSys) updateBloomFilter(ctx context.Context, current uint6
 			defer mu.Unlock()
 
 			if err != nil || !serverBF.Complete || bf == nil {
-				logger.LogOnceIf(ctx, err, fmt.Sprintf("host:%s, cycle:%d", client.host, current), client.cycleServerBloomFilter)
+				logger.LogOnceIf(ctx, err, client.host.String(), client.cycleServerBloomFilter)
 				bf = nil
 				return nil
 			}
@@ -743,7 +717,7 @@ func (sys *NotificationSys) InitBucketTargets(ctx context.Context, objAPI Object
 			if res.Err != nil {
 				reqInfo := &logger.ReqInfo{}
 				reqInfo.AppendTags("targetID", res.ID.Name)
-				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID)
+				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
 			}
 		}
 	}()

cmd/object-api-common.go

@@ -42,6 +42,9 @@ const (
 	// Buckets meta prefix.
 	bucketMetaPrefix = "buckets"
 
+	// Deleted Buckets prefix.
+	deletedBucketsPrefix = ".deleted"
+
 	// ETag (hex encoded md5sum) of empty string.
 	emptyETag = "d41d8cd98f00b204e9800998ecf8427e"
 )

cmd/object-api-datatypes.go

@@ -78,6 +78,7 @@ type BucketInfo struct {
 
 	// Date and time when the bucket was created.
 	Created time.Time
+	Deleted time.Time
 }
 
 // ObjectInfo - represents object metadata.

cmd/object-api-deleteobject_test.go

@@ -84,7 +84,7 @@ func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	for i, testCase := range testCases {
-		err := obj.MakeBucketWithLocation(context.Background(), testCase.bucketName, BucketOptions{})
+		err := obj.MakeBucketWithLocation(context.Background(), testCase.bucketName, MakeBucketOptions{})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}

cmd/object-api-getobjectinfo_test.go

@@ -31,7 +31,7 @@ func TestGetObjectInfo(t *testing.T) {
 // Testing GetObjectInfo().
 func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// This bucket is used for testing getObjectInfo operations.
-	err := obj.MakeBucketWithLocation(context.Background(), "test-getobjectinfo", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "test-getobjectinfo", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}

cmd/object-api-input-checks.go

@@ -198,12 +198,12 @@ func checkPutObjectArgs(ctx context.Context, bucket, object string, obj getBucke
 }
 
 type getBucketInfoI interface {
-	GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
+	GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error)
 }
 
 // Checks whether bucket exists and returns appropriate error if not.
 func checkBucketExist(ctx context.Context, bucket string, obj getBucketInfoI) error {
-	_, err := obj.GetBucketInfo(ctx, bucket)
+	_, err := obj.GetBucketInfo(ctx, bucket, BucketOptions{})
 	if err != nil {
 		return err
 	}

cmd/object-api-interface.go

@@ -102,18 +102,25 @@ type TransitionOptions struct {
 	ExpireRestored bool
 }
 
-// BucketOptions represents bucket options for ObjectLayer bucket operations
-type BucketOptions struct {
+// MakeBucketOptions represents bucket options for ObjectLayer bucket operations
+type MakeBucketOptions struct {
 	Location          string
 	LockEnabled       bool
 	VersioningEnabled bool
-	ForceCreate       bool // Create buckets even if they are already created.
+	ForceCreate       bool      // Create buckets even if they are already created.
+	CreatedAt         time.Time // only for site replication
 }
 
 // DeleteBucketOptions provides options for DeleteBucket calls.
 type DeleteBucketOptions struct {
-	Force      bool // Force deletion
-	NoRecreate bool // Do not recreate on delete failures
+	Force      bool             // Force deletion
+	NoRecreate bool             // Do not recreate on delete failures
+	SRDeleteOp SRBucketDeleteOp // only when site replication is enabled
+}
+
+// BucketOptions provides options for ListBuckets and GetBucketInfo call.
+type BucketOptions struct {
+	Deleted bool // true only when site replication is enabled
 }
 
 // SetReplicaStatus sets replica status and timestamp for delete operations in ObjectOptions
@@ -187,9 +194,9 @@ type ObjectLayer interface {
 	LocalStorageInfo(ctx context.Context) (StorageInfo, []error)
 
 	// Bucket operations.
-	MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error
-	GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
-	ListBuckets(ctx context.Context) (buckets []BucketInfo, err error)
+	MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error
+	GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error)
+	ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error)
 	DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error
 	ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)

cmd/object-api-listobjects_test.go

@@ -23,8 +23,6 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"strconv"
 	"strings"
 	"testing"
@@ -43,7 +41,7 @@ func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 Te
 		"test-bucket-files",
 	}
 	for _, bucket := range testBuckets {
-		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{
 			VersioningEnabled: true,
 		})
 		if err != nil {
@@ -331,7 +329,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		"test-bucket-max-keys-prefixes",
 	}
 	for _, bucket := range testBuckets {
-		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{
 			VersioningEnabled: versioned,
 		})
 		if err != nil {
@@ -1021,7 +1019,7 @@ func testDeleteObjectVersion(obj ObjectLayer, instanceType string, t1 TestErrHan
 		"bucket-suspended-version-id",
 	}
 	for _, bucket := range testBuckets {
-		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{
 			VersioningEnabled: true,
 		})
 		if err != nil {
@@ -1108,7 +1106,7 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		"test-bucket-max-keys-prefixes",
 	}
 	for _, bucket := range testBuckets {
-		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{VersioningEnabled: true})
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{VersioningEnabled: true})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1738,7 +1736,7 @@ func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestEr
 		"test-bucket-list-object-continuation-2",
 	}
 	for _, bucket := range testBuckets {
-		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1886,18 +1884,14 @@ func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) {
 // BenchmarkListObjects - Run ListObject Repeatedly and benchmark.
 func BenchmarkListObjects(b *testing.B) {
 	// Make a temporary directory to use as the obj.
-	directory, err := ioutil.TempDir(globalTestTmpDir, "minio-list-benchmark")
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer os.RemoveAll(directory)
+	directory := b.TempDir()
 
 	// Create the obj.
 	obj := initFSObjectsB(directory, b)
 
 	bucket := "ls-benchmark-bucket"
 	// Create a bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/object-api-multipart_test.go

@@ -56,7 +56,7 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
 	}
 
 	// Create bucket before intiating NewMultipartUpload.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -89,7 +89,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 	object := "minio-object"
 	opts := ObjectOptions{}
 	// Create bucket before intiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -135,7 +135,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
 	object := "minio-object"
 
 	// Create bucket before intiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -167,7 +167,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 	object := "minio-object"
 	opts := ObjectOptions{}
 	// Create bucket before intiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -179,7 +179,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	// Creating a dummy bucket for tests.
-	err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -314,7 +314,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	// objectNames[0].
 	// uploadIds [0].
 	// Create bucket before initiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -332,7 +332,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	// objectNames[0].
 	// uploadIds [1-3].
 	// Bucket to test for mutiple upload Id's for a given object.
-	err = obj.MakeBucketWithLocation(context.Background(), bucketNames[1], BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucketNames[1], MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -353,7 +353,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	// bucketnames[2].
 	// objectNames[0-2].
 	// uploadIds [4-9].
-	err = obj.MakeBucketWithLocation(context.Background(), bucketNames[2], BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucketNames[2], MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1195,7 +1195,7 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 	// objectNames[0].
 	// uploadIds [0].
 	// Create bucket before intiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1439,7 +1439,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 	// objectNames[0].
 	// uploadIds [0].
 	// Create bucket before intiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucketNames[0], MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1681,7 +1681,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 	// objectNames[0].
 	// uploadIds [0].
 	// Create bucket before intiating NewMultipartUpload.
-	err = obj.MakeBucketWithLocation(context.Background(), bucketNames[0], BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucketNames[0], MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err)

cmd/object-api-putobject_test.go

@@ -48,14 +48,14 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 	object := "minio-object"
 
 	// Create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
 	// Creating a dummy bucket for tests.
-	err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -239,14 +239,14 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
 	object := "minio-object"
 
 	// Create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
 	// Creating a dummy bucket for tests.
-	err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "unused-bucket", MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -351,7 +351,7 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk
 	object := "minio-object"
 
 	// Create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -396,7 +396,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 	object := "minio-object"
 
 	// Create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())

cmd/object-handlers.go

@@ -922,7 +922,7 @@ func isRemoteCallRequired(ctx context.Context, bucket string, objAPI ObjectLayer
 		return false
 	}
 	if globalBucketFederation {
-		_, err := objAPI.GetBucketInfo(ctx, bucket)
+		_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
 		return err == toObjectErr(errVolumeNotFound, bucket)
 	}
 	return false
@@ -2758,7 +2758,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -2918,7 +2918,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/object_api_suite_test.go

@@ -77,7 +77,7 @@ func TestMakeBucket(t *testing.T) {
 
 // Tests validate bucket creation.
 func testMakeBucket(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket-unknown", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket-unknown", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -91,7 +91,7 @@ func TestMultipartObjectCreation(t *testing.T) {
 // Tests validate creation of part files during Multipart operation.
 func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -135,7 +135,7 @@ func TestMultipartObjectAbort(t *testing.T) {
 // Tests validate abortion of Multipart operation.
 func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -181,7 +181,7 @@ func TestMultipleObjectCreation(t *testing.T) {
 func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	objects := make(map[string][]byte)
 	var opts ObjectOptions
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -236,7 +236,7 @@ func TestPaging(t *testing.T) {
 
 // Tests validate creation of objects and the order of listing using various filters for ListObjects operation.
 func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	result, err := obj.ListObjects(context.Background(), "bucket", "", "", "", 0)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
@@ -440,7 +440,7 @@ func TestObjectOverwriteWorks(t *testing.T) {
 
 // Tests validate overwriting of an existing object.
 func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -494,11 +494,11 @@ func TestBucketRecreateFails(t *testing.T) {
 
 // Tests validate that recreation of the bucket fails.
 func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "string", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "string", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.MakeBucketWithLocation(context.Background(), "string", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "string", MakeBucketOptions{})
 	if err == nil {
 		t.Fatalf("%s: Expected error but found nil.", instanceType)
 	}
@@ -599,7 +599,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	length := int64(len(content))
 	readerEOF := newTestReaderEOF(content)
 	readerNoEOF := newTestReaderNoEOF(content)
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -639,7 +639,7 @@ func TestPutObjectInSubdir(t *testing.T) {
 
 // Tests validate PutObject with subdirectory prefix.
 func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -672,7 +672,7 @@ func TestListBuckets(t *testing.T) {
 // Tests validate ListBuckets.
 func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// test empty list.
-	buckets, err := obj.ListBuckets(context.Background())
+	buckets, err := obj.ListBuckets(context.Background(), BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -681,12 +681,12 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	// add one and test exists.
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket1", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket1", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
-	buckets, err = obj.ListBuckets(context.Background())
+	buckets, err = obj.ListBuckets(context.Background(), BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -695,12 +695,12 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	// add two and test exists.
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
-	buckets, err = obj.ListBuckets(context.Background())
+	buckets, err = obj.ListBuckets(context.Background(), BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -709,12 +709,12 @@ func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	// add three and test exists + prefix.
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket22", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket22", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
-	buckets, err = obj.ListBuckets(context.Background())
+	buckets, err = obj.ListBuckets(context.Background(), BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -733,15 +733,15 @@ func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler
 	// if implementation contains a map, order of map keys will vary.
 	// this ensures they return in the same order each time.
 	// add one and test exists.
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket1", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket1", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), "bucket2", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	buckets, err := obj.ListBuckets(context.Background())
+	buckets, err := obj.ListBuckets(context.Background(), BucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -786,7 +786,7 @@ func TestNonExistantObjectInBucket(t *testing.T) {
 
 // Tests validate that GetObject fails on a non-existent bucket as expected.
 func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -814,7 +814,7 @@ func TestGetDirectoryReturnsObjectNotFound(t *testing.T) {
 // Tests validate that GetObject on an existing directory fails as expected.
 func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	bucketName := "bucket"
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucketName, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -856,7 +856,7 @@ func TestContentType(t *testing.T) {
 
 // Test content-type.
 func testContentType(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}

cmd/os-instrumented.go

@@ -35,6 +35,7 @@ type osMetric uint8
 const (
 	osMetricRemoveAll osMetric = iota
 	osMetricMkdirAll
+	osMetricMkdir
 	osMetricRename
 	osMetricOpenFile
 	osMetricOpen
@@ -114,10 +115,16 @@ func RemoveAll(dirPath string) error {
 	return os.RemoveAll(dirPath)
 }
 
+// Mkdir captures time taken to call os.Mkdir
+func Mkdir(dirPath string, mode os.FileMode) error {
+	defer updateOSMetrics(osMetricMkdir, dirPath)()
+	return os.Mkdir(dirPath, mode)
+}
+
 // MkdirAll captures time taken to call os.MkdirAll
 func MkdirAll(dirPath string, mode os.FileMode) error {
 	defer updateOSMetrics(osMetricMkdirAll, dirPath)()
-	return os.MkdirAll(dirPath, mode)
+	return osMkdirAll(dirPath, mode)
 }
 
 // Rename captures time taken to call os.Rename

cmd/os-readdir_test.go

@@ -67,25 +67,16 @@ type result struct {
 	entries []string
 }
 
-func mustSetupDir(t *testing.T) string {
-	// Create unique test directory.
-	dir, err := ioutil.TempDir(globalTestTmpDir, "minio-list-dir")
-	if err != nil {
-		t.Fatalf("Unable to setup directory, %s", err)
-	}
-	return dir
-}
-
 // Test to read empty directory.
 func setupTestReadDirEmpty(t *testing.T) (testResults []result) {
 	// Add empty entry slice for this test directory.
-	testResults = append(testResults, result{mustSetupDir(t), []string{}})
+	testResults = append(testResults, result{t.TempDir(), []string{}})
 	return testResults
 }
 
 // Test to read non-empty directory with only files.
 func setupTestReadDirFiles(t *testing.T) (testResults []result) {
-	dir := mustSetupDir(t)
+	dir := t.TempDir()
 	entries := []string{}
 	for i := 0; i < 10; i++ {
 		name := fmt.Sprintf("file-%d", i)
@@ -107,7 +98,7 @@ func setupTestReadDirFiles(t *testing.T) (testResults []result) {
 
 // Test to read non-empty directory with directories and files.
 func setupTestReadDirGeneric(t *testing.T) (testResults []result) {
-	dir := mustSetupDir(t)
+	dir := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(dir, "mydir"), 0o777); err != nil {
 		t.Fatalf("Unable to create prefix directory \"mydir\", %s", err)
 	}
@@ -134,7 +125,7 @@ func setupTestReadDirSymlink(t *testing.T) (testResults []result) {
 		t.Skip("symlinks not available on windows")
 		return nil
 	}
-	dir := mustSetupDir(t)
+	dir := t.TempDir()
 	entries := []string{}
 	for i := 0; i < 10; i++ {
 		name1 := fmt.Sprintf("file-%d", i)
@@ -241,7 +232,7 @@ func TestReadDirN(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		dir := mustSetupDir(t)
+		dir := t.TempDir()
 
 		for c := 1; c <= testCase.numFiles; c++ {
 			err := ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("%d", c)), []byte{}, os.ModePerm)

cmd/os-reliable.go

@@ -104,7 +104,7 @@ func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) {
 	i := 0
 	for {
 		// Creates all the parent directories, with mode 0777 mkdir honors system umask.
-		if err = MkdirAll(dirPath, mode); err != nil {
+		if err = osMkdirAll(dirPath, mode); err != nil {
 			// Retry only for the first retryable error.
 			if osIsNotExist(err) && i == 0 {
 				i++
@@ -166,6 +166,7 @@ func reliableRename(srcFilePath, dstFilePath string) (err error) {
 	if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777); err != nil {
 		return err
 	}
+
 	i := 0
 	for {
 		// After a successful parent directory create attempt a renameAll.

cmd/os-reliable_test.go

@@ -18,18 +18,16 @@
 package cmd
 
 import (
-	"os"
 	"testing"
 )
 
 // Tests - mkdirAll()
 func TestOSMkdirAll(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	if err = mkdirAll("", 0o777); err != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
@@ -47,11 +45,10 @@ func TestOSMkdirAll(t *testing.T) {
 // Tests - renameAll()
 func TestOSRenameAll(t *testing.T) {
 	// create xlStorage test setup
-	_, path, err := newXLStorageTestSetup()
+	_, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	if err = mkdirAll(pathJoin(path, "testvolume1"), 0o777); err != nil {
 		t.Fatal(err)

cmd/os_other.go

@@ -31,6 +31,10 @@ func access(name string) error {
 	return err
 }
 
+func osMkdirAll(dirPath string, perm os.FileMode) error {
+	return os.MkdirAll(dirPath, perm)
+}
+
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.

cmd/os_unix.go

@@ -38,6 +38,55 @@ func access(name string) error {
 	return nil
 }
 
+// Forked from Golang but chooses fast path upon os.Mkdir()
+// error to avoid os.Lstat() call.
+//
+// osMkdirAll creates a directory named path,
+// along with any necessary parents, and returns nil,
+// or else returns an error.
+// The permission bits perm (before umask) are used for all
+// directories that MkdirAll creates.
+// If path is already a directory, MkdirAll does nothing
+// and returns nil.
+func osMkdirAll(dirPath string, perm os.FileMode) error {
+	// Fast path: if we can tell whether path is a directory or file, stop with success or error.
+	err := Access(dirPath)
+	if err == nil {
+		return nil
+	}
+	if !osIsNotExist(err) {
+		return &os.PathError{Op: "mkdir", Path: dirPath, Err: err}
+	}
+
+	// Slow path: make sure parent exists and then call Mkdir for path.
+	i := len(dirPath)
+	for i > 0 && os.IsPathSeparator(dirPath[i-1]) { // Skip trailing path separator.
+		i--
+	}
+
+	j := i
+	for j > 0 && !os.IsPathSeparator(dirPath[j-1]) { // Scan backward over element.
+		j--
+	}
+
+	if j > 1 {
+		// Create parent.
+		if err = osMkdirAll(dirPath[:j-1], perm); err != nil {
+			return err
+		}
+	}
+
+	// Parent now exists; invoke Mkdir and use its result.
+	if err = Mkdir(dirPath, perm); err != nil {
+		if osIsExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	return nil
+}
+
 // The buffer must be at least a block long.
 // refer https://github.com/golang/go/issues/24015
 const blockSize = 8 << 10 // 8192
@@ -183,7 +232,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 // Return count entries at the directory dirPath and all entries
 // if count is set to -1
 func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
-	f, err := Open(dirPath)
+	f, err := OpenFile(dirPath, readMode, 0)
 	if err != nil {
 		return nil, osErrToFileErr(err)
 	}

cmd/os_windows.go

@@ -31,6 +31,10 @@ func access(name string) error {
 	return err
 }
 
+func osMkdirAll(dirPath string, perm os.FileMode) error {
+	return os.MkdirAll(dirPath, perm)
+}
+
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.

cmd/osmetric_string.go

@@ -10,24 +10,25 @@ func _() {
 	var x [1]struct{}
 	_ = x[osMetricRemoveAll-0]
 	_ = x[osMetricMkdirAll-1]
-	_ = x[osMetricRename-2]
-	_ = x[osMetricOpenFile-3]
-	_ = x[osMetricOpen-4]
-	_ = x[osMetricOpenFileDirectIO-5]
-	_ = x[osMetricLstat-6]
-	_ = x[osMetricRemove-7]
-	_ = x[osMetricStat-8]
-	_ = x[osMetricAccess-9]
-	_ = x[osMetricCreate-10]
-	_ = x[osMetricReadDirent-11]
-	_ = x[osMetricFdatasync-12]
-	_ = x[osMetricSync-13]
-	_ = x[osMetricLast-14]
+	_ = x[osMetricMkdir-2]
+	_ = x[osMetricRename-3]
+	_ = x[osMetricOpenFile-4]
+	_ = x[osMetricOpen-5]
+	_ = x[osMetricOpenFileDirectIO-6]
+	_ = x[osMetricLstat-7]
+	_ = x[osMetricRemove-8]
+	_ = x[osMetricStat-9]
+	_ = x[osMetricAccess-10]
+	_ = x[osMetricCreate-11]
+	_ = x[osMetricReadDirent-12]
+	_ = x[osMetricFdatasync-13]
+	_ = x[osMetricSync-14]
+	_ = x[osMetricLast-15]
 }
 
-const _osMetric_name = "RemoveAllMkdirAllRenameOpenFileOpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncLast"
+const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileOpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncLast"
 
-var _osMetric_index = [...]uint8{0, 9, 17, 23, 31, 35, 51, 56, 62, 66, 72, 78, 88, 97, 101, 105}
+var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 36, 40, 56, 61, 67, 71, 77, 83, 93, 102, 106, 110}
 
 func (i osMetric) String() string {
 	if i >= osMetric(len(_osMetric_index)-1) {

cmd/peer-rest-client.go

@@ -418,26 +418,36 @@ func (client *peerRESTClient) LoadGroup(group string) error {
 	return nil
 }
 
-type serverUpdateInfo struct {
+type binaryInfo struct {
 	URL         *url.URL
 	Sha256Sum   []byte
-	Time        time.Time
 	ReleaseInfo string
+	BinaryFile  []byte
 }
 
-// ServerUpdate - sends server update message to remote peers.
-func (client *peerRESTClient) ServerUpdate(ctx context.Context, u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string) error {
+// VerifyBinary - sends verify binary message to remote peers.
+func (client *peerRESTClient) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, readerInput []byte) error {
 	values := make(url.Values)
 	var reader bytes.Buffer
-	if err := gob.NewEncoder(&reader).Encode(serverUpdateInfo{
+	if err := gob.NewEncoder(&reader).Encode(binaryInfo{
 		URL:         u,
 		Sha256Sum:   sha256Sum,
-		Time:        lrTime,
 		ReleaseInfo: releaseInfo,
+		BinaryFile:  readerInput,
 	}); err != nil {
 		return err
 	}
-	respBody, err := client.callWithContext(ctx, peerRESTMethodServerUpdate, values, &reader, -1)
+	respBody, err := client.callWithContext(ctx, peerRESTMethodDownloadBinary, values, &reader, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// CommitBinary - sends commit binary message to remote peers.
+func (client *peerRESTClient) CommitBinary(ctx context.Context) error {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodCommitBinary, nil, nil, -1)
 	if err != nil {
 		return err
 	}

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v23" // Added /metrics
+	peerRESTVersion       = "v24" // Change ServerUpdate to DownloadBinary and CommitBinary
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -39,7 +39,8 @@ const (
 	peerRESTMethodLoadBucketMetadata          = "/loadbucketmetadata"
 	peerRESTMethodGetBucketStats              = "/getbucketstats"
 	peerRESTMethodGetAllBucketStats           = "/getallbucketstats"
-	peerRESTMethodServerUpdate                = "/serverupdate"
+	peerRESTMethodDownloadBinary              = "/downloadbinary"
+	peerRESTMethodCommitBinary                = "/commitbinary"
 	peerRESTMethodSignalService               = "/signalservice"
 	peerRESTMethodBackgroundHealStatus        = "/backgroundhealstatus"
 	peerRESTMethodGetLocks                    = "/getlocks"

cmd/peer-rest-server.go

@@ -751,8 +751,8 @@ func (s *peerRESTServer) GetLocalDiskIDs(w http.ResponseWriter, r *http.Request)
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(ids))
 }
 
-// ServerUpdateHandler - updates the current server.
-func (s *peerRESTServer) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
+// DownloadBinary - updates the current server.
+func (s *peerRESTServer) DownloadBinaryHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
 		return
@@ -763,14 +763,27 @@ func (s *peerRESTServer) ServerUpdateHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	var info serverUpdateInfo
+	var info binaryInfo
 	err := gob.NewDecoder(r.Body).Decode(&info)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
 
-	if _, err = updateServer(info.URL, info.Sha256Sum, info.Time, info.ReleaseInfo, getMinioMode()); err != nil {
+	if err = verifyBinary(info.URL, info.Sha256Sum, info.ReleaseInfo, getMinioMode(), info.BinaryFile); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+}
+
+// CommitBinary - overwrites the current binary with the new one.
+func (s *peerRESTServer) CommitBinaryHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	if err := commitBinary(); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -1297,7 +1310,8 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(httpTraceHdrs(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBucketStats).HandlerFunc(httpTraceHdrs(server.GetBucketStatsHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSignalService).HandlerFunc(httpTraceHdrs(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerUpdate).HandlerFunc(httpTraceHdrs(server.ServerUpdateHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadBinary).HandlerFunc(httpTraceHdrs(server.DownloadBinaryHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCommitBinary).HandlerFunc(httpTraceHdrs(server.CommitBinaryHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeletePolicy).HandlerFunc(httpTraceAll(server.DeletePolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicy).HandlerFunc(httpTraceAll(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicyMapping).HandlerFunc(httpTraceAll(server.LoadPolicyMappingHandler)).Queries(restQueries(peerRESTUserOrGroup)...)

cmd/post-policy_test.go

@@ -141,7 +141,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 	// objectNames[0].
 	// uploadIds [0].
 	// Create bucket before initiating NewMultipartUpload.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucketName, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -460,7 +460,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
 	curTime := UTCNow()
 	curTimePlus5Min := curTime.Add(time.Minute * 5)
 
-	err = obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
+	err = obj.MakeBucketWithLocation(context.Background(), bucketName, MakeBucketOptions{})
 	if err != nil {
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())

cmd/server-main.go

@@ -607,7 +607,7 @@ func serverMain(ctx *cli.Context) {
 		initDataScanner(GlobalContext, newObject)
 
 		// List buckets to heal, and be re-used for loading configs.
-		buckets, err := newObject.ListBuckets(GlobalContext)
+		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
 		if err != nil {
 			logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to heal: %w", err))
 		}

cmd/site-replication.go

@@ -627,7 +627,7 @@ const (
 // replication is enabled. It is responsible for the creation of the same bucket
 // on remote clusters, and creating replication rules on local and peer
 // clusters.
-func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string, opts BucketOptions) error {
+func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	// At this point, the local bucket is created.
 
 	c.RLock()
@@ -650,6 +650,9 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 	if opts.ForceCreate {
 		optsMap["forceCreate"] = "true"
 	}
+	createdAt, _ := globalBucketMetadataSys.CreatedAt(bucket)
+	optsMap["createdAt"] = createdAt.Format(time.RFC3339Nano)
+	opts.CreatedAt = createdAt
 
 	// Create bucket and enable versioning on all peers.
 	makeBucketConcErr := c.concDo(
@@ -726,7 +729,7 @@ func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string
 }
 
 // PeerBucketMakeWithVersioningHandler - creates bucket and enables versioning.
-func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Context, bucket string, opts BucketOptions) error {
+func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
@@ -751,6 +754,8 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 		return wrapSRErr(c.annotateErr(makeBucketWithVersion, err))
 	}
 
+	meta.SetCreatedAt(opts.CreatedAt)
+
 	meta.VersioningConfigXML = enabledBucketVersioningConfig
 	if opts.LockEnabled {
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
@@ -950,7 +955,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 
 // PeerBucketDeleteHandler - deletes bucket on local in response to a delete
 // bucket request from a peer.
-func (c *SiteReplicationSys) PeerBucketDeleteHandler(ctx context.Context, bucket string, forceDelete bool) error {
+func (c *SiteReplicationSys) PeerBucketDeleteHandler(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	c.RLock()
 	defer c.RUnlock()
 	if !c.enabled {
@@ -967,8 +972,7 @@ func (c *SiteReplicationSys) PeerBucketDeleteHandler(ctx context.Context, bucket
 			return err
 		}
 	}
-
-	err := objAPI.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete})
+	err := objAPI.DeleteBucket(ctx, bucket, opts)
 	if err != nil {
 		if globalDNSConfig != nil {
 			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
@@ -1498,7 +1502,7 @@ func (c *SiteReplicationSys) listBuckets(ctx context.Context) ([]BucketInfo, err
 	if objAPI == nil {
 		return nil, errSRObjectLayerNotReady
 	}
-	return objAPI.ListBuckets(ctx)
+	return objAPI.ListBuckets(ctx, BucketOptions{Deleted: true})
 }
 
 // syncToAllPeers is used for syncing local data to all remote peers, it is
@@ -1521,11 +1525,12 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 			}
 		}
 
-		var opts BucketOptions
+		var opts MakeBucketOptions
 		if lockConfig != nil {
 			opts.LockEnabled = lockConfig.ObjectLockEnabled == "Enabled"
 		}
 
+		opts.CreatedAt, _ = globalBucketMetadataSys.CreatedAt(bucket)
 		// Now call the MakeBucketHook on existing bucket - this will
 		// create buckets and replication rules on peer clusters.
 		err = c.MakeBucketHook(ctx, bucket, opts)
@@ -2161,7 +2166,7 @@ func (c *SiteReplicationSys) RemoveRemoteTargetsForEndpoint(ctx context.Context,
 			m[t.Arn] = t
 		}
 	}
-	buckets, err := objectAPI.ListBuckets(ctx)
+	buckets, err := objectAPI.ListBuckets(ctx, BucketOptions{})
 	for _, b := range buckets {
 		config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, b.Name)
 		if err != nil {
@@ -2361,10 +2366,8 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 
 	sris := make([]madmin.SRInfo, len(c.state.Peers))
 	depIdx := make(map[string]int, len(c.state.Peers))
-	depIDs := make([]string, 0, len(c.state.Peers))
 	i := 0
 	for d := range c.state.Peers {
-		depIDs = append(depIDs, d)
 		depIdx[d] = i
 		i++
 	}
@@ -2810,14 +2813,18 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 			info.BucketStats[b] = make(map[string]srBucketStatsSummary, numSites)
 			for i, s := range slc {
 				dIdx := depIdx[s.DeploymentID]
-				var hasBucket bool
-				if bi, ok := sris[dIdx].Buckets[s.Bucket]; ok {
-					hasBucket = !bi.CreatedAt.Equal(timeSentinel)
+				var hasBucket, isBucketMarkedDeleted bool
+
+				bi, ok := sris[dIdx].Buckets[s.Bucket]
+				if ok {
+					isBucketMarkedDeleted = !bi.DeletedAt.IsZero() && (bi.CreatedAt.IsZero() || bi.DeletedAt.After(bi.CreatedAt))
+					hasBucket = !bi.CreatedAt.IsZero()
 				}
 				quotaCfgSet := hasBucket && quotaCfgs[i] != nil && *quotaCfgs[i] != madmin.BucketQuota{}
 				ss := madmin.SRBucketStatsSummary{
 					DeploymentID:             s.DeploymentID,
 					HasBucket:                hasBucket,
+					BucketMarkedDeleted:      isBucketMarkedDeleted,
 					TagMismatch:              tagMismatch,
 					OLockConfigMismatch:      olockCfgMismatch,
 					SSEConfigMismatch:        sseCfgMismatch,
@@ -3083,13 +3090,16 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 			err     error
 		)
 		if opts.Entity == madmin.SRBucketEntity {
-			bi, err := objAPI.GetBucketInfo(ctx, opts.EntityValue)
+			bi, err := objAPI.GetBucketInfo(ctx, opts.EntityValue, BucketOptions{Deleted: opts.ShowDeleted})
 			if err != nil {
+				if isErrBucketNotFound(err) {
+					return info, nil
+				}
 				return info, errSRBackendIssue(err)
 			}
 			buckets = append(buckets, bi)
 		} else {
-			buckets, err = objAPI.ListBuckets(ctx)
+			buckets, err = objAPI.ListBuckets(ctx, BucketOptions{Deleted: opts.ShowDeleted})
 			if err != nil {
 				return info, errSRBackendIssue(err)
 			}
@@ -3097,16 +3107,17 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
 		info.Buckets = make(map[string]madmin.SRBucketInfo, len(buckets))
 		for _, bucketInfo := range buckets {
 			bucket := bucketInfo.Name
-			createdAt, err := globalBucketMetadataSys.CreatedAt(bucket)
-			if err != nil {
-				return info, errSRBackendIssue(err)
-			}
+			bucketExists := bucketInfo.Deleted.IsZero() || (!bucketInfo.Created.IsZero() && bucketInfo.Created.After(bucketInfo.Deleted))
 			bms := madmin.SRBucketInfo{
 				Bucket:    bucket,
-				CreatedAt: createdAt,
+				CreatedAt: bucketInfo.Created.UTC(),
+				DeletedAt: bucketInfo.Deleted.UTC(),
 				Location:  globalSite.Region,
 			}
-
+			if !bucketExists {
+				info.Buckets[bucket] = bms
+				continue
+			}
 			// Get bucket policy if present.
 			policy, updatedAt, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
 			found := true
@@ -3555,22 +3566,60 @@ type srStatusInfo struct {
 	GroupStats map[string]map[string]srGroupStatsSummary
 }
 
+// SRBucketDeleteOp - type of delete op
+type SRBucketDeleteOp string
+
+const (
+	// MarkDelete creates .minio.sys/buckets/.deleted/<bucket> vol entry to hold onto deleted bucket's state
+	// until peers are synced in site replication setup.
+	MarkDelete SRBucketDeleteOp = "MarkDelete"
+
+	// Purge deletes the .minio.sys/buckets/.deleted/<bucket> vol entry
+	Purge SRBucketDeleteOp = "Purge"
+	// NoOp no action needed
+	NoOp SRBucketDeleteOp = "NoOp"
+)
+
+// Empty returns true if this Op is not set
+func (s SRBucketDeleteOp) Empty() bool {
+	return string(s) == "" || string(s) == string(NoOp)
+}
+
+func getSRBucketDeleteOp(isSiteReplicated bool) SRBucketDeleteOp {
+	if !isSiteReplicated {
+		return NoOp
+	}
+	return MarkDelete
+}
+
 func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer) error {
-	info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
-		Buckets: true,
-	})
+	buckets, err := c.listBuckets(ctx)
 	if err != nil {
 		return err
 	}
-	for bucket := range info.BucketStats {
-		c.healCreateMissingBucket(ctx, objAPI, bucket, info)
-		c.healVersioningMetadata(ctx, objAPI, bucket, info)
-		c.healOLockConfigMetadata(ctx, objAPI, bucket, info)
-		c.healSSEMetadata(ctx, objAPI, bucket, info)
-		c.healBucketReplicationConfig(ctx, objAPI, bucket, info)
-		c.healBucketPolicies(ctx, objAPI, bucket, info)
-		c.healTagMetadata(ctx, objAPI, bucket, info)
-		c.healBucketQuotaConfig(ctx, objAPI, bucket, info)
+	for _, bi := range buckets {
+		bucket := bi.Name
+		info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
+			Entity:      madmin.SRBucketEntity,
+			EntityValue: bucket,
+			ShowDeleted: true,
+		})
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+
+		c.healBucket(ctx, objAPI, bucket, info)
+
+		if bi.Deleted.IsZero() || (!bi.Created.IsZero() && bi.Deleted.Before(bi.Created)) {
+			c.healVersioningMetadata(ctx, objAPI, bucket, info)
+			c.healOLockConfigMetadata(ctx, objAPI, bucket, info)
+			c.healSSEMetadata(ctx, objAPI, bucket, info)
+			c.healBucketReplicationConfig(ctx, objAPI, bucket, info)
+			c.healBucketPolicies(ctx, objAPI, bucket, info)
+			c.healTagMetadata(ctx, objAPI, bucket, info)
+			c.healBucketQuotaConfig(ctx, objAPI, bucket, info)
+		}
 		// Notification and ILM are site specific settings.
 	}
 	return nil
@@ -4014,84 +4063,176 @@ func (c *SiteReplicationSys) healOLockConfigMetadata(ctx context.Context, objAPI
 	return nil
 }
 
-func (c *SiteReplicationSys) healCreateMissingBucket(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
-	bs := info.BucketStats[bucket]
+func (c *SiteReplicationSys) purgeDeletedBucket(ctx context.Context, objAPI ObjectLayer, bucket string) {
+	z, ok := objAPI.(*erasureServerPools)
+	if !ok {
+		if z, ok := objAPI.(*erasureSingle); ok {
+			z.purgeDelete(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+		}
+		return
+	}
+	z.purgeDelete(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+}
 
+// healBucket creates/deletes the bucket according to latest state across clusters participating in site replication.
+func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
+	bs := info.BucketStats[bucket]
 	c.RLock()
 	defer c.RUnlock()
 	if !c.enabled {
 		return nil
 	}
+	numSites := len(c.state.Peers)
+	mostRecent := func(d1, d2 time.Time) time.Time {
+		if d1.IsZero() {
+			return d2
+		}
+		if d2.IsZero() {
+			return d1
+		}
+		if d1.After(d2) {
+			return d1
+		}
+		return d2
+	}
 
-	bucketCnt := 0
 	var (
 		latestID   string
 		lastUpdate time.Time
+		withB      []string
+		missingB   []string
+		deletedCnt int
 	)
-
-	var dIDs []string
 	for dID, ss := range bs {
-		if ss.HasBucket {
-			bucketCnt++
-		} else {
-			dIDs = append(dIDs, dID)
-		}
 		if lastUpdate.IsZero() {
-			lastUpdate = ss.meta.CreatedAt
+			lastUpdate = mostRecent(ss.meta.CreatedAt, ss.meta.DeletedAt)
 			latestID = dID
 		}
-		if ss.meta.CreatedAt.After(lastUpdate) {
-			lastUpdate = ss.meta.CreatedAt
+		recentUpdt := mostRecent(ss.meta.CreatedAt, ss.meta.DeletedAt)
+		if recentUpdt.After(lastUpdate) {
+			lastUpdate = recentUpdt
 			latestID = dID
 		}
+		if ss.BucketMarkedDeleted {
+			deletedCnt++
+		}
+		if ss.HasBucket {
+			withB = append(withB, dID)
+		} else {
+			missingB = append(missingB, dID)
+		}
 	}
+
 	latestPeerName := info.Sites[latestID].Name
 	bStatus := info.BucketStats[bucket][latestID].meta
-	var opts BucketOptions
-	optsMap := make(map[string]string)
-	if bStatus.Location != "" {
-		optsMap["location"] = bStatus.Location
-		opts.Location = bStatus.Location
+	isMakeBucket := len(missingB) > 0
+	deleteOp := NoOp
+	if latestID != globalDeploymentID {
+		return nil
 	}
+	if lastUpdate.Equal(bStatus.DeletedAt) {
+		isMakeBucket = false
+		switch {
+		case len(withB) == numSites && deletedCnt == numSites:
+			deleteOp = NoOp
+		case len(withB) == 0 && len(missingB) == numSites:
+			deleteOp = Purge
+		default:
+			deleteOp = MarkDelete
+		}
+	}
+	if isMakeBucket {
+		var opts MakeBucketOptions
+		optsMap := make(map[string]string)
+		if bStatus.Location != "" {
+			optsMap["location"] = bStatus.Location
+			opts.Location = bStatus.Location
+		}
 
-	optsMap["versioningEnabled"] = "true"
-	opts.VersioningEnabled = true
-	if bStatus.ObjectLockConfig != nil {
-		config, err := base64.StdEncoding.DecodeString(*bStatus.ObjectLockConfig)
-		if err != nil {
-			return err
+		optsMap["versioningEnabled"] = "true"
+		opts.VersioningEnabled = true
+		opts.CreatedAt = bStatus.CreatedAt
+		optsMap["createdAt"] = bStatus.CreatedAt.Format(time.RFC3339Nano)
+
+		if bStatus.ObjectLockConfig != nil {
+			config, err := base64.StdEncoding.DecodeString(*bStatus.ObjectLockConfig)
+			if err != nil {
+				return err
+			}
+			if bytes.Equal([]byte(string(config)), enabledBucketObjectLockConfig) {
+				optsMap["lockEnabled"] = "true"
+				opts.LockEnabled = true
+			}
 		}
-		if bytes.Equal([]byte(string(config)), enabledBucketObjectLockConfig) {
-			optsMap["lockEnabled"] = "true"
-			opts.LockEnabled = true
+		for _, dID := range missingB {
+			peerName := info.Sites[dID].Name
+			if dID == globalDeploymentID {
+				err := c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
+				if err != nil {
+					return c.annotateErr(makeBucketWithVersion, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
+						err, latestPeerName, peerName))
+				}
+			} else {
+				admClient, err := c.getAdminClient(ctx, dID)
+				if err != nil {
+					return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
+				}
+				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap); err != nil {
+					return c.annotatePeerErr(peerName, makeBucketWithVersion, err)
+				}
+				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil); err != nil {
+					return c.annotatePeerErr(peerName, configureReplication, err)
+				}
+			}
 		}
+		if len(missingB) > 0 {
+			// configure replication from current cluster to other clusters
+			err := c.PeerBucketConfigureReplHandler(ctx, bucket)
+			if err != nil {
+				return c.annotateErr(configureReplication, err)
+			}
+		}
+		return nil
 	}
-	for _, dID := range dIDs {
-		peerName := info.Sites[dID].Name
-		if dID == globalDeploymentID {
-			err := c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
-			if err != nil {
-				return c.annotateErr(makeBucketWithVersion, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
-					err, latestPeerName, peerName))
-			}
-		} else {
-			admClient, err := c.getAdminClient(ctx, dID)
-			if err != nil {
-				return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
-			}
-			if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap); err != nil {
-				return c.annotatePeerErr(peerName, makeBucketWithVersion, err)
-			}
-			if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil); err != nil {
-				return c.annotatePeerErr(peerName, configureReplication, err)
+	// all buckets are marked deleted across sites at this point. It should be safe to purge the .minio.sys/buckets/.deleted/<bucket> entry
+	// from disk
+	if deleteOp == Purge {
+		for _, dID := range missingB {
+			peerName := info.Sites[dID].Name
+			if dID == globalDeploymentID {
+				c.purgeDeletedBucket(ctx, objAPI, bucket)
+			} else {
+				admClient, err := c.getAdminClient(ctx, dID)
+				if err != nil {
+					return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
+				}
+				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.PurgeDeletedBucketOp, nil); err != nil {
+					return c.annotatePeerErr(peerName, deleteBucket, err)
+				}
 			}
 		}
 	}
-	if len(dIDs) > 0 {
-		// configure replication from current cluster to other clusters
-		err := c.PeerBucketConfigureReplHandler(ctx, bucket)
-		if err != nil {
-			return c.annotateErr(configureReplication, err)
+	// Mark buckets deleted on remaining peers
+	if deleteOp == MarkDelete {
+		for _, dID := range withB {
+			peerName := info.Sites[dID].Name
+			if dID == globalDeploymentID {
+				err := c.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
+					Force: true,
+				})
+				if err != nil {
+					return c.annotateErr(deleteBucket, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
+						err, latestPeerName, peerName))
+				}
+			} else {
+				admClient, err := c.getAdminClient(ctx, dID)
+				if err != nil {
+					return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
+				}
+				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ForceDeleteBucketBktOp, nil); err != nil {
+					return c.annotatePeerErr(peerName, deleteBucket, err)
+				}
+			}
 		}
 	}
 

cmd/speedtest.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"runtime"
 	"sort"
 	"time"
 
@@ -48,6 +49,38 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
 
 		concurrency := opts.concurrencyStart
 
+		if opts.autotune {
+			// if we have less drives than concurrency then choose
+			// only the concurrency to be number of drives to start
+			// with - since default '32' might be big and may not
+			// complete in total time of 10s.
+			if globalEndpoints.NEndpoints() < concurrency {
+				concurrency = globalEndpoints.NEndpoints()
+			}
+
+			// Check if we have local disks per pool less than
+			// the concurrency make sure we choose only the "start"
+			// concurrency to be equal to the lowest number of
+			// local disks per server.
+			for _, localDiskCount := range globalEndpoints.NLocalDisksPathsPerPool() {
+				if localDiskCount < concurrency {
+					concurrency = localDiskCount
+				}
+			}
+
+			// Any concurrency less than '4' just stick to '4' concurrent
+			// operations for now to begin with.
+			if concurrency < 4 {
+				concurrency = 4
+			}
+
+			// if GOMAXPROCS is set to a lower value then choose to use
+			// concurrency == GOMAXPROCS instead.
+			if runtime.GOMAXPROCS(0) < concurrency {
+				concurrency = runtime.GOMAXPROCS(0)
+			}
+		}
+
 		throughputHighestGet := uint64(0)
 		throughputHighestPut := uint64(0)
 		var throughputHighestResults []SpeedTestResult
@@ -97,7 +130,11 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
 			result.Version = Version
 			result.Concurrent = concurrency
 
-			ch <- result
+			select {
+			case ch <- result:
+			case <-ctx.Done():
+				return
+			}
 		}
 
 		for {

cmd/storage-rest_test.go

@@ -20,9 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
 	"net/http/httptest"
-	"os"
 	"reflect"
 	"runtime"
 	"testing"
@@ -439,25 +437,21 @@ func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) {
 	}
 }
 
-func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRESTClient, string) {
+func newStorageRESTHTTPServerClient(t *testing.T) *storageRESTClient {
 	prevHost, prevPort := globalMinioHost, globalMinioPort
 	defer func() {
 		globalMinioHost, globalMinioPort = prevHost, prevPort
 	}()
 
-	endpointPath, err := ioutil.TempDir("", ".TestStorageREST.")
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
 	router := mux.NewRouter()
 	httpServer := httptest.NewServer(router)
+	t.Cleanup(httpServer.Close)
 
 	url, err := xnet.ParseHTTPURL(httpServer.URL)
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
-	url.Path = endpointPath
+	url.Path = t.TempDir()
 
 	globalMinioHost, globalMinioPort = mustSplitHostPort(url.Host)
 
@@ -476,101 +470,77 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES
 
 	restClient := newStorageRESTClient(endpoint, false)
 
-	return httpServer, restClient, endpointPath
+	return restClient
 }
 
 func TestStorageRESTClientDiskInfo(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIDiskInfo(t, restClient)
 }
 
 func TestStorageRESTClientMakeVol(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIMakeVol(t, restClient)
 }
 
 func TestStorageRESTClientListVols(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIListVols(t, restClient)
 }
 
 func TestStorageRESTClientStatVol(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIStatVol(t, restClient)
 }
 
 func TestStorageRESTClientDeleteVol(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIDeleteVol(t, restClient)
 }
 
 func TestStorageRESTClientStatInfoFile(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIStatInfoFile(t, restClient)
 }
 
 func TestStorageRESTClientListDir(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIListDir(t, restClient)
 }
 
 func TestStorageRESTClientReadAll(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIReadAll(t, restClient)
 }
 
 func TestStorageRESTClientReadFile(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIReadFile(t, restClient)
 }
 
 func TestStorageRESTClientAppendFile(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIAppendFile(t, restClient)
 }
 
 func TestStorageRESTClientDeleteFile(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIDeleteFile(t, restClient)
 }
 
 func TestStorageRESTClientRenameFile(t *testing.T) {
-	httpServer, restClient, endpointPath := newStorageRESTHTTPServerClient(t)
-	defer httpServer.Close()
-	defer os.RemoveAll(endpointPath)
+	restClient := newStorageRESTHTTPServerClient(t)
 
 	testStorageAPIRenameFile(t, restClient)
 }

cmd/test-utils_test.go

@@ -1452,11 +1452,6 @@ func getListenNotificationURL(endPoint, bucketName string, prefixes, suffixes, e
 	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
 }
 
-// returns temp root directory. `
-func getTestRoot() (string, error) {
-	return ioutil.TempDir(globalTestTmpDir, "api-")
-}
-
 // getRandomDisks - Creates a slice of N random disks, each of the form - minio-XXX
 func getRandomDisks(N int) ([]string, error) {
 	var erasureDisks []string
@@ -1528,7 +1523,7 @@ func initAPIHandlerTest(ctx context.Context, obj ObjectLayer, endpoints []string
 	bucketName := getRandomBucketName()
 
 	// Create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
+	err := obj.MakeBucketWithLocation(context.Background(), bucketName, MakeBucketOptions{})
 	if err != nil {
 		// failed to create newbucket, return err.
 		return "", nil, err

cmd/tree-walk_test.go

@@ -20,8 +20,6 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
-	"os"
 	"reflect"
 	"sort"
 	"strings"
@@ -137,11 +135,7 @@ func testTreeWalkMarker(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, is
 
 // Test tree-walk.
 func TestTreeWalk(t *testing.T) {
-	fsDir, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Unable to create tmp directory: %s", err)
-	}
-	defer os.RemoveAll(fsDir)
+	fsDir := t.TempDir()
 
 	endpoints := mustGetNewEndpoints(fsDir)
 	disk, err := newStorageAPI(endpoints[0])
@@ -181,11 +175,7 @@ func TestTreeWalk(t *testing.T) {
 
 // Test if tree walk go-routine exits cleanly if tree walk is aborted because of timeout.
 func TestTreeWalkTimeout(t *testing.T) {
-	fsDir, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Unable to create tmp directory: %s", err)
-	}
-	defer os.RemoveAll(fsDir)
+	fsDir := t.TempDir()
 	endpoints := mustGetNewEndpoints(fsDir)
 	disk, err := newStorageAPI(endpoints[0])
 	if err != nil {
@@ -254,11 +244,7 @@ func TestTreeWalkTimeout(t *testing.T) {
 // without recursively traversing prefixes.
 func TestRecursiveTreeWalk(t *testing.T) {
 	// Create a backend directories fsDir1.
-	fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Unable to create tmp directory: %s", err)
-	}
-	defer os.RemoveAll(fsDir1)
+	fsDir1 := t.TempDir()
 
 	endpoints := mustGetNewEndpoints(fsDir1)
 	disk1, err := newStorageAPI(endpoints[0])
@@ -365,11 +351,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
 
 func TestSortedness(t *testing.T) {
 	// Create a backend directories fsDir1.
-	fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Errorf("Unable to create tmp directory: %s", err)
-	}
-	defer os.RemoveAll(fsDir1)
+	fsDir1 := t.TempDir()
 
 	endpoints := mustGetNewEndpoints(fsDir1)
 	disk1, err := newStorageAPI(endpoints[0])
@@ -440,11 +422,7 @@ func TestSortedness(t *testing.T) {
 
 func TestTreeWalkIsEnd(t *testing.T) {
 	// Create a backend directories fsDir1.
-	fsDir1, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Errorf("Unable to create tmp directory: %s", err)
-	}
-	defer os.RemoveAll(fsDir1)
+	fsDir1 := t.TempDir()
 
 	endpoints := mustGetNewEndpoints(fsDir1)
 	disk1, err := newStorageAPI(endpoints[0])

cmd/update.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bufio"
+	"bytes"
 	"crypto"
 	"crypto/tls"
 	"encoding/hex"
@@ -291,63 +292,50 @@ func getUserAgent(mode string) string {
 }
 
 func downloadReleaseURL(u *url.URL, timeout time.Duration, mode string) (content string, err error) {
-	var reader io.ReadCloser
-	if u.Scheme == "https" || u.Scheme == "http" {
-		req, err := http.NewRequest(http.MethodGet, u.String(), nil)
-		if err != nil {
-			return content, AdminError{
-				Code:       AdminUpdateUnexpectedFailure,
-				Message:    err.Error(),
-				StatusCode: http.StatusInternalServerError,
-			}
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return content, AdminError{
+			Code:       AdminUpdateUnexpectedFailure,
+			Message:    err.Error(),
+			StatusCode: http.StatusInternalServerError,
 		}
-		req.Header.Set("User-Agent", getUserAgent(mode))
+	}
+	req.Header.Set("User-Agent", getUserAgent(mode))
 
-		client := &http.Client{Transport: getUpdateTransport(timeout)}
-		resp, err := client.Do(req)
-		if err != nil {
-			if xnet.IsNetworkOrHostDown(err, false) {
-				return content, AdminError{
-					Code:       AdminUpdateURLNotReachable,
-					Message:    err.Error(),
-					StatusCode: http.StatusServiceUnavailable,
-				}
-			}
-			return content, AdminError{
-				Code:       AdminUpdateUnexpectedFailure,
-				Message:    err.Error(),
-				StatusCode: http.StatusInternalServerError,
-			}
-		}
-		if resp == nil {
-			return content, AdminError{
-				Code:       AdminUpdateUnexpectedFailure,
-				Message:    fmt.Sprintf("No response from server to download URL %s", u),
-				StatusCode: http.StatusInternalServerError,
-			}
-		}
-		reader = resp.Body
-		defer xhttp.DrainBody(resp.Body)
-
-		if resp.StatusCode != http.StatusOK {
-			return content, AdminError{
-				Code:       AdminUpdateUnexpectedFailure,
-				Message:    fmt.Sprintf("Error downloading URL %s. Response: %v", u, resp.Status),
-				StatusCode: resp.StatusCode,
-			}
-		}
-	} else {
-		reader, err = os.Open(u.Path)
-		if err != nil {
+	client := &http.Client{Transport: getUpdateTransport(timeout)}
+	resp, err := client.Do(req)
+	if err != nil {
+		if xnet.IsNetworkOrHostDown(err, false) {
 			return content, AdminError{
 				Code:       AdminUpdateURLNotReachable,
 				Message:    err.Error(),
 				StatusCode: http.StatusServiceUnavailable,
 			}
 		}
+		return content, AdminError{
+			Code:       AdminUpdateUnexpectedFailure,
+			Message:    err.Error(),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	if resp == nil {
+		return content, AdminError{
+			Code:       AdminUpdateUnexpectedFailure,
+			Message:    fmt.Sprintf("No response from server to download URL %s", u),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+	defer xhttp.DrainBody(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		return content, AdminError{
+			Code:       AdminUpdateUnexpectedFailure,
+			Message:    fmt.Sprintf("Error downloading URL %s. Response: %v", u, resp.Status),
+			StatusCode: resp.StatusCode,
+		}
 	}
 
-	contentBytes, err := ioutil.ReadAll(reader)
+	contentBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return content, AdminError{
 			Code:       AdminUpdateUnexpectedFailure,
@@ -518,26 +506,38 @@ func getUpdateReaderFromURL(u *url.URL, transport http.RoundTripper, mode string
 
 var updateInProgress uint32
 
-func doUpdate(u *url.URL, lrTime time.Time, sha256Sum []byte, releaseInfo string, mode string) (err error) {
+// Function to get the reader from an architecture
+func downloadBinary(u *url.URL, mode string) (readerReturn []byte, err error) {
+	transport := getUpdateTransport(30 * time.Second)
+	var reader io.ReadCloser
+	if u.Scheme == "https" || u.Scheme == "http" {
+		reader, err = getUpdateReaderFromURL(u, transport, mode)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		reader, err = getUpdateReaderFromFile(u)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// convert a Reader to bytes
+	binaryFile, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	return binaryFile, nil
+}
+
+func verifyBinary(u *url.URL, sha256Sum []byte, releaseInfo string, mode string, reader []byte) (err error) {
 	if !atomic.CompareAndSwapUint32(&updateInProgress, 0, 1) {
 		return errors.New("update already in progress")
 	}
 	defer atomic.StoreUint32(&updateInProgress, 0)
 
 	transport := getUpdateTransport(30 * time.Second)
-	var reader io.ReadCloser
-	if u.Scheme == "https" || u.Scheme == "http" {
-		reader, err = getUpdateReaderFromURL(u, transport, mode)
-		if err != nil {
-			return err
-		}
-	} else {
-		reader, err = getUpdateReaderFromFile(u)
-		if err != nil {
-			return err
-		}
-	}
-
 	opts := selfupdate.Options{
 		Hash:     crypto.SHA256,
 		Checksum: sha256Sum,
@@ -565,7 +565,35 @@ func doUpdate(u *url.URL, lrTime time.Time, sha256Sum []byte, releaseInfo string
 		opts.Verifier = v
 	}
 
-	if err = selfupdate.Apply(reader, opts); err != nil {
+	if err = selfupdate.PrepareAndCheckBinary(bytes.NewReader(reader), opts); err != nil {
+		var pathErr *os.PathError
+		if errors.As(err, &pathErr) {
+			return AdminError{
+				Code: AdminUpdateApplyFailure,
+				Message: fmt.Sprintf("Unable to update the binary at %s: %v",
+					filepath.Dir(pathErr.Path), pathErr.Err),
+				StatusCode: http.StatusForbidden,
+			}
+		}
+		return AdminError{
+			Code:       AdminUpdateApplyFailure,
+			Message:    err.Error(),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+
+	return nil
+}
+
+func commitBinary() (err error) {
+	if !atomic.CompareAndSwapUint32(&updateInProgress, 0, 1) {
+		return errors.New("update already in progress")
+	}
+	defer atomic.StoreUint32(&updateInProgress, 0)
+
+	opts := selfupdate.Options{}
+
+	if err = selfupdate.CommitBinary(opts); err != nil {
 		if rerr := selfupdate.RollbackError(err); rerr != nil {
 			return AdminError{
 				Code:       AdminUpdateApplyFailure,

cmd/xl-storage.go

@@ -1647,6 +1647,8 @@ func (s *xlStorage) openFileSync(filePath string, mode int) (f *os.File, err err
 			return nil, errIsNotRegular
 		case osIsPermission(err):
 			return nil, errFileAccessDenied
+		case isSysErrNotDir(err):
+			return nil, errFileAccessDenied
 		case isSysErrIO(err):
 			return nil, errFaultyDisk
 		case isSysErrTooManyFiles(err):
@@ -2487,8 +2489,10 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 				return errFileAccessDenied
 			}
 			if err = Remove(dstFilePath); err != nil {
-				if isSysErrNotEmpty(err) {
+				if isSysErrNotEmpty(err) || isSysErrNotDir(err) {
 					return errFileAccessDenied
+				} else if isSysErrIO(err) {
+					return errFaultyDisk
 				}
 				return err
 			}
@@ -2496,6 +2500,9 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 	}
 
 	if err = renameAll(srcFilePath, dstFilePath); err != nil {
+		if isSysErrNotEmpty(err) || isSysErrNotDir(err) {
+			return errFileAccessDenied
+		}
 		return osErrToFileErr(err)
 	}
 

cmd/xl-storage_test.go

@@ -117,11 +117,8 @@ func TestIsValidVolname(t *testing.T) {
 
 // creates a temp dir and sets up xlStorage layer.
 // returns xlStorage layer, temp dir path to be used for the purpose of tests.
-func newXLStorageTestSetup() (*xlStorageDiskIDCheck, string, error) {
-	diskPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		return nil, "", err
-	}
+func newXLStorageTestSetup(tb testing.TB) (*xlStorageDiskIDCheck, string, error) {
+	diskPath := tb.TempDir()
 
 	// Initialize a new xlStorage layer.
 	storage, err := newLocalXLStorage(diskPath)
@@ -141,64 +138,37 @@ func newXLStorageTestSetup() (*xlStorageDiskIDCheck, string, error) {
 
 // createPermDeniedFile - creates temporary directory and file with path '/mybucket/myobject'
 func createPermDeniedFile(t *testing.T) (permDeniedDir string) {
-	var errMsg string
-
-	defer func() {
-		if errMsg == "" {
-			return
-		}
-
-		if permDeniedDir != "" {
-			os.RemoveAll(permDeniedDir)
-		}
-
-		t.Fatalf(errMsg)
-	}()
-
 	var err error
-	if permDeniedDir, err = ioutil.TempDir(globalTestTmpDir, "minio-"); err != nil {
-		errMsg = fmt.Sprintf("Unable to create temporary directory. %v", err)
-		return permDeniedDir
-	}
+	permDeniedDir = t.TempDir()
 
 	if err = os.Mkdir(slashpath.Join(permDeniedDir, "mybucket"), 0o775); err != nil {
-		errMsg = fmt.Sprintf("Unable to create temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err)
-		return permDeniedDir
+		t.Fatalf(fmt.Sprintf("Unable to create temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err))
 	}
 
 	if err = ioutil.WriteFile(slashpath.Join(permDeniedDir, "mybucket", "myobject"), []byte(""), 0o400); err != nil {
-		errMsg = fmt.Sprintf("Unable to create file %v. %v", slashpath.Join(permDeniedDir, "mybucket", "myobject"), err)
-		return permDeniedDir
+		t.Fatalf(fmt.Sprintf("Unable to create file %v. %v", slashpath.Join(permDeniedDir, "mybucket", "myobject"), err))
 	}
 
 	if err = os.Chmod(slashpath.Join(permDeniedDir, "mybucket"), 0o400); err != nil {
-		errMsg = fmt.Sprintf("Unable to change permission to temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err)
-		return permDeniedDir
+		t.Fatalf(fmt.Sprintf("Unable to change permission to temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err))
 	}
+	t.Cleanup(func() {
+		os.Chmod(slashpath.Join(permDeniedDir, "mybucket"), 0o775)
+	})
 
 	if err = os.Chmod(permDeniedDir, 0o400); err != nil {
-		errMsg = fmt.Sprintf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err)
+		t.Fatalf(fmt.Sprintf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err))
 	}
+	t.Cleanup(func() {
+		os.Chmod(permDeniedDir, 0o775)
+	})
 
 	return permDeniedDir
 }
 
-// removePermDeniedFile - removes temporary directory and file with path '/mybucket/myobject'
-func removePermDeniedFile(permDeniedDir string) {
-	if err := os.Chmod(permDeniedDir, 0o775); err == nil {
-		if err = os.Chmod(slashpath.Join(permDeniedDir, "mybucket"), 0o775); err == nil {
-			os.RemoveAll(permDeniedDir)
-		}
-	}
-}
-
 // TestXLStorages xlStorage.getDiskInfo()
 func TestXLStorageGetDiskInfo(t *testing.T) {
-	path, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Unable to create a temporary directory, %s", err)
-	}
-	defer os.RemoveAll(path)
+	path := t.TempDir()
 
 	testCases := []struct {
 		diskPath    string
@@ -217,11 +187,7 @@ func TestXLStorageGetDiskInfo(t *testing.T) {
 }
 
 func TestXLStorageIsDirEmpty(t *testing.T) {
-	tmp, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmp)
+	tmp := t.TempDir()
 
 	// Should give false on non-existent directory.
 	dir1 := slashpath.Join(tmp, "non-existent-directory")
@@ -231,7 +197,7 @@ func TestXLStorageIsDirEmpty(t *testing.T) {
 
 	// Should give false for not-a-directory.
 	dir2 := slashpath.Join(tmp, "file")
-	err = ioutil.WriteFile(dir2, []byte("hello"), 0o777)
+	err := ioutil.WriteFile(dir2, []byte("hello"), 0o777)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -256,13 +222,11 @@ func TestXLStorageReadVersionLegacy(t *testing.T) {
 	const legacyJSON = `{"version":"1.0.1","format":"xl","stat":{"size":2016,"modTime":"2021-10-11T23:40:34.914361617Z"},"erasure":{"algorithm":"klauspost/reedsolomon/vandermonde","data":2,"parity":2,"blockSize":10485760,"index":2,"distribution":[2,3,4,1],"checksum":[{"name":"part.1","algorithm":"highwayhash256S"}]},"minio":{"release":"RELEASE.2019-12-30T05-45-39Z"},"meta":{"X-Minio-Internal-Server-Side-Encryption-Iv":"kInsJB/0yxyz/40ZI+lmQYJfZacDYqZsGh2wEiv+N50=","X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":"my-minio-key","X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key":"eyJhZWFkIjoiQUVTLTI1Ni1HQ00tSE1BQy1TSEEtMjU2IiwiaWQiOiJjMzEwNDVjODFmMTA2MWU5NTI4ODcxZmNhMmRkYzA3YyIsIml2IjoiOWQ5cUxGMFhSaFBXbEVqT2JDMmo0QT09Iiwibm9uY2UiOiJYaERsemlCU1cwSENuK2RDIiwiYnl0ZXMiOiJUM0lmY1haQ1dtMWpLeWxBWmFUUnczbDVoYldLWW95dm5iNTZVaWJEbE5LOFZVU2tuQmx3NytIMG8yZnRzZ1UrIn0=","X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":"IAAfANqt801MT+wwzQRkfFhTrndmhfNiN0alKwDS4AQ1dznNADRQgoq6I4pPVfRsbDp5rQawlripQZvPWUSNJA==","X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":"DAREv2-HMAC-SHA256","content-type":"application/octet-stream","etag":"20000f00cf5e68d3d6b60e44fcd8b9e8-1"},"parts":[{"number":1,"name":"part.1","etag":"","size":2016,"actualSize":1984}]}`
 
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to cfgreate xlStorage test setup, %s", err)
 	}
 
-	defer os.RemoveAll(path)
-
 	// Create files for the test cases.
 	if err = xlStorage.MakeVol(context.Background(), "exists-legacy"); err != nil {
 		t.Fatalf("Unable to create a volume \"exists-legacy\", %s", err)
@@ -285,13 +249,11 @@ func TestXLStorageReadVersionLegacy(t *testing.T) {
 // TestXLStorageReadVersion - TestXLStorages the functionality implemented by xlStorage ReadVersion storage API.
 func TestXLStorageReadVersion(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to cfgreate xlStorage test setup, %s", err)
 	}
 
-	defer os.RemoveAll(path)
-
 	xlMeta, _ := ioutil.ReadFile("testdata/xl.meta")
 
 	// Create files for the test cases.
@@ -370,13 +332,11 @@ func TestXLStorageReadVersion(t *testing.T) {
 // TestXLStorageReadAll - TestXLStorages the functionality implemented by xlStorage ReadAll storage API.
 func TestXLStorageReadAll(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
 
-	defer os.RemoveAll(path)
-
 	// Create files for the test cases.
 	if err = xlStorage.MakeVol(context.Background(), "exists"); err != nil {
 		t.Fatalf("Unable to create a volume \"exists\", %s", err)
@@ -505,11 +465,10 @@ func TestNewXLStorage(t *testing.T) {
 // Asserts the failures too against the expected failures.
 func TestXLStorageMakeVol(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	// Create a file.
@@ -558,11 +517,7 @@ func TestXLStorageMakeVol(t *testing.T) {
 
 	// TestXLStorage for permission denied.
 	if runtime.GOOS != globalWindowsOSName {
-		permDeniedDir, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-		if err != nil {
-			t.Fatalf("Unable to create temporary directory. %v", err)
-		}
-		defer os.RemoveAll(permDeniedDir)
+		permDeniedDir := createPermDeniedFile(t)
 		if err = os.Chmod(permDeniedDir, 0o400); err != nil {
 			t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err)
 		}
@@ -596,11 +551,10 @@ func TestXLStorageMakeVol(t *testing.T) {
 // TestXLStorageDeleteVol - Validates the expected behavior of xlStorage.DeleteVol for various cases.
 func TestXLStorageDeleteVol(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
@@ -654,17 +608,20 @@ func TestXLStorageDeleteVol(t *testing.T) {
 
 	// TestXLStorage for permission denied.
 	if runtime.GOOS != globalWindowsOSName {
-		var permDeniedDir string
-		if permDeniedDir, err = ioutil.TempDir(globalTestTmpDir, "minio-"); err != nil {
-			t.Fatalf("Unable to create temporary directory. %v", err)
-		}
-		defer removePermDeniedFile(permDeniedDir)
+		permDeniedDir := t.TempDir()
 		if err = os.Mkdir(slashpath.Join(permDeniedDir, "mybucket"), 0o400); err != nil {
 			t.Fatalf("Unable to create temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err)
 		}
+		t.Cleanup(func() {
+			os.Chmod(slashpath.Join(permDeniedDir, "mybucket"), 0o775)
+		})
+
 		if err = os.Chmod(permDeniedDir, 0o400); err != nil {
 			t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err)
 		}
+		t.Cleanup(func() {
+			os.Chmod(permDeniedDir, 0o775)
+		})
 
 		// Initialize xlStorage storage layer for permission denied error.
 		_, err = newLocalXLStorage(permDeniedDir)
@@ -691,7 +648,7 @@ func TestXLStorageDeleteVol(t *testing.T) {
 		}
 	}
 
-	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup()
+	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
@@ -709,11 +666,10 @@ func TestXLStorageDeleteVol(t *testing.T) {
 // TestXLStorageStatVol - TestXLStorages validate the volume info returned by xlStorage.StatVol() for various inputs.
 func TestXLStorageStatVol(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
@@ -756,7 +712,7 @@ func TestXLStorageStatVol(t *testing.T) {
 		}
 	}
 
-	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup()
+	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
@@ -774,7 +730,7 @@ func TestXLStorageStatVol(t *testing.T) {
 // TestXLStorageListVols - Validates the result and the error output for xlStorage volume listing functionality xlStorage.ListVols().
 func TestXLStorageListVols(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
@@ -821,14 +777,13 @@ func TestXLStorageListVols(t *testing.T) {
 // TestXLStorageListDir -  TestXLStorages validate the directory listing functionality provided by xlStorage.ListDir .
 func TestXLStorageListDir(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// create xlStorage test setup.
-	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup()
+	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
@@ -916,7 +871,6 @@ func TestXLStorageListDir(t *testing.T) {
 	// TestXLStorage for permission denied.
 	if runtime.GOOS != globalWindowsOSName {
 		permDeniedDir := createPermDeniedFile(t)
-		defer removePermDeniedFile(permDeniedDir)
 
 		// Initialize xlStorage storage layer for permission denied error.
 		_, err = newLocalXLStorage(permDeniedDir)
@@ -959,11 +913,10 @@ func TestXLStorageDeleteFile(t *testing.T) {
 	}
 
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
@@ -983,6 +936,9 @@ func TestXLStorageDeleteFile(t *testing.T) {
 	if err = os.Chmod(pathJoin(path, "no-permissions"), 0o555); err != nil {
 		t.Fatalf("Unable to chmod directory, %s", err.Error())
 	}
+	t.Cleanup(func() {
+		os.Chmod(pathJoin(path, "no-permissions"), 0o775)
+	})
 
 	testCases := []struct {
 		srcVol      string
@@ -1046,7 +1002,6 @@ func TestXLStorageDeleteFile(t *testing.T) {
 	// TestXLStorage for permission denied.
 	if runtime.GOOS != globalWindowsOSName {
 		permDeniedDir := createPermDeniedFile(t)
-		defer removePermDeniedFile(permDeniedDir)
 
 		// Initialize xlStorage storage layer for permission denied error.
 		_, err = newLocalXLStorage(permDeniedDir)
@@ -1072,7 +1027,7 @@ func TestXLStorageDeleteFile(t *testing.T) {
 	}
 
 	// create xlStorage test setup
-	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup()
+	xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
@@ -1096,11 +1051,10 @@ func TestXLStorageDeleteFile(t *testing.T) {
 // TestXLStorageReadFile - TestXLStorages xlStorage.ReadFile with wide range of cases and asserts the result and error response.
 func TestXLStorageReadFile(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	volume := "success-vol"
 	// Setup test environment.
@@ -1269,7 +1223,6 @@ func TestXLStorageReadFile(t *testing.T) {
 	// TestXLStorage for permission denied.
 	if runtime.GOOS != globalWindowsOSName {
 		permDeniedDir := createPermDeniedFile(t)
-		defer removePermDeniedFile(permDeniedDir)
 
 		// Initialize xlStorage storage layer for permission denied error.
 		_, err = newLocalXLStorage(permDeniedDir)
@@ -1325,22 +1278,18 @@ var xlStorageReadFileWithVerifyTests = []struct {
 // TestXLStorageReadFile.
 func TestXLStorageReadFileWithVerify(t *testing.T) {
 	volume, object := "test-vol", "myobject"
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
-		os.RemoveAll(path)
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
 	if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
-		os.RemoveAll(path)
 		t.Fatalf("Unable to create volume %s: %v", volume, err)
 	}
 	data := make([]byte, 8*1024)
 	if _, err = io.ReadFull(rand.Reader, data); err != nil {
-		os.RemoveAll(path)
 		t.Fatalf("Unable to create generate random data: %v", err)
 	}
 	if err = xlStorage.AppendFile(context.Background(), volume, object, data); err != nil {
-		os.RemoveAll(path)
 		t.Fatalf("Unable to create object: %v", err)
 	}
 
@@ -1369,11 +1318,10 @@ func TestXLStorageReadFileWithVerify(t *testing.T) {
 
 // TestXLStorageFormatFileChange - to test if changing the diskID makes the calls fail.
 func TestXLStorageFormatFileChange(t *testing.T) {
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	if err = xlStorage.MakeVol(context.Background(), volume); err != nil {
 		t.Fatalf("MakeVol failed with %s", err)
@@ -1393,11 +1341,10 @@ func TestXLStorageFormatFileChange(t *testing.T) {
 // TestXLStorage xlStorage.AppendFile()
 func TestXLStorageAppendFile(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
@@ -1438,7 +1385,6 @@ func TestXLStorageAppendFile(t *testing.T) {
 	// TestXLStorage for permission denied.
 	if runtime.GOOS != globalWindowsOSName {
 		permDeniedDir := createPermDeniedFile(t)
-		defer removePermDeniedFile(permDeniedDir)
 
 		var xlStoragePermStorage StorageAPI
 		// Initialize xlStorage storage layer for permission denied error.
@@ -1472,11 +1418,10 @@ func TestXLStorageAppendFile(t *testing.T) {
 // TestXLStorage xlStorage.RenameFile()
 func TestXLStorageRenameFile(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err := xlStorage.MakeVol(context.Background(), "src-vol"); err != nil {
@@ -1690,11 +1635,10 @@ func TestXLStorageRenameFile(t *testing.T) {
 // TestXLStorageDeleteVersion will test if version deletes and bulk deletes work as expected.
 func TestXLStorageDeleteVersion(t *testing.T) {
 	// create xlStorage test setup
-	xl, path, err := newXLStorageTestSetup()
+	xl, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 	ctx := context.Background()
 
 	volume := "myvol-vol"
@@ -1782,11 +1726,10 @@ func TestXLStorageDeleteVersion(t *testing.T) {
 // TestXLStorage xlStorage.StatInfoFile()
 func TestXLStorageStatInfoFile(t *testing.T) {
 	// create xlStorage test setup
-	xlStorage, path, err := newXLStorageTestSetup()
+	xlStorage, _, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err := xlStorage.MakeVol(context.Background(), "success-vol"); err != nil {
@@ -1878,11 +1821,10 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	// 4) Streaming bitrot check on corrupted file
 
 	// create xlStorage test setup
-	storage, path, err := newXLStorageTestSetup()
+	storage, path, err := newXLStorageTestSetup(t)
 	if err != nil {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
-	defer os.RemoveAll(path)
 
 	volName := "testvol"
 	fileName := "testfile"
@@ -1976,11 +1918,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
 // TestXLStorageReadMetadata tests readMetadata
 func TestXLStorageReadMetadata(t *testing.T) {
 	volume, object := "test-vol", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
-	tmpDir, err := ioutil.TempDir("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
+	tmpDir := t.TempDir()
 
 	disk, err := newLocalXLStorage(tmpDir)
 	if err != nil {

cmd/xl-storage_unix_test.go

@@ -22,7 +22,6 @@ package cmd
 
 import (
 	"context"
-	"io/ioutil"
 	"os"
 	"path"
 	"syscall"
@@ -39,10 +38,7 @@ func getUmask() int {
 
 // Tests if the directory and file creations happen with proper umask.
 func TestIsValidUmaskVol(t *testing.T) {
-	tmpPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Initializing temporary directory failed with %s.", err)
-	}
+	tmpPath := t.TempDir()
 	testCases := []struct {
 		volName       string
 		expectedUmask int
@@ -62,7 +58,6 @@ func TestIsValidUmaskVol(t *testing.T) {
 	if err = disk.MakeVol(context.Background(), testCase.volName); err != nil {
 		t.Fatalf("Creating a volume failed with %s expected to pass.", err)
 	}
-	defer os.RemoveAll(tmpPath)
 
 	// Stat to get permissions bits.
 	st, err := os.Stat(path.Join(tmpPath, testCase.volName))
@@ -81,10 +76,7 @@ func TestIsValidUmaskVol(t *testing.T) {
 
 // Tests if the file creations happen with proper umask.
 func TestIsValidUmaskFile(t *testing.T) {
-	tmpPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Initializing temporary directory failed with %s.", err)
-	}
+	tmpPath := t.TempDir()
 	testCases := []struct {
 		volName       string
 		expectedUmask int
@@ -105,8 +97,6 @@ func TestIsValidUmaskFile(t *testing.T) {
 		t.Fatalf("Creating a volume failed with %s expected to pass.", err)
 	}
 
-	defer os.RemoveAll(tmpPath)
-
 	// Attempt to create a file to verify the permissions later.
 	// AppendFile creates file with 0666 perms.
 	if err = disk.AppendFile(context.Background(), testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil {