improve error message for bucket metadata export/import API (#15120)

site replication errors were printed at
various random locations, repeatedly - this
PR attempts to remove double logging and
capture all of them at a common place.

This PR also enhances the code to show
partial success and errors as well.

.gitignore

@@ -9,8 +9,7 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-vendor/**/*.js
-vendor/**/*.json
+vendor/
 .DS_Store
 *.syso
 coverage.txt
@@ -32,4 +31,10 @@ hash-set
 minio.RELEASE*
 mc
 nancy
-inspects/*
+inspects/*
+docs/debugging/s3-verify/s3-verify
+docs/debugging/xl-meta/xl-meta
+docs/debugging/s3-check-md5/s3-check-md5
+docs/debugging/hash-set/hash-set
+docs/debugging/healing-bin/healing-bin
+docs/debugging/inspect/inspect

README.md

@@ -236,16 +236,16 @@ Upgrades require zero downtime in MinIO, all upgrades are non-disruptive, all tr
 mc admin update <minio alias, e.g., myminio>
 ```
 
-- For deployments without external internet access (e.g. airgapped environments), download the binary from <https://dl.min.io> and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and do `mc admin service restart alias/`.
+- For deployments without external internet access (e.g. airgapped environments), download the binary from <https://dl.min.io> and replace the existing MinIO binary let's say for example `/opt/bin/minio`, apply executable permissions `chmod +x /opt/bin/minio` and proceed to perform `mc admin service restart alias/`.
 
-- For installations using Systemd MinIO service, upgrade via RPM/DEB packages **parallelly** on all servers or replace the binary lets say `/opt/bin/minio` on all nodes, apply executable permissions `chmod +x /opt/bin/minio`. Proceed to perform `systemctl restart minio` across all nodes in **parallel**.
+- For installations using Systemd MinIO service, upgrade via RPM/DEB packages **parallelly** on all servers or replace the binary lets say `/opt/bin/minio` on all nodes, apply executable permissions `chmod +x /opt/bin/minio` and process to perform `mc admin service restart alias/`.
 
 ### Upgrade Checklist
 
 - Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
-- Read the release notes for the targeted MinIO release *before* performing any installation, there is no forced requirement to upgrade to latest releases every week. If it has a bug fix you are looking for then yes, else avoid actively upgrading a running production system.
-- If you plan to use `mc admin update`, MinIO process must have write access to the parent directory to provide in-place upgrades.
-- `mc admin update` is not supported in kubernetes/container environments, container environments provide their own mechanisms for container updates.
+- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest releases upon every releases. Some releases may not be relevant to your setup, avoid upgrading production environments unnecessarily.
+- If you plan to use `mc admin update`, MinIO process must have write access to the parent directory where the binary is present on the host system.
+- `mc admin update` is not supported and should be avoided in kubernetes/container environments, please upgrade containers by upgrading relevant container images.
 - **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
 
 ## Explore Further

buildscripts/gen-ldflags.go

@@ -24,14 +24,18 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 )
 
 func genLDFlags(version string) string {
+	releaseTag, date := releaseTag(version)
+	copyrightYear := strconv.Itoa(date.Year())
 	ldflagsStr := "-s -w"
 	ldflagsStr += " -X github.com/minio/minio/cmd.Version=" + version
-	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag(version)
+	ldflagsStr += " -X github.com/minio/minio/cmd.CopyrightYear=" + copyrightYear
+	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
@@ -40,7 +44,7 @@ func genLDFlags(version string) string {
 }
 
 // genReleaseTag prints release tag to the console for easy git tagging.
-func releaseTag(version string) string {
+func releaseTag(version string) (string, time.Time) {
 	relPrefix := "DEVELOPMENT"
 	if prefix := os.Getenv("MINIO_RELEASE"); prefix != "" {
 		relPrefix = prefix
@@ -53,14 +57,17 @@ func releaseTag(version string) string {
 
 	relTag := strings.Replace(version, " ", "-", -1)
 	relTag = strings.Replace(relTag, ":", "-", -1)
+	t, err := time.Parse("2006-01-02T15-04-05Z", relTag)
+	if err != nil {
+		panic(err)
+	}
 	relTag = strings.Replace(relTag, ",", "", -1)
 	relTag = relPrefix + "." + relTag
-
 	if relSuffix != "" {
 		relTag += "." + relSuffix
 	}
 
-	return relTag
+	return relTag, t
 }
 
 // commitID returns the abbreviated commit-id hash of the last commit.

cmd/admin-bucket-handlers.go

@@ -18,15 +18,31 @@
 package cmd
 
 import (
+	"bytes"
+	"encoding/base64"
 	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/gorilla/mux"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/klauspost/compress/zip"
+	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
@@ -354,3 +370,688 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	// Write success response.
 	writeSuccessNoContent(w)
 }
+
+// ExportBucketMetadataHandler - exports all bucket metadata as a zipped file
+func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ExportBucketMetadata")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	bucket := pathClean(r.Form.Get("bucket"))
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	var (
+		buckets []BucketInfo
+		err     error
+	)
+	if bucket != "" {
+		// Check if bucket exists.
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		buckets = append(buckets, BucketInfo{Name: bucket})
+	} else {
+		buckets, err = objectAPI.ListBuckets(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	// Initialize a zip writer which will provide a zipped content
+	// of bucket metadata
+	zipWriter := zip.NewWriter(w)
+	defer zipWriter.Close()
+	rawDataFn := func(r io.Reader, filename string, sz int) error {
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    filename,
+			size:    int64(sz),
+			mode:    0o600,
+			modTime: time.Now(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		header.Method = zip.Deflate
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		if _, err := io.Copy(zwriter, r); err != nil {
+			logger.LogIf(ctx, err)
+		}
+		return nil
+	}
+
+	cfgFiles := []string{
+		bucketPolicyConfig,
+		bucketNotificationConfig,
+		bucketLifecycleConfig,
+		bucketSSEConfig,
+		bucketTaggingConfig,
+		bucketQuotaConfigFile,
+		objectLockConfig,
+		bucketVersioningConfig,
+		bucketReplicationConfig,
+		bucketTargetsFile,
+	}
+	for _, bi := range buckets {
+		for _, cfgFile := range cfgFiles {
+			cfgPath := pathJoin(bi.Name, cfgFile)
+			bucket := bi.Name
+			switch cfgFile {
+			case bucketNotificationConfig:
+				config, err := globalBucketMetadataSys.GetNotificationConfig(bucket)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketLifecycleConfig:
+				config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
+						continue
+					}
+					logger.LogIf(ctx, err)
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketQuotaConfigFile:
+				config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
+				if err != nil {
+					if errors.Is(err, BucketQuotaConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				configData, err := json.Marshal(config)
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketSSEConfig:
+				config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketSSEConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketTaggingConfig:
+				config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketTaggingNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case objectLockConfig:
+				config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketVersioningConfig:
+				config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				// ignore empty versioning configs
+				if config.Status != versioning.Enabled && config.Status != versioning.Suspended {
+					continue
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketReplicationConfig:
+				config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+				if err != nil {
+					if errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			case bucketTargetsFile:
+				config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
+				if err != nil {
+					if errors.Is(err, BucketRemoteTargetNotFound{Bucket: bucket}) {
+						continue
+					}
+
+					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+					return
+				}
+				configData, err := xml.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+			}
+		}
+	}
+}
+
+// ImportBucketMetadataHandler - imports all bucket metadata from a zipped file and overwrite bucket metadata config
+// There are some caveats regarding the following:
+// 1. object lock config - object lock should have been specified at time of bucket creation. Only default retention settings are imported here.
+// 2. Replication config - is omitted from import as remote target credentials are not available from exported data for security reasons.
+// 3. lifecycle config - if transition rules are present, tier name needs to have been defined.
+func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ImportBucketMetadata")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	bucket := pathClean(r.Form.Get("bucket"))
+
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+	// Get current object layer instance.
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+	reader := bytes.NewReader(data)
+	zr, err := zip.NewReader(reader, int64(len(data)))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+	bucketMap := make(map[string]struct{}, 1)
+
+	// import object lock config if any - order of import matters here.
+	for _, file := range zr.File {
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+			return
+		}
+		b, fileName := slc[0], slc[1]
+		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
+			bucket = b
+		}
+		switch fileName {
+		case objectLockConfig:
+			reader, err := file.Open()
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+			config, err := objectlock.ParseObjectLockConfig(reader)
+			if err != nil {
+				apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+				apiErr.Description = err.Error()
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+
+			configData, err := xml.Marshal(config)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+			if _, ok := bucketMap[bucket]; !ok {
+				opts := BucketOptions{
+					LockEnabled: config.ObjectLockEnabled == "Enabled",
+				}
+				err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
+				if err != nil {
+					if _, ok := err.(BucketExists); !ok {
+						writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+						return
+					}
+				}
+				bucketMap[bucket] = struct{}{}
+			}
+
+			// Deny object locking configuration settings on existing buckets without object lock enabled.
+			if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			if err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			// Call site replication hook.
+			//
+			// We encode the xml bytes as base64 to ensure there are no encoding
+			// errors.
+			cfgStr := base64.StdEncoding.EncodeToString(configData)
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:             madmin.SRBucketMetaTypeObjectLockConfig,
+				Bucket:           bucket,
+				ObjectLockConfig: &cfgStr,
+			}); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+		}
+	}
+
+	// import versioning metadata
+	for _, file := range zr.File {
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+			return
+		}
+		b, fileName := slc[0], slc[1]
+		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
+			bucket = b
+		}
+		switch fileName {
+		case bucketVersioningConfig:
+			reader, err := file.Open()
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+			v, err := versioning.ParseConfig(io.LimitReader(reader, maxBucketVersioningConfigSize))
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+			if _, ok := bucketMap[bucket]; !ok {
+				err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+				if err != nil {
+					if _, ok := err.(BucketExists); !ok {
+						writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+						return
+					}
+				}
+				bucketMap[bucket] = struct{}{}
+			}
+
+			if globalSiteReplicationSys.isEnabled() && v.Suspended() {
+				writeErrorResponse(ctx, w, APIError{
+					Code:           "InvalidBucketState",
+					Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
+					HTTPStatusCode: http.StatusConflict,
+				}, r.URL)
+				return
+			}
+
+			if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
+				writeErrorResponse(ctx, w, APIError{
+					Code:           "InvalidBucketState",
+					Description:    "An Object Lock configuration is present on this bucket, so the versioning state cannot be changed.",
+					HTTPStatusCode: http.StatusConflict,
+				}, r.URL)
+				return
+			}
+			if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
+				writeErrorResponse(ctx, w, APIError{
+					Code:           "InvalidBucketState",
+					Description:    "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
+					HTTPStatusCode: http.StatusConflict,
+				}, r.URL)
+				return
+			}
+
+			configData, err := xml.Marshal(v)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+		}
+	}
+
+	for _, file := range zr.File {
+		reader, err := file.Open()
+		if err != nil {
+			writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+			return
+		}
+		sz := file.FileInfo().Size()
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+			return
+		}
+		b, fileName := slc[0], slc[1]
+		if bucket == "" { // use bucket requested in query parameters if specified. Otherwise default bucket name to directory name within zip
+			bucket = b
+		}
+		// create bucket if it does not exist yet.
+		if _, ok := bucketMap[bucket]; !ok {
+			err = objectAPI.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			if err != nil {
+				if _, ok := err.(BucketExists); !ok {
+					writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+					return
+				}
+			}
+			bucketMap[bucket] = struct{}{}
+		}
+		switch fileName {
+		case bucketNotificationConfig:
+			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalNotificationSys.targetList)
+			if err != nil {
+				apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+				if event.IsEventError(err) {
+					apiErr = importError(ctx, err, file.Name, bucket)
+				}
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+
+			configData, err := xml.Marshal(config)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+			rulesMap := config.ToRulesMap()
+			globalNotificationSys.AddRulesMap(bucket, rulesMap)
+		case bucketPolicyConfig:
+			// Error out if Content-Length is beyond allowed size.
+			if sz > maxBucketPolicySize {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL)
+				return
+			}
+
+			bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(reader, sz))
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			// Version in policy must not be empty
+			if bucketPolicy.Version == "" {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL)
+				return
+			}
+
+			configData, err := json.Marshal(bucketPolicy)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+			// Call site replication hook.
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:   madmin.SRBucketMetaTypePolicy,
+				Bucket: bucket,
+				Policy: bucketPolicyBytes,
+			}); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+		case bucketLifecycleConfig:
+			bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			// Validate the received bucket policy document
+			if err = bucketLifecycle.Validate(); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			// Validate the transition storage ARNs
+			if err = validateTransitionTier(bucketLifecycle); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			configData, err := xml.Marshal(bucketLifecycle)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+		case bucketSSEConfig:
+			// Parse bucket encryption xml
+			encConfig, err := validateBucketSSEConfig(io.LimitReader(reader, maxBucketSSEConfigSize))
+			if err != nil {
+				apiErr := APIError{
+					Code:           "MalformedXML",
+					Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, err),
+					HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
+				}
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+
+			// Return error if KMS is not initialized
+			if GlobalKMS == nil {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
+				return
+			}
+			kmsKey := encConfig.KeyID()
+			if kmsKey != "" {
+				kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
+				_, err := GlobalKMS.GenerateKey(kmsKey, kmsContext)
+				if err != nil {
+					if errors.Is(err, kes.ErrKeyNotFound) {
+						writeErrorResponse(ctx, w, importError(ctx, errKMSKeyNotFound, file.Name, bucket), r.URL)
+						return
+					}
+					writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+					return
+				}
+			}
+
+			configData, err := xml.Marshal(encConfig)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			// Store the bucket encryption configuration in the object layer
+			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			// Call site replication hook.
+			//
+			// We encode the xml bytes as base64 to ensure there are no encoding
+			// errors.
+			cfgStr := base64.StdEncoding.EncodeToString(configData)
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:      madmin.SRBucketMetaTypeSSEConfig,
+				Bucket:    bucket,
+				SSEConfig: &cfgStr,
+			}); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+		case bucketTaggingConfig:
+			tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
+			if err != nil {
+				apiErr := errorCodes.ToAPIErrWithErr(ErrMalformedXML, fmt.Errorf("error importing %s with %w", file.Name, err))
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+
+			configData, err := xml.Marshal(tags)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+			// Call site replication hook.
+			//
+			// We encode the xml bytes as base64 to ensure there are no encoding
+			// errors.
+			cfgStr := base64.StdEncoding.EncodeToString(configData)
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+				Type:   madmin.SRBucketMetaTypeTags,
+				Bucket: bucket,
+				Tags:   &cfgStr,
+			}); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+		case bucketQuotaConfigFile:
+			data, err := ioutil.ReadAll(reader)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+				return
+			}
+
+			quotaConfig, err := parseBucketQuota(bucket, data)
+			if err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			if quotaConfig.Type == "fifo" {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+				return
+			}
+
+			if err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+
+			bucketMeta := madmin.SRBucketMeta{
+				Type:   madmin.SRBucketMetaTypeQuotaConfig,
+				Bucket: bucket,
+				Quota:  data,
+			}
+			if quotaConfig.Quota == 0 {
+				bucketMeta.Quota = nil
+			}
+
+			// Call site replication hook.
+			if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
+				writeErrorResponse(ctx, w, importError(ctx, err, file.Name, bucket), r.URL)
+				return
+			}
+		}
+	}
+}

cmd/admin-handler-utils.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 
 	"github.com/minio/kes"
@@ -96,6 +97,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 		}
 	default:
 		switch {
+		case errors.Is(err, errTooManyPolicies):
+			apiErr = APIError{
+				Code:           "XMinioAdminInvalidRequest",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, errDecommissionAlreadyRunning):
 			apiErr = APIError{
 				Code:           "XMinioDecommissionNotAllowed",
@@ -217,3 +224,19 @@ func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
 		return toAPIErrorCode(ctx, err)
 	}
 }
+
+// wraps export error for more context
+func exportError(ctx context.Context, err error, fname, entity string) APIError {
+	if entity == "" {
+		return toAPIError(ctx, fmt.Errorf("error exporting %s with: %w", fname, err))
+	}
+	return toAPIError(ctx, fmt.Errorf("error exporting %s from %s with: %w", entity, fname, err))
+}
+
+// wraps import error for more context
+func importError(ctx context.Context, err error, fname, entity string) APIError {
+	if entity == "" {
+		return toAPIError(ctx, fmt.Errorf("error importing %s with: %w", fname, err))
+	}
+	return toAPIError(ctx, fmt.Errorf("error importing %s from %s with: %w", entity, fname, err))
+}

cmd/admin-handlers.go

@@ -38,7 +38,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -275,6 +274,8 @@ type ServerConnStats struct {
 	Throughput       uint64 `json:"throughput,omitempty"`
 	S3InputBytes     uint64 `json:"transferredS3"`
 	S3OutputBytes    uint64 `json:"receivedS3"`
+	AdminInputBytes  uint64 `json:"transferredAdmin"`
+	AdminOutputBytes uint64 `json:"receivedAdmin"`
 }
 
 // ServerHTTPAPIStats holds total number of HTTP operations from/to the server,
@@ -291,6 +292,8 @@ type ServerHTTPStats struct {
 	CurrentS3Requests      ServerHTTPAPIStats `json:"currentS3Requests"`
 	TotalS3Requests        ServerHTTPAPIStats `json:"totalS3Requests"`
 	TotalS3Errors          ServerHTTPAPIStats `json:"totalS3Errors"`
+	TotalS35xxErrors       ServerHTTPAPIStats `json:"totalS35xxErrors"`
+	TotalS34xxErrors       ServerHTTPAPIStats `json:"totalS34xxErrors"`
 	TotalS3Canceled        ServerHTTPAPIStats `json:"totalS3Canceled"`
 	TotalS3RejectedAuth    uint64             `json:"totalS3RejectedAuth"`
 	TotalS3RejectedTime    uint64             `json:"totalS3RejectedTime"`
@@ -1108,7 +1111,6 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 	}
 
 	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
-
 	if !sufficientCapacity {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
 			Code:       "XMinioSpeedtestInsufficientCapacity",
@@ -1261,43 +1263,29 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 
-	enc := json.NewEncoder(w)
 	ch := globalNotificationSys.DriveSpeedTest(ctx, opts)
 
-	var wg sync.WaitGroup
-	wg.Add(1)
-
-	// local driveSpeedTest
-	go func() {
-		defer wg.Done()
-		enc.Encode(driveSpeedTest(ctx, opts))
-		if wf, ok := w.(http.Flusher); ok {
-			wf.Flush()
-		}
-	}()
-
+	enc := json.NewEncoder(w)
 	for {
 		select {
 		case <-ctx.Done():
-			goto endloop
+			return
 		case <-keepAliveTicker.C:
 			// Write a blank entry to prevent client from disconnecting
 			if err := enc.Encode(madmin.DriveSpeedTestResult{}); err != nil {
-				goto endloop
+				return
 			}
 			w.(http.Flusher).Flush()
 		case result, ok := <-ch:
 			if !ok {
-				goto endloop
+				return
 			}
 			if err := enc.Encode(result); err != nil {
-				goto endloop
+				return
 			}
 			w.(http.Flusher).Flush()
 		}
 	}
-endloop:
-	wg.Wait()
 }
 
 // Admin API errors
@@ -1872,6 +1860,12 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
 	defer healthCancel()
 
+	// Freeze all incoming S3 API calls before running speedtest.
+	globalNotificationSys.ServiceFreeze(ctx, true)
+
+	// unfreeze all incoming S3 API calls after speedtest.
+	defer globalNotificationSys.ServiceFreeze(ctx, false)
+
 	hostAnonymizer := createHostAnonymizer()
 	// anonAddr - Anonymizes hosts in given input string.
 	anonAddr := func(addr string) string {
@@ -2109,11 +2103,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	getAndWriteDrivePerfInfo := func() {
 		if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" {
-			// Freeze all incoming S3 API calls before running speedtest.
-			globalNotificationSys.ServiceFreeze(ctx, true)
-			// unfreeze all incoming S3 API calls after speedtest.
-			defer globalNotificationSys.ServiceFreeze(ctx, false)
-
 			opts := madmin.DriveSpeedTestOpts{
 				Serial:    false,
 				BlockSize: 4 * humanize.MiByte,
@@ -2134,6 +2123,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	getAndWriteObjPerfInfo := func() {
 		if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
 			concurrent := 32
+			if runtime.GOMAXPROCS(0) < concurrent {
+				concurrent = runtime.GOMAXPROCS(0)
+			}
+
 			size := 64 * humanize.MiByte
 			autotune := true
 
@@ -2160,12 +2153,6 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				defer deleteObjectPerfBucket(objectAPI)
 			}
 
-			// Freeze all incoming S3 API calls before running speedtest.
-			globalNotificationSys.ServiceFreeze(ctx, true)
-
-			// unfreeze all incoming S3 API calls after speedtest.
-			defer globalNotificationSys.ServiceFreeze(ctx, false)
-
 			opts := speedTestOpts{
 				throughputSize:   size,
 				concurrencyStart: concurrent,
@@ -2187,19 +2174,12 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				return
 			}
 
-			nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf")
-			lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
-			if err != nil {
-				healthInfo.Perf.Error = "Could not acquire lock for netperf: " + err.Error()
-			} else {
-				defer nsLock.Unlock(lkctx.Cancel)
-
-				netPerf := globalNotificationSys.Netperf(ctx, time.Second*10)
-				for _, np := range netPerf {
-					np.Endpoint = anonAddr(np.Endpoint)
-					healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np)
-				}
+			netPerf := globalNotificationSys.Netperf(ctx, time.Second*10)
+			for _, np := range netPerf {
+				np.Endpoint = anonAddr(np.Endpoint)
+				healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np)
 			}
+
 			partialWrite(healthInfo)
 		}
 	}

cmd/admin-router.go

@@ -188,6 +188,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 
+		// Bucket migration operations
+		// ExportBucketMetaHandler
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ExportBucketMetadataHandler)))
+		// ImportBucketMetaHandler
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-bucket-metadata").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.ImportBucketMetadataHandler)))
+
 		// Remote Tier management operations
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))

cmd/api-errors.go

@@ -2231,7 +2231,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 		case crypto.Error:
 			apiErr = APIError{
-				Code:           "XMinIOEncryptionError",
+				Code:           "XMinioEncryptionError",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}

cmd/auth-handler_test.go

@@ -32,6 +32,12 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
+type nullReader struct{}
+
+func (r *nullReader) Read(b []byte) (int, error) {
+	return len(b), nil
+}
+
 // Test get request auth type.
 func TestGetRequestAuthType(t *testing.T) {
 	type testCase struct {

cmd/bucket-handlers.go

@@ -806,19 +806,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
-	if _, ok := err.(BucketExists); ok {
-		// Though bucket exists locally, we send the site-replication
-		// hook to ensure all sites have this bucket. If the hook
-		// succeeds, the client will still receive a bucket exists
-		// message.
-		err2 := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
-		if err2 != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+	if err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
+		if _, ok := err.(BucketExists); ok {
+			// Though bucket exists locally, we send the site-replication
+			// hook to ensure all sites have this bucket. If the hook
+			// succeeds, the client will still receive a bucket exists
+			// message.
+			globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
 		}
-	}
-	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -827,8 +822,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Call site replication hook
-	err = globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
-	if err != nil {
+	if err := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -338,6 +338,9 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, ti
 func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) {
 	meta, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, time.Time{}, BucketQuotaConfigNotFound{Bucket: bucket}
+		}
 		return nil, time.Time{}, err
 	}
 	return meta.quotaConfig, meta.QuotaConfigUpdatedAt, nil

cmd/build-constants.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -37,4 +37,7 @@ var (
 
 	// ShortCommitID - first 12 characters from CommitID.
 	ShortCommitID = "DEVELOPMENT.GOGET"
+
+	// CopyrightYear - dynamic value of the copyright end year
+	CopyrightYear = "0000"
 )

cmd/common-main.go

@@ -526,9 +526,6 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
 	f, err := os.Open(envConfigFile)
 	if err != nil {
-		if os.IsNotExist(err) { // ignore if file doesn't exist.
-			return nil, nil
-		}
 		return nil, err
 	}
 	defer f.Close()
@@ -624,7 +621,7 @@ func loadEnvVarsFromFiles() {
 
 	if env.IsSet(config.EnvConfigEnvFile) {
 		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
-		if err != nil {
+		if err != nil && !os.IsNotExist(err) {
 			logger.Fatal(err, "Unable to read the config environment file")
 		}
 		for _, ekv := range ekvs {
@@ -787,17 +784,22 @@ func handleCommonEnvVars() {
 		}
 		globalActiveCred = cred
 	}
+}
 
+// Initialize KMS global variable after valiadating and loading the configuration.
+// It depends on KMS env variables and global cli flags.
+func handleKMSConfig() {
 	switch {
 	case env.IsSet(config.EnvKMSSecretKey) && env.IsSet(config.EnvKESEndpoint):
 		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKMSSecretKey, config.EnvKESEndpoint))
 	}
 
 	if env.IsSet(config.EnvKMSSecretKey) {
-		GlobalKMS, err = kms.Parse(env.Get(config.EnvKMSSecretKey, ""))
+		KMS, err := kms.Parse(env.Get(config.EnvKMSSecretKey, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
 		}
+		GlobalKMS = KMS
 	}
 	if env.IsSet(config.EnvKESEndpoint) {
 		var endpoints []string

cmd/config-current.go

@@ -338,7 +338,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			etcdClnt.Close()
 		}
 	case config.IdentityOpenIDSubSys:
-		if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys],
+		if _, err := openid.LookupConfig(s,
 			NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 			return err
 		}
@@ -560,7 +560,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
-	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys],
+	globalOpenIDConfig, err = openid.LookupConfig(s,
 		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))

cmd/erasure-healing.go

@@ -306,9 +306,13 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
 	if isAllNotFound(errs) {
+		err := errFileNotFound
+		if versionID != "" {
+			err = errFileVersionNotFound
+		}
 		// Nothing to do, file is already gone.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
-			errs, bucket, object, versionID), nil
+			errs, bucket, object, versionID), err
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
@@ -327,7 +331,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// present, it is as good as object not found.
 	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, readQuorum)
 	if err != nil {
-		return result, toObjectErr(err, bucket, object, versionID)
+		return result, err
 	}
 
 	// List of disks having all parts as per latest metadata.
@@ -397,8 +401,12 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	if isAllNotFound(errs) {
 		// File is fully gone, fileInfo is empty.
+		err := errFileNotFound
+		if versionID != "" {
+			err = errFileVersionNotFound
+		}
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs,
-			bucket, object, versionID), nil
+			bucket, object, versionID), err
 	}
 
 	// If less than read quorum number of disks have all the parts
@@ -420,7 +428,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	if !latestMeta.XLV1 && !latestMeta.Deleted && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// When disk to heal count is greater than parity blocks we should simply error out.
-		err := fmt.Errorf("more disks are expected to heal than parity, returned errors: %v -> %s/%s(%s)", errs, bucket, object, versionID)
+		err := fmt.Errorf("more disks are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", errs, dataErrs, bucket, object, versionID)
 		logger.LogIf(ctx, err)
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
@@ -485,7 +493,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		erasure, err := NewErasure(ctx, latestMeta.Erasure.DataBlocks,
 			latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
 		if err != nil {
-			return result, toObjectErr(err, bucket, object)
+			return result, err
 		}
 
 		erasureInfo := latestMeta.Erasure
@@ -523,7 +531,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
 			if err != nil {
-				return result, toObjectErr(err, bucket, object)
+				return result, err
 			}
 
 			// outDatedDisks that had write errors should not be
@@ -578,7 +586,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		// Attempt a rename now from healed data to final location.
 		if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
 			logger.LogIf(ctx, err)
-			return result, toObjectErr(err, bucket, object)
+			return result, err
 		}
 
 		// Remove any remaining parts from outdated disks from before transition.
@@ -674,10 +682,16 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 			hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateCorrupt}
 		}
 	}
-	if dryRun || danglingObject || isAllNotFound(errs) {
+	if danglingObject || isAllNotFound(errs) {
 		// Nothing to do, file is already gone.
+		return hr, errFileNotFound
+	}
+
+	if dryRun {
+		// Quit without try to heal the object dir
 		return hr, nil
 	}
+
 	for i, err := range errs {
 		if err == errVolumeNotFound || err == errFileNotFound {
 			// Bucket or prefix/directory not found
@@ -837,8 +851,13 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 		// Dangling object successfully purged, size is '0'
 		m.Size = 0
 	}
+	// Generate file/version not found with default heal result
+	err = errFileNotFound
+	if versionID != "" {
+		err = errFileVersionNotFound
+	}
 	return er.defaultHealResult(m, storageDisks, storageEndpoints,
-		errs, bucket, object, versionID), nil
+		errs, bucket, object, versionID), err
 }
 
 // Object is considered dangling/corrupted if any only
@@ -908,12 +927,6 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.
 func (er erasureObjects) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (hr madmin.HealResultItem, err error) {
-	defer func() {
-		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-			err = nil
-		}
-	}()
-
 	// Create context that also contains information about the object and bucket.
 	// The top level handler might not have this information.
 	reqInfo := logger.GetReqInfo(ctx)
@@ -927,7 +940,8 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 
 	// Healing directories handle it separately.
 	if HasSuffix(object, SlashSeparator) {
-		return er.healObjectDir(healCtx, bucket, object, opts.DryRun, opts.Remove)
+		hr, err := er.healObjectDir(healCtx, bucket, object, opts.DryRun, opts.Remove)
+		return hr, toObjectErr(err, bucket, object)
 	}
 
 	storageDisks := er.getDisks()
@@ -942,9 +956,13 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	// This allows to quickly check if all is ok or all are missing.
 	_, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
 	if isAllNotFound(errs) {
+		err := errFileNotFound
+		if versionID != "" {
+			err = errFileVersionNotFound
+		}
 		// Nothing to do, file is already gone.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
-			errs, bucket, object, versionID), nil
+			errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID)
 	}
 
 	// Heal the object.
@@ -955,5 +973,5 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 		opts.ScanMode = madmin.HealDeepScan
 		hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
 	}
-	return hr, err
+	return hr, toObjectErr(err, bucket, object, versionID)
 }

cmd/erasure-healing_test.go

@@ -684,16 +684,18 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
-	// Create an object with multiple parts uploaded in decreasing
-	// part number.
-	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	// Upload a multipart object in the second pool
+	z := objLayer.(*erasureServerPools)
+	set := z.serverPools[1]
+
+	uploadID, err := set.NewMultipartUpload(ctx, bucket, object, opts)
 	if err != nil {
 		t.Fatalf("Failed to create a multipart upload - %v", err)
 	}
 
 	var uploadedParts []CompletePart
 	for _, partID := range []int{2, 1} {
-		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		pInfo, err1 := set.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 		if err1 != nil {
 			t.Fatalf("Failed to upload a part - %v", err1)
 		}
@@ -703,120 +705,114 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		})
 	}
 
-	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	_, err = set.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
 	// Test 1: Remove the object backend files from the first disk.
-	z := objLayer.(*erasureServerPools)
-	for _, set := range z.serverPools {
-		er := set.sets[0]
-		erasureDisks := er.getDisks()
-		firstDisk := erasureDisks[0]
-		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-		if err != nil {
-			t.Fatalf("Failed to delete a file - %v", err)
-		}
+	er := set.sets[0]
+	erasureDisks := er.getDisks()
+	firstDisk := erasureDisks[0]
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	if err != nil {
+		t.Fatalf("Failed to delete a file - %v", err)
+	}
 
-		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
-		if err != nil {
-			t.Fatalf("Failed to heal object - %v", err)
-		}
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Fatalf("Failed to heal object - %v", err)
+	}
 
-		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-		fi, err := getLatestFileInfo(ctx, fileInfos, errs)
-		if errors.Is(err, errFileNotFound) {
-			continue
-		}
-		if err != nil {
-			t.Fatalf("Failed to getLatestFileInfo - %v", err)
-		}
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
 
-		if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-			t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
-		}
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	}
 
-		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
-		if err != nil {
-			t.Errorf("Failure during deleting part.1 - %v", err)
-		}
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	if err != nil {
+		t.Errorf("Failure during deleting part.1 - %v", err)
+	}
 
-		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
-		if err != nil {
-			t.Errorf("Failure during creating part.1 - %v", err)
-		}
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+	if err != nil {
+		t.Errorf("Failure during creating part.1 - %v", err)
+	}
 
-		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
-		if err != nil {
-			t.Errorf("Expected nil but received %v", err)
-		}
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err != nil {
+		t.Errorf("Expected nil but received %v", err)
+	}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
-		if err != nil {
-			t.Fatalf("Failed to getLatestFileInfo - %v", err)
-		}
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
 
-		fi.DiskMTime = time.Time{}
-		nfi.DiskMTime = time.Time{}
-		if !reflect.DeepEqual(fi, nfi) {
-			t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
-		}
+	fi.DiskMTime = time.Time{}
+	nfi.DiskMTime = time.Time{}
+	if !reflect.DeepEqual(fi, nfi) {
+		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
+	}
 
-		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
-		if err != nil {
-			t.Errorf("Failure during deleting part.1 - %v", err)
-		}
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	if err != nil {
+		t.Errorf("Failure during deleting part.1 - %v", err)
+	}
 
-		bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
-		if err != nil {
-			t.Errorf("Failure during creating part.1 - %v", err)
-		}
+	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+	if err != nil {
+		t.Errorf("Failure during creating part.1 - %v", err)
+	}
 
-		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
-		if err != nil {
-			t.Errorf("Expected nil but received %v", err)
-		}
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err != nil {
+		t.Errorf("Expected nil but received %v", err)
+	}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
-		if err != nil {
-			t.Fatalf("Failed to getLatestFileInfo - %v", err)
-		}
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
 
-		fi.DiskMTime = time.Time{}
-		nfi.DiskMTime = time.Time{}
-		if !reflect.DeepEqual(fi, nfi) {
-			t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
-		}
+	fi.DiskMTime = time.Time{}
+	nfi.DiskMTime = time.Time{}
+	if !reflect.DeepEqual(fi, nfi) {
+		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
+	}
 
-		// Test 4: checks if HealObject returns an error when xl.meta is not found
-		// in more than read quorum number of disks, to create a corrupted situation.
-		for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
-			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-		}
+	// Test 4: checks if HealObject returns an error when xl.meta is not found
+	// in more than read quorum number of disks, to create a corrupted situation.
+	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	}
 
-		// Try healing now, expect to receive errFileNotFound.
-		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
-		if err != nil {
-			if _, ok := err.(ObjectNotFound); !ok {
-				t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
-			}
-		}
-
-		// since majority of xl.meta's are not available, object should be successfully deleted.
-		_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	// Try healing now, expect to receive errFileNotFound.
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err != nil {
 		if _, ok := err.(ObjectNotFound); !ok {
 			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
 		}
+	}
 
-		for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
-			_, err = erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-			if err == nil {
-				t.Errorf("Expected xl.meta file to be not present, but succeeeded")
-			}
+	// since majority of xl.meta's are not available, object should be successfully deleted.
+	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if _, ok := err.(ObjectNotFound); !ok {
+		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	}
+
+	for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
+		_, err = erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		if err == nil {
+			t.Errorf("Expected xl.meta file to be not present, but succeeeded")
 		}
 	}
 }
@@ -1202,7 +1198,7 @@ func TestHealObjectErasure(t *testing.T) {
 	// since majority of xl.meta's are not available, object quorum
 	// can't be read properly will be deleted automatically and
 	// err is nil
-	if err != nil {
+	if !isErrObjectNotFound(err) {
 		t.Fatal(err)
 	}
 }

cmd/erasure-metadata-utils.go

@@ -143,6 +143,9 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 
 	errs := g.Wait()
 	for index, err := range errs {
+		if err == nil {
+			continue
+		}
 		if !IsErr(err, []error{
 			errFileNotFound,
 			errVolumeNotFound,

cmd/erasure-object.go

@@ -493,6 +493,9 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 
 	errs := g.Wait()
 	for index, err := range errs {
+		if err == nil {
+			continue
+		}
 		if !IsErr(err, []error{
 			errFileNotFound,
 			errVolumeNotFound,

cmd/erasure-server-pool.go

@@ -1939,7 +1939,8 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		}
 
 		for _, version := range fivs.Versions {
-			if err := healObjectFn(bucket, version.Name, version.VersionID); err != nil {
+			err := healObjectFn(bucket, version.Name, version.VersionID)
+			if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 				return err
 			}
 		}
@@ -2000,18 +2001,21 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 	}
 	wg.Wait()
 
-	for _, err := range errs {
-		if err != nil {
-			return madmin.HealResultItem{}, err
+	// Return the first nil error
+	for idx, err := range errs {
+		if err == nil {
+			return results[idx], nil
 		}
 	}
 
-	for _, result := range results {
-		if result.Object != "" {
-			return result, nil
+	// No pool returned a nil error, return the first non 'not found' error
+	for idx, err := range errs {
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return results[idx], err
 		}
 	}
 
+	// At this stage, all errors are 'not found'
 	if versionID != "" {
 		return madmin.HealResultItem{}, VersionNotFound{
 			Bucket:    bucket,
@@ -2082,11 +2086,14 @@ func (z *erasureServerPools) ReadHealth(ctx context.Context) bool {
 	}
 
 	b := z.BackendInfo()
-	readQuorum := b.StandardSCData[0]
+	poolReadQuorums := make([]int, len(b.StandardSCData))
+	for i, data := range b.StandardSCData {
+		poolReadQuorums[i] = data
+	}
 
 	for poolIdx := range erasureSetUpCount {
 		for setIdx := range erasureSetUpCount[poolIdx] {
-			if erasureSetUpCount[poolIdx][setIdx] < readQuorum {
+			if erasureSetUpCount[poolIdx][setIdx] < poolReadQuorums[poolIdx] {
 				return false
 			}
 		}
@@ -2123,9 +2130,12 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance))
 
 	b := z.BackendInfo()
-	writeQuorum := b.StandardSCData[0]
-	if writeQuorum == b.StandardSCParity {
-		writeQuorum++
+	poolWriteQuorums := make([]int, len(b.StandardSCData))
+	for i, data := range b.StandardSCData {
+		poolWriteQuorums[i] = data
+		if data == b.StandardSCParity {
+			poolWriteQuorums[i] = data + 1
+		}
 	}
 
 	var aggHealStateResult madmin.BgHealState
@@ -2149,34 +2159,44 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 
 	for poolIdx := range erasureSetUpCount {
 		for setIdx := range erasureSetUpCount[poolIdx] {
-			if erasureSetUpCount[poolIdx][setIdx] < writeQuorum {
+			if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
-						poolIdx, setIdx, writeQuorum))
+						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
 				return HealthResult{
 					Healthy:       false,
 					HealingDrives: len(aggHealStateResult.HealDisks),
 					PoolID:        poolIdx,
 					SetID:         setIdx,
-					WriteQuorum:   writeQuorum,
+					WriteQuorum:   poolWriteQuorums[poolIdx],
 				}
 			}
 		}
 	}
 
+	var maximumWriteQuorum int
+	for _, writeQuorum := range poolWriteQuorums {
+		if maximumWriteQuorum == 0 {
+			maximumWriteQuorum = writeQuorum
+		}
+		if writeQuorum > maximumWriteQuorum {
+			maximumWriteQuorum = writeQuorum
+		}
+	}
+
 	// when maintenance is not specified we don't have
 	// to look at the healing side of the code.
 	if !opts.Maintenance {
 		return HealthResult{
 			Healthy:     true,
-			WriteQuorum: writeQuorum,
+			WriteQuorum: maximumWriteQuorum,
 		}
 	}
 
 	return HealthResult{
 		Healthy:       len(aggHealStateResult.HealDisks) == 0,
 		HealingDrives: len(aggHealStateResult.HealDisks),
-		WriteQuorum:   writeQuorum,
+		WriteQuorum:   maximumWriteQuorum,
 	}
 }
 

cmd/erasure-single-drive.go

@@ -1203,7 +1203,14 @@ func (es *erasureSingle) DeleteObjects(ctx context.Context, bucket string, objec
 		// VersionID is not set means delete is not specific about
 		// any version, look for if the bucket is versioned or not.
 		if objects[i].VersionID == "" {
-			if opts.Versioned || opts.VersionSuspended {
+			// MinIO extension to bucket version configuration
+			suspended := opts.VersionSuspended
+			versioned := opts.Versioned
+			if opts.PrefixEnabledFn != nil {
+				versioned = opts.PrefixEnabledFn(objects[i].ObjectName)
+			}
+
+			if versioned || suspended {
 				// Bucket is versioned and no version was explicitly
 				// mentioned for deletes, create a delete marker instead.
 				vr.ModTime = UTCNow()
@@ -1211,7 +1218,7 @@ func (es *erasureSingle) DeleteObjects(ctx context.Context, bucket string, objec
 				// Versioning suspended means that we add a `null` version
 				// delete marker, if not add a new version for this delete
 				// marker.
-				if opts.Versioned {
+				if versioned {
 					vr.VersionID = mustGetUUID()
 				}
 			}
@@ -3289,3 +3296,39 @@ func (es *erasureSingle) NSScanner(ctx context.Context, bf *bloomFilter, updates
 	}
 	return firstErr
 }
+
+// GetRawData will return all files with a given raw path to the callback.
+// Errors are ignored, only errors from the callback are returned.
+// For now only direct file paths are supported.
+func (es *erasureSingle) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error {
+	found := 0
+	stats, err := es.disk.StatInfoFile(ctx, volume, file, true)
+	if err != nil {
+		return err
+	}
+	for _, si := range stats {
+		found++
+		var r io.ReadCloser
+		if !si.Dir {
+			r, err = es.disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size)
+			if err != nil {
+				continue
+			}
+		} else {
+			r = io.NopCloser(bytes.NewBuffer([]byte{}))
+		}
+		// Keep disk path instead of ID, to ensure that the downloaded zip file can be
+		// easily automated with `minio server hostname{1...n}/disk{1...m}`.
+		err = fn(r, es.disk.Hostname(), es.disk.Endpoint().Path, pathJoin(volume, si.Name), si)
+		r.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	if found == 0 {
+		return errFileNotFound
+	}
+
+	return nil
+}

cmd/gateway-main.go

@@ -212,6 +212,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Handle gateway specific env
 	gatewayHandleEnvVars()
 
+	// Initialize KMS configuration
+	handleKMSConfig()
+
 	// Set system resources to maximum.
 	setMaxResources()
 

cmd/generic-handlers.go

@@ -115,9 +115,11 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 
 // Reserved bucket.
 const (
-	minioReservedBucket     = "minio"
-	minioReservedBucketPath = SlashSeparator + minioReservedBucket
-	loginPathPrefix         = SlashSeparator + "login"
+	minioReservedBucket              = "minio"
+	minioReservedBucketPath          = SlashSeparator + minioReservedBucket
+	minioReservedBucketPathWithSlash = SlashSeparator + minioReservedBucket + SlashSeparator
+
+	loginPathPrefix = SlashSeparator + "login"
 )
 
 func guessIsBrowserReq(r *http.Request) bool {
@@ -269,6 +271,22 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
 	return time.Time{}, ErrMissingDateHeader
 }
 
+// splitStr splits a string into n parts, empty strings are added
+// if we are not able to reach n elements
+func splitStr(path, sep string, n int) []string {
+	splits := strings.SplitN(path, sep, n)
+	// Add empty strings if we found elements less than nr
+	for i := n - len(splits); i > 0; i-- {
+		splits = append(splits, "")
+	}
+	return splits
+}
+
+func url2Bucket(p string) (bucket string) {
+	tokens := splitStr(p, SlashSeparator, 3)
+	return tokens[1]
+}
+
 // setHttpStatsHandler sets a http Stats handler to gather HTTP statistics
 func setHTTPStatsHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -280,12 +298,28 @@ func setHTTPStatsHandler(h http.Handler) http.Handler {
 		r.Body = meteredRequest
 		h.ServeHTTP(meteredResponse, r)
 
-		if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
+		if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
+			strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
+			strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
 			globalConnStats.incInputBytes(meteredRequest.BytesRead())
 			globalConnStats.incOutputBytes(meteredResponse.BytesWritten())
-		} else {
-			globalConnStats.incS3InputBytes(meteredRequest.BytesRead())
-			globalConnStats.incS3OutputBytes(meteredResponse.BytesWritten())
+			return
+		}
+		if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
+			globalConnStats.incAdminInputBytes(meteredRequest.BytesRead())
+			globalConnStats.incAdminOutputBytes(meteredResponse.BytesWritten())
+			return
+		}
+
+		globalConnStats.incS3InputBytes(meteredRequest.BytesRead())
+		globalConnStats.incS3OutputBytes(meteredResponse.BytesWritten())
+
+		if r.URL != nil {
+			bucket := url2Bucket(r.URL.Path)
+			if bucket != "" && bucket != minioReservedBucket {
+				globalBucketConnStats.incS3InputBytes(bucket, meteredRequest.BytesRead())
+				globalBucketConnStats.incS3OutputBytes(bucket, meteredResponse.BytesWritten())
+			}
 		}
 	})
 }

cmd/globals.go

@@ -255,6 +255,9 @@ var (
 	// Global HTTP request statisitics
 	globalHTTPStats = newHTTPStats()
 
+	// Global bucket network statistics
+	globalBucketConnStats = newBucketConnStats()
+
 	// Time when the server is started
 	globalBootTime = UTCNow()
 

cmd/handler-api.go

@@ -247,7 +247,12 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 			if val := globalServiceFreeze.Load(); val != nil {
 				if unlock, ok := val.(chan struct{}); ok && unlock != nil {
 					// Wait until unfrozen.
-					<-unlock
+					select {
+					case <-unlock:
+					case <-r.Context().Done():
+						// if client canceled we don't need to wait here forever.
+						return
+					}
 				}
 			}
 		}

cmd/healthinfo.go

@@ -1,113 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"math"
-	"os"
-	"sync"
-
-	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/disk"
-)
-
-// round returns value rounding to specified decimal places.
-func round(f float64, n int) float64 {
-	if n <= 0 {
-		return math.Round(f)
-	}
-
-	p := math.Pow10(n)
-	return math.Round(f*p) / p
-}
-
-func getDrivePerfInfo(ctx context.Context, parallel bool) []madmin.DrivePerfInfo {
-	pools := globalEndpoints
-	info := []madmin.DrivePerfInfo{}
-	var wg sync.WaitGroup
-	for _, pool := range pools {
-		for _, endpoint := range pool.Endpoints {
-			if !endpoint.IsLocal {
-				continue
-			}
-
-			if _, err := os.Stat(endpoint.Path); err != nil {
-				info = append(info, madmin.DrivePerfInfo{
-					Path:  endpoint.Path,
-					Error: err.Error(),
-				})
-				continue
-			}
-
-			getHealthInfo := func(path string) {
-				defer wg.Done()
-
-				latency, throughput, err := disk.GetHealthInfo(
-					ctx, path, pathJoin(path, minioMetaTmpBucket, mustGetUUID()),
-				)
-				if err != nil {
-					info = append(info, madmin.DrivePerfInfo{
-						Path:  path,
-						Error: err.Error(),
-					})
-				} else {
-					info = append(info, madmin.DrivePerfInfo{
-						Path: path,
-						Latency: madmin.Latency{
-							Avg:          round(latency.Avg, 3),
-							Max:          round(latency.Max, 3),
-							Min:          round(latency.Min, 3),
-							Percentile50: round(latency.Percentile50, 3),
-							Percentile90: round(latency.Percentile90, 3),
-							Percentile99: round(latency.Percentile99, 3),
-						},
-						Throughput: madmin.Throughput{
-							Avg:          uint64(round(throughput.Avg, 0)),
-							Max:          uint64(round(throughput.Max, 0)),
-							Min:          uint64(round(throughput.Min, 0)),
-							Percentile50: uint64(round(throughput.Percentile50, 0)),
-							Percentile90: uint64(round(throughput.Percentile90, 0)),
-							Percentile99: uint64(round(throughput.Percentile99, 0)),
-						},
-					})
-				}
-			}
-
-			wg.Add(1)
-			if parallel {
-				go getHealthInfo(endpoint.Path)
-			} else {
-				getHealthInfo(endpoint.Path)
-			}
-		}
-	}
-
-	wg.Wait()
-	return info
-}
-
-func getDrivePerfInfos(ctx context.Context, addr string) madmin.DrivePerfInfos {
-	serialPerf := getDrivePerfInfo(ctx, false)
-	parallelPerf := getDrivePerfInfo(ctx, true)
-	return madmin.DrivePerfInfos{
-		NodeCommon:   madmin.NodeCommon{Addr: addr},
-		SerialPerf:   serialPerf,
-		ParallelPerf: parallelPerf,
-	}
-}

cmd/http-stats.go

@@ -35,19 +35,21 @@ type ConnStats struct {
 	totalOutputBytes uint64
 	s3InputBytes     uint64
 	s3OutputBytes    uint64
+	adminInputBytes  uint64
+	adminOutputBytes uint64
 }
 
-// Increase total input bytes
+// Increase internode total input bytes
 func (s *ConnStats) incInputBytes(n int64) {
 	atomic.AddUint64(&s.totalInputBytes, uint64(n))
 }
 
-// Increase total output bytes
+// Increase internode total output bytes
 func (s *ConnStats) incOutputBytes(n int64) {
 	atomic.AddUint64(&s.totalOutputBytes, uint64(n))
 }
 
-// Return total input bytes
+// Return internode total input bytes
 func (s *ConnStats) getTotalInputBytes() uint64 {
 	return atomic.LoadUint64(&s.totalInputBytes)
 }
@@ -57,33 +59,55 @@ func (s *ConnStats) getTotalOutputBytes() uint64 {
 	return atomic.LoadUint64(&s.totalOutputBytes)
 }
 
-// Increase outbound input bytes
+// Increase S3 total input bytes
 func (s *ConnStats) incS3InputBytes(n int64) {
 	atomic.AddUint64(&s.s3InputBytes, uint64(n))
 }
 
-// Increase outbound output bytes
+// Increase S3 total output bytes
 func (s *ConnStats) incS3OutputBytes(n int64) {
 	atomic.AddUint64(&s.s3OutputBytes, uint64(n))
 }
 
-// Return outbound input bytes
+// Return S3 total input bytes
 func (s *ConnStats) getS3InputBytes() uint64 {
 	return atomic.LoadUint64(&s.s3InputBytes)
 }
 
-// Return outbound output bytes
+// Return S3 total output bytes
 func (s *ConnStats) getS3OutputBytes() uint64 {
 	return atomic.LoadUint64(&s.s3OutputBytes)
 }
 
+// Increase Admin total input bytes
+func (s *ConnStats) incAdminInputBytes(n int64) {
+	atomic.AddUint64(&s.adminInputBytes, uint64(n))
+}
+
+// Increase Admin total output bytes
+func (s *ConnStats) incAdminOutputBytes(n int64) {
+	atomic.AddUint64(&s.adminOutputBytes, uint64(n))
+}
+
+// Return Admin total input bytes
+func (s *ConnStats) getAdminInputBytes() uint64 {
+	return atomic.LoadUint64(&s.adminInputBytes)
+}
+
+// Return Admin total output bytes
+func (s *ConnStats) getAdminOutputBytes() uint64 {
+	return atomic.LoadUint64(&s.adminOutputBytes)
+}
+
 // Return connection stats (total input/output bytes and total s3 input/output bytes)
 func (s *ConnStats) toServerConnStats() ServerConnStats {
 	return ServerConnStats{
-		TotalInputBytes:  s.getTotalInputBytes(),  // Traffic including reserved bucket
-		TotalOutputBytes: s.getTotalOutputBytes(), // Traffic including reserved bucket
-		S3InputBytes:     s.getS3InputBytes(),     // Traffic for client buckets
-		S3OutputBytes:    s.getS3OutputBytes(),    // Traffic for client buckets
+		TotalInputBytes:  s.getTotalInputBytes(),  // Traffic internode received
+		TotalOutputBytes: s.getTotalOutputBytes(), // Traffic internode sent
+		S3InputBytes:     s.getS3InputBytes(),     // Traffic S3 received
+		S3OutputBytes:    s.getS3OutputBytes(),    // Traffic S3 sent
+		AdminInputBytes:  s.getAdminInputBytes(),  // Traffic admin calls received
+		AdminOutputBytes: s.getAdminOutputBytes(), // Traffic admin calls sent
 	}
 }
 
@@ -92,6 +116,84 @@ func newConnStats() *ConnStats {
 	return &ConnStats{}
 }
 
+type bucketS3RXTX struct {
+	s3InputBytes  uint64
+	s3OutputBytes uint64
+}
+
+type bucketConnStats struct {
+	sync.RWMutex
+	stats map[string]*bucketS3RXTX
+}
+
+func newBucketConnStats() *bucketConnStats {
+	return &bucketConnStats{
+		stats: make(map[string]*bucketS3RXTX),
+	}
+}
+
+// Increase S3 total input bytes for input bucket
+func (s *bucketConnStats) incS3InputBytes(bucket string, n int64) {
+	s.Lock()
+	defer s.Unlock()
+	stats, ok := s.stats[bucket]
+	if !ok {
+		stats = &bucketS3RXTX{
+			s3InputBytes: uint64(n),
+		}
+	} else {
+		stats.s3InputBytes += uint64(n)
+	}
+	s.stats[bucket] = stats
+}
+
+// Increase S3 total output bytes for input bucket
+func (s *bucketConnStats) incS3OutputBytes(bucket string, n int64) {
+	s.Lock()
+	defer s.Unlock()
+	stats, ok := s.stats[bucket]
+	if !ok {
+		stats = &bucketS3RXTX{
+			s3OutputBytes: uint64(n),
+		}
+	} else {
+		stats.s3OutputBytes += uint64(n)
+	}
+	s.stats[bucket] = stats
+}
+
+// Return S3 total input bytes for input bucket
+func (s *bucketConnStats) getS3InputBytes(bucket string) uint64 {
+	s.RLock()
+	defer s.RUnlock()
+
+	stats := s.stats[bucket]
+	if stats == nil {
+		return 0
+	}
+	return stats.s3InputBytes
+}
+
+// Return S3 total output bytes
+func (s *bucketConnStats) getS3OutputBytes(bucket string) uint64 {
+	s.RLock()
+	defer s.RUnlock()
+
+	stats := s.stats[bucket]
+	if stats == nil {
+		return 0
+	}
+	return stats.s3OutputBytes
+}
+
+// delete metrics once bucket is deleted.
+func (s *bucketConnStats) delete(bucket string) {
+	s.Lock()
+	defer s.Unlock()
+
+	delete(s.stats, bucket)
+}
+
 // HTTPAPIStats holds statistics information about
 // a given API in the requests.
 type HTTPAPIStats struct {
@@ -148,6 +250,8 @@ type HTTPStats struct {
 	currentS3Requests       HTTPAPIStats
 	totalS3Requests         HTTPAPIStats
 	totalS3Errors           HTTPAPIStats
+	totalS34xxErrors        HTTPAPIStats
+	totalS35xxErrors        HTTPAPIStats
 	totalS3Canceled         HTTPAPIStats
 }
 
@@ -178,6 +282,12 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 	serverStats.TotalS3Errors = ServerHTTPAPIStats{
 		APIStats: st.totalS3Errors.Load(),
 	}
+	serverStats.TotalS34xxErrors = ServerHTTPAPIStats{
+		APIStats: st.totalS34xxErrors.Load(),
+	}
+	serverStats.TotalS35xxErrors = ServerHTTPAPIStats{
+		APIStats: st.totalS35xxErrors.Load(),
+	}
 	serverStats.TotalS3Canceled = ServerHTTPAPIStats{
 		APIStats: st.totalS3Canceled.Load(),
 	}
@@ -186,27 +296,31 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 
 // Update statistics from http request and response data
 func (st *HTTPStats) updateStats(api string, r *http.Request, w *logger.ResponseWriter) {
-	// A successful request has a 2xx response code or < 4xx response
-	successReq := w.StatusCode >= 200 && w.StatusCode < 400
-
-	if !strings.HasSuffix(r.URL.Path, prometheusMetricsPathLegacy) ||
-		!strings.HasSuffix(r.URL.Path, prometheusMetricsV2ClusterPath) ||
-		!strings.HasSuffix(r.URL.Path, prometheusMetricsV2NodePath) {
-		st.totalS3Requests.Inc(api)
-		if !successReq {
-			switch w.StatusCode {
-			case 0:
-			case 499:
-				// 499 is a good error, shall be counted as canceled.
-				st.totalS3Canceled.Inc(api)
-			default:
-				st.totalS3Errors.Inc(api)
-			}
-		}
+	// Ignore non S3 requests
+	if strings.HasSuffix(r.URL.Path, minioReservedBucketPathWithSlash) {
+		return
 	}
 
+	st.totalS3Requests.Inc(api)
+
 	// Increment the prometheus http request response histogram with appropriate label
 	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
+
+	code := w.StatusCode
+
+	switch {
+	case code == 0:
+	case code == 499:
+		// 499 is a good error, shall be counted as canceled.
+		st.totalS3Canceled.Inc(api)
+	case code >= http.StatusBadRequest:
+		st.totalS3Errors.Inc(api)
+		if code >= http.StatusInternalServerError {
+			st.totalS35xxErrors.Inc(api)
+		} else {
+			st.totalS34xxErrors.Inc(api)
+		}
+	}
 }
 
 // Prepare new HTTPStats structure

cmd/main.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
 	"sort"
 
 	"github.com/minio/cli"
+	"github.com/minio/minio/internal/color"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/trie"
 	"github.com/minio/pkg/words"
@@ -69,11 +71,6 @@ var GlobalFlags = []cli.Flag{
 	},
 }
 
-var versionFlag = cli.BoolFlag{
-	Name:  "version, v",
-	Usage: "print version information",
-}
-
 // Help template for minio.
 var minioHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}
@@ -137,20 +134,16 @@ func newApp(name string) *cli.App {
 		Name:  "help, h",
 		Usage: "show help",
 	}
-
-	topLevelFlags := make([]cli.Flag, len(GlobalFlags)+1)
-	copy(topLevelFlags, GlobalFlags)
-	topLevelFlags[len(GlobalFlags)] = versionFlag
+	cli.VersionPrinter = printMinIOVersion
 
 	app := cli.NewApp()
 	app.Name = name
 	app.Author = "MinIO, Inc."
-	app.Action = versionAndHelpAction
+	app.Version = ReleaseTag
 	app.Usage = "High Performance Object Storage"
 	app.Description = `Build high performance data infrastructure for machine learning, analytics and application data workloads with MinIO`
-	app.Flags = topLevelFlags
+	app.Flags = GlobalFlags
 	app.HideHelpCommand = true // Hide `help, h` command, we already have `minio --help`.
-	app.HideVersion = true
 	app.Commands = commands
 	app.CustomAppHelpTemplate = minioHelpTemplate
 	app.CommandNotFound = func(ctx *cli.Context, command string) {
@@ -170,16 +163,11 @@ func newApp(name string) *cli.App {
 	return app
 }
 
-func versionAndHelpAction(ctx *cli.Context) {
-	if ctx.IsSet("version") {
-		console.Printf("%s version %s\n", ctx.App.Name, ReleaseTag)
-		console.Printf("commit: %s\n", CommitID)
-		console.Printf("go version: %s\n", runtime.Version())
-
-		return
-	}
-
-	cli.ShowAppHelpAndExit(ctx, 1)
+func printMinIOVersion(c *cli.Context) {
+	fmt.Fprintln(c.App.Writer, color.Greenf("%s version %s (commit-id=%s)", c.App.Name, c.App.Version, CommitID))
+	fmt.Fprintln(c.App.Writer, color.Greenf("Runtime: %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH))
+	fmt.Fprintln(c.App.Writer, color.Greenf("Copyright (c) 2015-%s MinIO, Inc.", CopyrightYear))
+	fmt.Fprintln(c.App.Writer, color.Red("Licence AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"))
 }
 
 // Main main for minio server.

cmd/metacache-walk.go

@@ -182,7 +182,12 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Unlock()
 				diskHealthCheckOK(ctx, err)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					// It is totally possible that xl.meta was overwritten
+					// while being concurrently listed at the same time in
+					// such scenarios the 'xl.meta' might get truncated
+					if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
+						logger.LogIf(ctx, err)
+					}
 					continue
 				}
 				meta.name = strings.TrimSuffix(entry, xlStorageFormatFile)
@@ -200,7 +205,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Unlock()
 				diskHealthCheckOK(ctx, err)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
+						logger.LogIf(ctx, err)
+					}
 					continue
 				}
 				meta.name = strings.TrimSuffix(entry, xlStorageFormatFileV1)
@@ -245,7 +252,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 					if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, pop) {
 						forward = strings.TrimPrefix(opts.ForwardTo, pop)
 					}
-					logger.LogIf(ctx, scanDir(pop))
+
+					err := scanDir(pop)
+					if err != nil && !IsErrIgnored(err, context.Canceled) {
+						logger.LogIf(ctx, err)
+					}
 				}
 				dirStack = dirStack[:len(dirStack)-1]
 			}

cmd/metrics-v2.go

@@ -293,7 +293,7 @@ func getClusterCapacityTotalBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityRawSubsystem,
 		Name:      totalBytes,
-		Help:      "Total capacity online in the cluster.",
+		Help:      "Total capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -303,7 +303,7 @@ func getClusterCapacityFreeBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityRawSubsystem,
 		Name:      freeBytes,
-		Help:      "Total free capacity online in the cluster.",
+		Help:      "Total free capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -313,7 +313,7 @@ func getClusterCapacityUsageBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityUsableSubsystem,
 		Name:      totalBytes,
-		Help:      "Total usable capacity online in the cluster.",
+		Help:      "Total usable capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -323,7 +323,7 @@ func getClusterCapacityUsageFreeBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityUsableSubsystem,
 		Name:      freeBytes,
-		Help:      "Total free usable capacity online in the cluster.",
+		Help:      "Total free usable capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -333,7 +333,7 @@ func getNodeDiskAPILatencyMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      apiLatencyMicroSec,
-		Help:      "Average last minute latency in µs for disk API storage operations.",
+		Help:      "Average last minute latency in µs for disk API storage operations",
 		Type:      gaugeMetric,
 	}
 }
@@ -343,7 +343,7 @@ func getNodeDiskUsedBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      usedBytes,
-		Help:      "Total storage used on a disk.",
+		Help:      "Total storage used on a disk",
 		Type:      gaugeMetric,
 	}
 }
@@ -353,7 +353,7 @@ func getNodeDiskFreeBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      freeBytes,
-		Help:      "Total storage available on a disk.",
+		Help:      "Total storage available on a disk",
 		Type:      gaugeMetric,
 	}
 }
@@ -363,7 +363,7 @@ func getClusterDisksOfflineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      offlineTotal,
-		Help:      "Total disks offline.",
+		Help:      "Total disks offline",
 		Type:      gaugeMetric,
 	}
 }
@@ -373,7 +373,7 @@ func getClusterDisksOnlineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      onlineTotal,
-		Help:      "Total disks online.",
+		Help:      "Total disks online",
 		Type:      gaugeMetric,
 	}
 }
@@ -383,7 +383,7 @@ func getClusterDisksTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      total,
-		Help:      "Total disks.",
+		Help:      "Total disks",
 		Type:      gaugeMetric,
 	}
 }
@@ -393,7 +393,7 @@ func getClusterDisksFreeInodes() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      freeInodes,
-		Help:      "Total free inodes.",
+		Help:      "Total free inodes",
 		Type:      gaugeMetric,
 	}
 }
@@ -403,7 +403,7 @@ func getNodeDiskTotalBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      totalBytes,
-		Help:      "Total storage on a disk.",
+		Help:      "Total storage on a disk",
 		Type:      gaugeMetric,
 	}
 }
@@ -428,6 +428,26 @@ func getBucketUsageQuotaTotalBytesMD() MetricDescription {
 	}
 }
 
+func getBucketTrafficReceivedBytes() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: trafficSubsystem,
+		Name:      receivedBytes,
+		Help:      "Total number of S3 bytes received for this bucket",
+		Type:      gaugeMetric,
+	}
+}
+
+func getBucketTrafficSentBytes() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: trafficSubsystem,
+		Name:      sentBytes,
+		Help:      "Total number of S3 bytes sent for this bucket",
+		Type:      gaugeMetric,
+	}
+}
+
 func getBucketUsageTotalBytesMD() MetricDescription {
 	return MetricDescription{
 		Namespace: bucketMetricNamespace,
@@ -453,7 +473,7 @@ func getBucketRepLatencyMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      latencyMilliSec,
-		Help:      "Replication latency in milliseconds.",
+		Help:      "Replication latency in milliseconds",
 		Type:      histogramMetric,
 	}
 }
@@ -463,7 +483,7 @@ func getBucketRepFailedBytesMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      failedBytes,
-		Help:      "Total number of bytes failed at least once to replicate.",
+		Help:      "Total number of bytes failed at least once to replicate",
 		Type:      gaugeMetric,
 	}
 }
@@ -473,7 +493,7 @@ func getBucketRepSentBytesMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      sentBytes,
-		Help:      "Total number of bytes replicated to the target bucket.",
+		Help:      "Total number of bytes replicated to the target bucket",
 		Type:      gaugeMetric,
 	}
 }
@@ -483,7 +503,7 @@ func getBucketRepReceivedBytesMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      receivedBytes,
-		Help:      "Total number of bytes replicated to this bucket from another source bucket.",
+		Help:      "Total number of bytes replicated to this bucket from another source bucket",
 		Type:      gaugeMetric,
 	}
 }
@@ -503,7 +523,7 @@ func getBucketObjectDistributionMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: objectsSubsystem,
 		Name:      sizeDistribution,
-		Help:      "Distribution of object sizes in the bucket, includes label for the bucket name.",
+		Help:      "Distribution of object sizes in the bucket, includes label for the bucket name",
 		Type:      histogramMetric,
 	}
 }
@@ -513,7 +533,7 @@ func getInternodeFailedRequests() MetricDescription {
 		Namespace: interNodeMetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      errorsTotal,
-		Help:      "Total number of failed internode calls.",
+		Help:      "Total number of failed internode calls",
 		Type:      counterMetric,
 	}
 }
@@ -523,7 +543,7 @@ func getInterNodeSentBytesMD() MetricDescription {
 		Namespace: interNodeMetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      sentBytes,
-		Help:      "Total number of bytes sent to the other peer nodes.",
+		Help:      "Total number of bytes sent to the other peer nodes",
 		Type:      counterMetric,
 	}
 }
@@ -533,7 +553,7 @@ func getInterNodeReceivedBytesMD() MetricDescription {
 		Namespace: interNodeMetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      receivedBytes,
-		Help:      "Total number of bytes received from other peer nodes.",
+		Help:      "Total number of bytes received from other peer nodes",
 		Type:      counterMetric,
 	}
 }
@@ -553,7 +573,7 @@ func getS3ReceivedBytesMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      receivedBytes,
-		Help:      "Total number of s3 bytes received.",
+		Help:      "Total number of s3 bytes received",
 		Type:      counterMetric,
 	}
 }
@@ -603,7 +623,27 @@ func getS3RequestsErrorsMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      errorsTotal,
-		Help:      "Total number S3 requests with errors",
+		Help:      "Total number S3 requests with (4xx and 5xx) errors",
+		Type:      counterMetric,
+	}
+}
+
+func getS3Requests4xxErrorsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: s3MetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      "4xx_" + errorsTotal,
+		Help:      "Total number S3 requests with (4xx) errors",
+		Type:      counterMetric,
+	}
+}
+
+func getS3Requests5xxErrorsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: s3MetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      "5xx_" + errorsTotal,
+		Help:      "Total number S3 requests with (5xx) errors",
 		Type:      counterMetric,
 	}
 }
@@ -623,7 +663,7 @@ func getS3RejectedAuthRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      authTotal,
-		Help:      "Total number S3 requests rejected for auth failure.",
+		Help:      "Total number S3 requests rejected for auth failure",
 		Type:      counterMetric,
 	}
 }
@@ -633,7 +673,7 @@ func getS3RejectedHeaderRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      headerTotal,
-		Help:      "Total number S3 requests rejected for invalid header.",
+		Help:      "Total number S3 requests rejected for invalid header",
 		Type:      counterMetric,
 	}
 }
@@ -643,7 +683,7 @@ func getS3RejectedTimestampRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      timestampTotal,
-		Help:      "Total number S3 requests rejected for invalid timestamp.",
+		Help:      "Total number S3 requests rejected for invalid timestamp",
 		Type:      counterMetric,
 	}
 }
@@ -653,7 +693,7 @@ func getS3RejectedInvalidRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      invalidTotal,
-		Help:      "Total number S3 invalid requests.",
+		Help:      "Total number S3 invalid requests",
 		Type:      counterMetric,
 	}
 }
@@ -773,7 +813,7 @@ func getNodeOnlineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: nodesSubsystem,
 		Name:      onlineTotal,
-		Help:      "Total number of MinIO nodes online.",
+		Help:      "Total number of MinIO nodes online",
 		Type:      gaugeMetric,
 	}
 }
@@ -783,7 +823,7 @@ func getNodeOfflineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: nodesSubsystem,
 		Name:      offlineTotal,
-		Help:      "Total number of MinIO nodes offline.",
+		Help:      "Total number of MinIO nodes offline",
 		Type:      gaugeMetric,
 	}
 }
@@ -803,7 +843,7 @@ func getMinIOCommitMD() MetricDescription {
 		Namespace: minioMetricNamespace,
 		Subsystem: softwareSubsystem,
 		Name:      commitInfo,
-		Help:      "Git commit hash for the MinIO release.",
+		Help:      "Git commit hash for the MinIO release",
 		Type:      gaugeMetric,
 	}
 }
@@ -813,7 +853,7 @@ func getS3TTFBDistributionMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: timeSubsystem,
 		Name:      ttfbDistribution,
-		Help:      "Distribution of the time to first byte across API calls.",
+		Help:      "Distribution of the time to first byte across API calls",
 		Type:      gaugeMetric,
 	}
 }
@@ -823,7 +863,7 @@ func getMinioFDOpenMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: fileDescriptorSubsystem,
 		Name:      openTotal,
-		Help:      "Total number of open file descriptors by the MinIO Server process.",
+		Help:      "Total number of open file descriptors by the MinIO Server process",
 		Type:      gaugeMetric,
 	}
 }
@@ -833,7 +873,7 @@ func getMinioFDLimitMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: fileDescriptorSubsystem,
 		Name:      limitTotal,
-		Help:      "Limit on total number of open file descriptors for the MinIO Server process.",
+		Help:      "Limit on total number of open file descriptors for the MinIO Server process",
 		Type:      gaugeMetric,
 	}
 }
@@ -903,7 +943,7 @@ func getMinIOGORoutineCountMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: goRoutines,
 		Name:      total,
-		Help:      "Total number of go routines running.",
+		Help:      "Total number of go routines running",
 		Type:      gaugeMetric,
 	}
 }
@@ -913,7 +953,7 @@ func getMinIOProcessStartTimeMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      startTime,
-		Help:      "Start time for MinIO process per node, time in seconds since Unix epoc.",
+		Help:      "Start time for MinIO process per node, time in seconds since Unix epoc",
 		Type:      gaugeMetric,
 	}
 }
@@ -923,7 +963,7 @@ func getMinIOProcessUptimeMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      upTime,
-		Help:      "Uptime for MinIO process per node in seconds.",
+		Help:      "Uptime for MinIO process per node in seconds",
 		Type:      gaugeMetric,
 	}
 }
@@ -933,7 +973,7 @@ func getMinIOProcessResidentMemory() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      memory,
-		Help:      "Resident memory size in bytes.",
+		Help:      "Resident memory size in bytes",
 		Type:      gaugeMetric,
 	}
 }
@@ -943,7 +983,7 @@ func getMinIOProcessCPUTime() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      cpu,
-		Help:      "Total user and system CPU time spent in seconds.",
+		Help:      "Total user and system CPU time spent in seconds",
 		Type:      counterMetric,
 	}
 }
@@ -1128,7 +1168,7 @@ func getTransitionPendingTasksMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: ilmSubsystem,
 		Name:      transitionPendingTasks,
-		Help:      "Number of pending ILM transition tasks in the queue.",
+		Help:      "Number of pending ILM transition tasks in the queue",
 		Type:      gaugeMetric,
 	}
 }
@@ -1138,7 +1178,7 @@ func getTransitionActiveTasksMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: ilmSubsystem,
 		Name:      transitionActiveTasks,
-		Help:      "Number of active ILM transition tasks.",
+		Help:      "Number of active ILM transition tasks",
 		Type:      gaugeMetric,
 	}
 }
@@ -1148,7 +1188,7 @@ func getExpiryPendingTasksMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: ilmSubsystem,
 		Name:      expiryPendingTasks,
-		Help:      "Number of pending ILM expiry tasks in the queue.",
+		Help:      "Number of pending ILM expiry tasks in the queue",
 		Type:      gaugeMetric,
 	}
 }
@@ -1190,7 +1230,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Namespace: nodeMetricNamespace,
 					Subsystem: scannerSubsystem,
 					Name:      "objects_scanned",
-					Help:      "Total number of unique objects scanned since server start.",
+					Help:      "Total number of unique objects scanned since server start",
 					Type:      counterMetric,
 				},
 				Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalObjects)),
@@ -1200,7 +1240,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Namespace: nodeMetricNamespace,
 					Subsystem: scannerSubsystem,
 					Name:      "versions_scanned",
-					Help:      "Total number of object versions scanned since server start.",
+					Help:      "Total number of object versions scanned since server start",
 					Type:      counterMetric,
 				},
 				Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalVersions)),
@@ -1210,7 +1250,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Namespace: nodeMetricNamespace,
 					Subsystem: scannerSubsystem,
 					Name:      "directories_scanned",
-					Help:      "Total number of directories scanned since server start.",
+					Help:      "Total number of directories scanned since server start",
 					Type:      counterMetric,
 				},
 				Value: float64(atomic.LoadUint64(&globalScannerStats.accFolders)),
@@ -1347,7 +1387,7 @@ func getNodeHealthMetrics() *MetricsGroup {
 			return
 		}
 		metrics = make([]Metric, 0, 16)
-		nodesUp, nodesDown := GetPeerOnlineCount()
+		nodesUp, nodesDown := globalNotificationSys.GetPeerOnlineCount()
 		metrics = append(metrics, Metric{
 			Description: getNodeOnlineTotalMD(),
 			Value:       float64(nodesUp),
@@ -1488,7 +1528,9 @@ func getHTTPMetrics() *MetricsGroup {
 		metrics = make([]Metric, 0, 3+
 			len(httpStats.CurrentS3Requests.APIStats)+
 			len(httpStats.TotalS3Requests.APIStats)+
-			len(httpStats.TotalS3Errors.APIStats))
+			len(httpStats.TotalS3Errors.APIStats)+
+			len(httpStats.TotalS35xxErrors.APIStats)+
+			len(httpStats.TotalS34xxErrors.APIStats))
 		metrics = append(metrics, Metric{
 			Description: getS3RejectedAuthRequestsTotalMD(),
 			Value:       float64(httpStats.TotalS3RejectedAuth),
@@ -1535,6 +1577,20 @@ func getHTTPMetrics() *MetricsGroup {
 				VariableLabels: map[string]string{"api": api},
 			})
 		}
+		for api, value := range httpStats.TotalS35xxErrors.APIStats {
+			metrics = append(metrics, Metric{
+				Description:    getS3Requests5xxErrorsMD(),
+				Value:          float64(value),
+				VariableLabels: map[string]string{"api": api},
+			})
+		}
+		for api, value := range httpStats.TotalS34xxErrors.APIStats {
+			metrics = append(metrics, Metric{
+				Description:    getS3Requests4xxErrorsMD(),
+				Value:          float64(value),
+				VariableLabels: map[string]string{"api": api},
+			})
+		}
 		for api, value := range httpStats.TotalS3Canceled.APIStats {
 			metrics = append(metrics, Metric{
 				Description:    getS3RequestsCanceledMD(),
@@ -1551,19 +1607,21 @@ func getNetworkMetrics() *MetricsGroup {
 	mg := &MetricsGroup{}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		metrics = make([]Metric, 0, 10)
-		metrics = append(metrics, Metric{
-			Description: getInternodeFailedRequests(),
-			Value:       float64(loadAndResetRPCNetworkErrsCounter()),
-		})
 		connStats := globalConnStats.toServerConnStats()
-		metrics = append(metrics, Metric{
-			Description: getInterNodeSentBytesMD(),
-			Value:       float64(connStats.TotalOutputBytes),
-		})
-		metrics = append(metrics, Metric{
-			Description: getInterNodeReceivedBytesMD(),
-			Value:       float64(connStats.TotalInputBytes),
-		})
+		if globalIsDistErasure {
+			metrics = append(metrics, Metric{
+				Description: getInternodeFailedRequests(),
+				Value:       float64(loadAndResetRPCNetworkErrsCounter()),
+			})
+			metrics = append(metrics, Metric{
+				Description: getInterNodeSentBytesMD(),
+				Value:       float64(connStats.TotalOutputBytes),
+			})
+			metrics = append(metrics, Metric{
+				Description: getInterNodeReceivedBytesMD(),
+				Value:       float64(connStats.TotalInputBytes),
+			})
+		}
 		metrics = append(metrics, Metric{
 			Description: getS3SentBytesMD(),
 			Value:       float64(connStats.S3OutputBytes),
@@ -1636,6 +1694,24 @@ func getBucketUsageMetrics() *MetricsGroup {
 				})
 			}
 
+			recvBytes := globalBucketConnStats.getS3InputBytes(bucket)
+			if recvBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficReceivedBytes(),
+					Value:          float64(recvBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+
+			sentBytes := globalBucketConnStats.getS3OutputBytes(bucket)
+			if sentBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficSentBytes(),
+					Value:          float64(sentBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+
 			if stats.hasReplicationUsage() {
 				for arn, stat := range stats.Stats {
 					metrics = append(metrics, Metric{
@@ -1932,11 +2008,9 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 	}
 
 	// Call peer api to fetch metrics
-	peerCh := globalNotificationSys.GetClusterMetrics(GlobalContext)
-	selfCh := ReportMetrics(GlobalContext, c.metricsGroups)
 	wg.Add(2)
-	go publish(peerCh)
-	go publish(selfCh)
+	go publish(ReportMetrics(GlobalContext, c.metricsGroups))
+	go publish(globalNotificationSys.GetClusterMetrics(GlobalContext))
 	wg.Wait()
 }
 

cmd/metrics.go

@@ -110,7 +110,7 @@ func nodeHealthMetricsPrometheus(ch chan<- prometheus.Metric) {
 		return
 	}
 
-	nodesUp, nodesDown := GetPeerOnlineCount()
+	nodesUp, nodesDown := globalNotificationSys.GetPeerOnlineCount()
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
 			prometheus.BuildFQName(minioNamespace, "nodes", "online"),

cmd/notification.go

@@ -26,7 +26,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -35,7 +34,6 @@ import (
 	"github.com/cespare/xxhash/v2"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio-go/v7/pkg/set"
 	bucketBandwidth "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
@@ -579,6 +577,7 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName
 	globalBucketMetadataSys.Remove(bucketName)
 	globalBucketTargetSys.Delete(bucketName)
 	globalNotificationSys.RemoveNotification(bucketName)
+	globalBucketConnStats.delete(bucketName)
 	if localMetacacheMgr != nil {
 		localMetacacheMgr.deleteBucketCache(bucketName)
 	}
@@ -855,199 +854,6 @@ func (sys *NotificationSys) Send(args eventArgs) {
 	sys.targetList.Send(args.ToEvent(true), targetIDSet, sys.targetResCh)
 }
 
-// GetNetPerfInfo - Net information
-func (sys *NotificationSys) GetNetPerfInfo(ctx context.Context) madmin.NetPerfInfo {
-	var sortedGlobalEndpoints []string
-
-	/*
-			Ensure that only untraversed links are visited by this server
-		        i.e. if net perf tests have been performed between a -> b, then do
-			not run it between b -> a
-
-		        The graph of tests looks like this
-
-		            a   b   c   d
-		        a | o | x | x | x |
-		        b | o | o | x | x |
-		        c | o | o | o | x |
-		        d | o | o | o | o |
-
-		        'x's should be tested, and 'o's should be skipped
-	*/
-
-	hostSet := set.NewStringSet()
-	for _, ez := range globalEndpoints {
-		for _, e := range ez.Endpoints {
-			if !hostSet.Contains(e.Host) {
-				sortedGlobalEndpoints = append(sortedGlobalEndpoints, e.Host)
-				hostSet.Add(e.Host)
-			}
-		}
-	}
-
-	sort.Strings(sortedGlobalEndpoints)
-	var remoteTargets []*peerRESTClient
-	search := func(host string) *peerRESTClient {
-		for index, client := range sys.peerClients {
-			if client == nil {
-				continue
-			}
-			if sys.peerClients[index].host.String() == host {
-				return client
-			}
-		}
-		return nil
-	}
-
-	for i := 0; i < len(sortedGlobalEndpoints); i++ {
-		if sortedGlobalEndpoints[i] != globalLocalNodeName {
-			continue
-		}
-		for j := 0; j < len(sortedGlobalEndpoints); j++ {
-			if j > i {
-				remoteTarget := search(sortedGlobalEndpoints[j])
-				if remoteTarget != nil {
-					remoteTargets = append(remoteTargets, remoteTarget)
-				}
-			}
-		}
-	}
-
-	netInfos := make([]madmin.PeerNetPerfInfo, len(remoteTargets))
-
-	for index, client := range remoteTargets {
-		if client == nil {
-			continue
-		}
-		var err error
-		netInfos[index], err = client.GetNetPerfInfo(ctx)
-
-		addr := client.host.String()
-		reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogIf(ctx, err)
-		netInfos[index].Addr = addr
-		if err != nil {
-			netInfos[index].Error = err.Error()
-		}
-	}
-	return madmin.NetPerfInfo{
-		NodeCommon:  madmin.NodeCommon{Addr: globalLocalNodeName},
-		RemotePeers: netInfos,
-	}
-}
-
-// DispatchNetPerfInfo - Net perf information from other nodes
-func (sys *NotificationSys) DispatchNetPerfInfo(ctx context.Context) []madmin.NetPerfInfo {
-	serverNetInfos := []madmin.NetPerfInfo{}
-
-	for index, client := range sys.peerClients {
-		if client == nil {
-			continue
-		}
-		serverNetInfo, err := sys.peerClients[index].DispatchNetInfo(ctx)
-		if err != nil {
-			serverNetInfo.Addr = client.host.String()
-			serverNetInfo.Error = err.Error()
-		}
-		serverNetInfos = append(serverNetInfos, serverNetInfo)
-	}
-	return serverNetInfos
-}
-
-// DispatchNetPerfChan - Net perf information from other nodes
-func (sys *NotificationSys) DispatchNetPerfChan(ctx context.Context) chan madmin.NetPerfInfo {
-	serverNetInfos := make(chan madmin.NetPerfInfo)
-	wg := sync.WaitGroup{}
-
-	wg.Add(1)
-	go func() {
-		for _, client := range sys.peerClients {
-			if client == nil {
-				continue
-			}
-			serverNetInfo, err := client.DispatchNetInfo(ctx)
-			if err != nil {
-				serverNetInfo.Addr = client.host.String()
-				serverNetInfo.Error = err.Error()
-			}
-			serverNetInfos <- serverNetInfo
-		}
-		wg.Done()
-	}()
-
-	go func() {
-		wg.Wait()
-		close(serverNetInfos)
-	}()
-
-	return serverNetInfos
-}
-
-// GetParallelNetPerfInfo - Performs Net parallel tests
-func (sys *NotificationSys) GetParallelNetPerfInfo(ctx context.Context) madmin.NetPerfInfo {
-	netInfos := []madmin.PeerNetPerfInfo{}
-	wg := sync.WaitGroup{}
-
-	for index, client := range sys.peerClients {
-		if client == nil {
-			continue
-		}
-
-		wg.Add(1)
-		go func(index int) {
-			netInfo, err := sys.peerClients[index].GetNetPerfInfo(ctx)
-			netInfo.Addr = sys.peerClients[index].host.String()
-			if err != nil {
-				netInfo.Error = err.Error()
-			}
-			netInfos = append(netInfos, netInfo)
-			wg.Done()
-		}(index)
-	}
-	wg.Wait()
-	return madmin.NetPerfInfo{
-		NodeCommon:  madmin.NodeCommon{Addr: globalLocalNodeName},
-		RemotePeers: netInfos,
-	}
-}
-
-// GetDrivePerfInfos - Drive performance information
-func (sys *NotificationSys) GetDrivePerfInfos(ctx context.Context) chan madmin.DrivePerfInfos {
-	updateChan := make(chan madmin.DrivePerfInfos)
-	wg := sync.WaitGroup{}
-
-	for _, client := range sys.peerClients {
-		if client == nil {
-			continue
-		}
-		wg.Add(1)
-		go func(client *peerRESTClient) {
-			reply, err := client.GetDrivePerfInfos(ctx)
-
-			addr := client.host.String()
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-
-			reply.Addr = addr
-			if err != nil {
-				reply.Error = err.Error()
-			}
-
-			updateChan <- reply
-			wg.Done()
-		}(client)
-	}
-
-	go func() {
-		wg.Wait()
-		close(updateChan)
-	}()
-
-	return updateChan
-}
-
 // GetCPUs - Get all CPU information.
 func (sys *NotificationSys) GetCPUs(ctx context.Context) []madmin.CPUs {
 	reply := make([]madmin.CPUs, len(sys.peerClients))
@@ -1344,6 +1150,35 @@ func (sys *NotificationSys) restClientFromHash(s string) (client *peerRESTClient
 	return peerClients[idx]
 }
 
+// GetPeerOnlineCount gets the count of online and offline nodes.
+func (sys *NotificationSys) GetPeerOnlineCount() (nodesOnline, nodesOffline int) {
+	nodesOnline = 1 // Self is always online.
+	nodesOffline = 0
+	nodesOnlineIndex := make([]bool, len(sys.peerClients))
+	var wg sync.WaitGroup
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(idx int, client *peerRESTClient) {
+			defer wg.Done()
+			nodesOnlineIndex[idx] = client.restClient.HealthCheckFn()
+		}(idx, client)
+
+	}
+	wg.Wait()
+
+	for _, online := range nodesOnlineIndex {
+		if online {
+			nodesOnline++
+		} else {
+			nodesOffline++
+		}
+	}
+	return
+}
+
 // NewNotificationSys - creates new notification system object.
 func NewNotificationSys(endpoints EndpointServerPools) *NotificationSys {
 	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.Init()
@@ -1358,21 +1193,6 @@ func NewNotificationSys(endpoints EndpointServerPools) *NotificationSys {
 	}
 }
 
-// GetPeerOnlineCount gets the count of online and offline nodes.
-func GetPeerOnlineCount() (nodesOnline, nodesOffline int) {
-	nodesOnline = 1 // Self is always online.
-	nodesOffline = 0
-	servers := globalNotificationSys.ServerInfo()
-	for _, s := range servers {
-		if s.State == string(madmin.ItemOnline) {
-			nodesOnline++
-			continue
-		}
-		nodesOffline++
-	}
-	return
-}
-
 type eventArgs struct {
 	EventName    event.Name
 	BucketName   string
@@ -1524,7 +1344,7 @@ func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...
 }
 
 // GetClusterMetrics - gets the cluster metrics from all nodes excluding self.
-func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) chan Metric {
+func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric {
 	if sys == nil {
 		return nil
 	}
@@ -1545,11 +1365,14 @@ func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) chan Metric {
 	ch := make(chan Metric)
 	var wg sync.WaitGroup
 	for index, err := range g.Wait() {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
-			sys.peerClients[index].host.String())
-		ctx := logger.SetReqInfo(ctx, reqInfo)
 		if err != nil {
-			logger.LogOnceIf(ctx, err, sys.peerClients[index].host.String())
+			if sys.peerClients[index] != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
+					sys.peerClients[index].host.String())
+				logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String())
+			} else {
+				logger.LogOnceIf(ctx, err, "peer-offline")
+			}
 			continue
 		}
 		wg.Add(1)
@@ -1722,7 +1545,6 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
 func (sys *NotificationSys) DriveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) chan madmin.DriveSpeedTestResult {
 	ch := make(chan madmin.DriveSpeedTestResult)
 	var wg sync.WaitGroup
-
 	for _, client := range sys.peerClients {
 		if client == nil {
 			continue
@@ -1735,7 +1557,10 @@ func (sys *NotificationSys) DriveSpeedTest(ctx context.Context, opts madmin.Driv
 				resp.Error = err.Error()
 			}
 
-			ch <- resp
+			select {
+			case <-ctx.Done():
+			case ch <- resp:
+			}
 
 			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String())
 			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
@@ -1743,10 +1568,19 @@ func (sys *NotificationSys) DriveSpeedTest(ctx context.Context, opts madmin.Driv
 		}(client)
 	}
 
+	wg.Add(1)
 	go func() {
+		defer wg.Done()
+		select {
+		case <-ctx.Done():
+		case ch <- driveSpeedTest(ctx, opts):
+		}
+	}()
+
+	go func(wg *sync.WaitGroup, ch chan madmin.DriveSpeedTestResult) {
 		wg.Wait()
 		close(ch)
-	}()
+	}(&wg, ch)
 
 	return ch
 }

cmd/object-handlers.go

@@ -3101,10 +3101,12 @@ func (w *whiteSpaceWriter) WriteHeader(statusCode int) {
 // we do background append as and when the parts arrive and completeMultiPartUpload
 // is quick. Only in a rare case where parts would be out of order will
 // FS:completeMultiPartUpload() take a longer time.
-func sendWhiteSpace(w http.ResponseWriter) <-chan bool {
+func sendWhiteSpace(ctx context.Context, w http.ResponseWriter) <-chan bool {
 	doneCh := make(chan bool)
 	go func() {
+		defer close(doneCh)
 		ticker := time.NewTicker(time.Second * 10)
+		defer ticker.Stop()
 		headerWritten := false
 		for {
 			select {
@@ -3124,7 +3126,8 @@ func sendWhiteSpace(w http.ResponseWriter) <-chan bool {
 				}
 				w.(http.Flusher).Flush()
 			case doneCh <- headerWritten:
-				ticker.Stop()
+				return
+			case <-ctx.Done():
 				return
 			}
 		}
@@ -3298,7 +3301,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	w = &whiteSpaceWriter{ResponseWriter: w, Flusher: w.(http.Flusher)}
-	completeDoneCh := sendWhiteSpace(w)
+	completeDoneCh := sendWhiteSpace(ctx, w)
 	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, complMultipartUpload.Parts, opts)
 	// Stop writing white spaces to the client. Note that close(doneCh) style is not used as it
 	// can cause white space to be written after we send XML response in a race condition.

cmd/peer-rest-client.go

@@ -24,15 +24,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"net/url"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"time"
 
-	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/http"
@@ -111,271 +107,6 @@ func (client *peerRESTClient) ServerInfo() (info madmin.ServerProperties, err er
 	return info, err
 }
 
-type networkOverloadedErr struct{}
-
-var networkOverloaded networkOverloadedErr
-
-func (n networkOverloadedErr) Error() string {
-	return "network overloaded"
-}
-
-type nullReader struct{}
-
-func (r *nullReader) Read(b []byte) (int, error) {
-	return len(b), nil
-}
-
-func (client *peerRESTClient) doNetTest(ctx context.Context, dataSize int64, threadCount uint) (info madmin.PeerNetPerfInfo, err error) {
-	var mu sync.Mutex // mutex used to protect these slices in go-routines
-	latencies := []float64{}
-	throughputs := []float64{}
-
-	buflimiter := make(chan struct{}, threadCount)
-	errChan := make(chan error, threadCount)
-
-	var totalTransferred int64
-
-	// ensure enough samples to obtain normal distribution
-	maxSamples := int(10 * threadCount)
-	if maxSamples > 50 {
-		maxSamples = 50
-	}
-
-	innerCtx, cancel := context.WithCancel(ctx)
-
-	slowSamples := int32(0)
-	maxSlowSamples := int32(maxSamples/20) + 1 // 5% of total
-	slowSample := func() {
-		if slowSamples > maxSlowSamples {
-			return
-		}
-		if atomic.AddInt32(&slowSamples, 1) > maxSlowSamples {
-			errChan <- networkOverloaded
-			cancel()
-		}
-	}
-
-	var wg sync.WaitGroup
-	finish := func() {
-		<-buflimiter
-		wg.Done()
-	}
-
-	for i := 0; i < maxSamples; i++ {
-		if slowSamples > maxSlowSamples {
-			break
-		}
-
-		select {
-		case <-ctx.Done():
-			cancel()
-			return info, ctx.Err()
-		case err = <-errChan:
-		case buflimiter <- struct{}{}:
-			if slowSamples > maxSlowSamples {
-				break
-			}
-			wg.Add(1)
-
-			if innerCtx.Err() != nil {
-				finish()
-				continue
-			}
-
-			go func(i int) {
-				start := time.Now()
-				before := atomic.LoadInt64(&totalTransferred)
-
-				ctx, cancel := context.WithTimeout(innerCtx, 3*time.Second)
-				defer cancel()
-
-				progress := io.LimitReader(&nullReader{}, dataSize)
-
-				// Turn off healthCheckFn for health tests to cater for higher load on the peers.
-				clnt := newPeerRESTClient(client.host)
-				clnt.restClient.HealthCheckFn = nil
-
-				respBody, err := clnt.callWithContext(ctx, peerRESTMethodNetInfo, nil, progress, dataSize)
-				if err != nil {
-					if errors.Is(err, context.DeadlineExceeded) {
-						slowSample()
-						finish()
-						return
-					}
-
-					errChan <- err
-					finish()
-					return
-				}
-				http.DrainBody(respBody)
-
-				finish()
-				atomic.AddInt64(&totalTransferred, dataSize)
-				after := atomic.LoadInt64(&totalTransferred)
-				end := time.Now()
-
-				latency := end.Sub(start).Seconds()
-
-				if latency > maxLatencyForSizeThreads(dataSize, threadCount) {
-					slowSample()
-				}
-
-				/* Throughput = (total data transferred across all threads / time taken) */
-				throughput := float64((after - before)) / latency
-
-				// Protect updating latencies and throughputs slices from
-				// multiple go-routines.
-				mu.Lock()
-				latencies = append(latencies, latency)
-				throughputs = append(throughputs, throughput)
-				mu.Unlock()
-			}(i)
-		}
-	}
-	wg.Wait()
-
-	if slowSamples > maxSlowSamples {
-		return info, networkOverloaded
-	}
-	if err != nil {
-		return info, err
-	}
-
-	latency, throughput, err := xnet.ComputePerfStats(latencies, throughputs)
-	return madmin.PeerNetPerfInfo{
-		Latency: madmin.Latency{
-			Avg:          round(latency.Avg, 3),
-			Max:          round(latency.Max, 3),
-			Min:          round(latency.Min, 3),
-			Percentile50: round(latency.Percentile50, 3),
-			Percentile90: round(latency.Percentile90, 3),
-			Percentile99: round(latency.Percentile99, 3),
-		},
-		Throughput: madmin.Throughput{
-			Avg:          uint64(round(throughput.Avg, 0)),
-			Max:          uint64(round(throughput.Max, 0)),
-			Min:          uint64(round(throughput.Min, 0)),
-			Percentile50: uint64(round(throughput.Percentile50, 0)),
-			Percentile90: uint64(round(throughput.Percentile90, 0)),
-			Percentile99: uint64(round(throughput.Percentile99, 0)),
-		},
-	}, nil
-}
-
-func maxLatencyForSizeThreads(size int64, threadCount uint) float64 {
-	Gbit100 := 12.5 * float64(humanize.GiByte)
-	Gbit40 := 5.00 * float64(humanize.GiByte)
-	Gbit25 := 3.25 * float64(humanize.GiByte)
-	Gbit10 := 1.25 * float64(humanize.GiByte)
-	// Gbit1 := 0.25 * float64(humanize.GiByte)
-
-	// Given the current defaults, each combination of size/thread
-	// is supposed to fully saturate the intended pipe when all threads are active
-	// i.e. if the test is performed in a perfectly controlled environment, i.e. without
-	// CPU scheduling latencies and/or network jitters, then all threads working
-	// simultaneously should result in each of them completing in 1s
-	//
-	// In reality, I've assumed a normal distribution of latency with expected mean of 1s and min of 0s
-	// Then, 95% of threads should complete within 2 seconds (2 std. deviations from the mean). The 2s comes
-	// from fitting the normal curve such that the mean is 1.
-	//
-	// i.e. we expect that no more than 5% of threads to take longer than 2s to push the data.
-	//
-	// throughput  |  max latency
-	//   100 Gbit  |  2s
-	//    40 Gbit  |  2s
-	//    25 Gbit  |  2s
-	//    10 Gbit  |  2s
-	//     1 Gbit  |  inf
-
-	throughput := float64(size * int64(threadCount))
-	if throughput >= Gbit100 {
-		return 2.0
-	} else if throughput >= Gbit40 {
-		return 2.0
-	} else if throughput >= Gbit25 {
-		return 2.0
-	} else if throughput >= Gbit10 {
-		return 2.0
-	}
-	return math.MaxFloat64
-}
-
-// GetNetPerfInfo - fetch network information for a remote node.
-func (client *peerRESTClient) GetNetPerfInfo(ctx context.Context) (info madmin.PeerNetPerfInfo, err error) {
-	// 100 Gbit ->  256 MiB  *  50 threads
-	// 40 Gbit  ->  256 MiB  *  20 threads
-	// 25 Gbit  ->  128 MiB  *  25 threads
-	// 10 Gbit  ->  128 MiB  *  10 threads
-	// 1 Gbit   ->  64  MiB  *  2  threads
-
-	type step struct {
-		size    int64
-		threads uint
-	}
-	steps := []step{
-		{ // 100 Gbit
-			size:    256 * humanize.MiByte,
-			threads: 50,
-		},
-		{ // 40 Gbit
-			size:    256 * humanize.MiByte,
-			threads: 20,
-		},
-		{ // 25 Gbit
-			size:    128 * humanize.MiByte,
-			threads: 25,
-		},
-		{ // 10 Gbit
-			size:    128 * humanize.MiByte,
-			threads: 10,
-		},
-		{ // 1 Gbit
-			size:    64 * humanize.MiByte,
-			threads: 2,
-		},
-	}
-
-	for i := range steps {
-		size := steps[i].size
-		threads := steps[i].threads
-
-		if info, err = client.doNetTest(ctx, size, threads); err != nil {
-			if err == networkOverloaded {
-				continue
-			}
-		}
-		return info, err
-	}
-	return info, err
-}
-
-// DispatchNetInfo - dispatch other nodes to run Net info.
-func (client *peerRESTClient) DispatchNetInfo(ctx context.Context) (info madmin.NetPerfInfo, err error) {
-	respBody, err := client.callWithContext(ctx, peerRESTMethodDispatchNetInfo, nil, nil, -1)
-	if err != nil {
-		return
-	}
-	defer http.DrainBody(respBody)
-	waitReader, err := waitForHTTPResponse(respBody)
-	if err != nil {
-		return
-	}
-	err = gob.NewDecoder(waitReader).Decode(&info)
-	return
-}
-
-// GetDrivePerfInfos - fetch all disk's serial/parallal performance information for a remote node.
-func (client *peerRESTClient) GetDrivePerfInfos(ctx context.Context) (info madmin.DrivePerfInfos, err error) {
-	respBody, err := client.callWithContext(ctx, peerRESTMethodDriveInfo, nil, nil, -1)
-	if err != nil {
-		return
-	}
-	defer http.DrainBody(respBody)
-	err = gob.NewDecoder(respBody).Decode(&info)
-	return info, err
-}
-
 // GetCPUs - fetch CPU information for a remote node.
 func (client *peerRESTClient) GetCPUs(ctx context.Context) (info madmin.CPUs, err error) {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodCPUInfo, nil, nil, -1)

cmd/peer-rest-common.go

@@ -27,8 +27,6 @@ const (
 const (
 	peerRESTMethodHealth                      = "/health"
 	peerRESTMethodServerInfo                  = "/serverinfo"
-	peerRESTMethodDriveInfo                   = "/driveinfo"
-	peerRESTMethodNetInfo                     = "/netinfo"
 	peerRESTMethodCPUInfo                     = "/cpuinfo"
 	peerRESTMethodDiskHwInfo                  = "/diskhwinfo"
 	peerRESTMethodOsInfo                      = "/osinfo"
@@ -37,7 +35,6 @@ const (
 	peerRESTMethodSysErrors                   = "/syserrors"
 	peerRESTMethodSysServices                 = "/sysservices"
 	peerRESTMethodSysConfig                   = "/sysconfig"
-	peerRESTMethodDispatchNetInfo             = "/dispatchnetinfo"
 	peerRESTMethodDeleteBucketMetadata        = "/deletebucketmetadata"
 	peerRESTMethodLoadBucketMetadata          = "/loadbucketmetadata"
 	peerRESTMethodGetBucketStats              = "/getbucketstats"

cmd/peer-rest-server.go

@@ -23,7 +23,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
@@ -337,68 +336,6 @@ func (s *peerRESTServer) ServerInfoHandler(w http.ResponseWriter, r *http.Reques
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
-func (s *peerRESTServer) NetInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "NetInfo")
-	if !s.IsValid(w, r) {
-		s.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	// Use this trailer to send additional headers after sending body
-	w.Header().Set("Trailer", "FinalStatus")
-
-	w.Header().Set("Content-Type", "application/octet-stream")
-	w.WriteHeader(http.StatusOK)
-
-	n, err := io.Copy(ioutil.Discard, r.Body)
-	if err == io.ErrUnexpectedEOF {
-		w.Header().Set("FinalStatus", err.Error())
-		return
-	}
-	if err != nil && err != io.EOF {
-		logger.LogIf(ctx, err)
-		w.Header().Set("FinalStatus", err.Error())
-		return
-	}
-	if n != r.ContentLength {
-		err := fmt.Errorf("Subnet health: short read: expected %d found %d", r.ContentLength, n)
-		logger.LogIf(ctx, err)
-		w.Header().Set("FinalStatus", err.Error())
-		return
-	}
-	w.Header().Set("FinalStatus", "Success")
-}
-
-func (s *peerRESTServer) DispatchNetInfoHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		s.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	done := keepHTTPResponseAlive(w)
-
-	ctx := newContext(r, w, "DispatchNetInfo")
-	info := globalNotificationSys.GetNetPerfInfo(ctx)
-
-	done(nil)
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
-}
-
-// GetDrivePerfInfosHandler - returns all disk's serial/parallal performance information.
-func (s *peerRESTServer) GetDrivePerfInfosHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		s.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	ctx, cancel := context.WithCancel(newContext(r, w, "DriveInfo"))
-	defer cancel()
-
-	info := getDrivePerfInfos(ctx, r.Host)
-
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
-}
-
 // GetCPUsHandler - returns CPU info.
 func (s *peerRESTServer) GetCPUsHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -538,6 +475,8 @@ func (s *peerRESTServer) DeleteBucketMetadataHandler(w http.ResponseWriter, r *h
 	globalReplicationStats.Delete(bucketName)
 	globalBucketMetadataSys.Remove(bucketName)
 	globalBucketTargetSys.Delete(bucketName)
+	globalNotificationSys.RemoveNotification(bucketName)
+	globalBucketConnStats.delete(bucketName)
 	if localMetacacheMgr != nil {
 		localMetacacheMgr.deleteBucketCache(bucketName)
 	}
@@ -1174,13 +1113,9 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 		s.writeErrorResponse(w, errors.New("invalid request"))
 	}
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
 	enc := gob.NewEncoder(w)
 
-	ch := ReportMetrics(r.Context(), peerMetricsGroups)
-	for m := range ch {
+	for m := range ReportMetrics(r.Context(), peerMetricsGroups) {
 		if err := enc.Encode(m); err != nil {
 			s.writeErrorResponse(w, errors.New("Encoding metric failed: "+err.Error()))
 			return
@@ -1315,8 +1250,8 @@ func (s *peerRESTServer) DevNull(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// Netperf - perform netperf
-func (s *peerRESTServer) Netperf(w http.ResponseWriter, r *http.Request) {
+// NetSpeedTestHandlers - perform network speedtest
+func (s *peerRESTServer) NetSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
 		return
@@ -1346,9 +1281,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveInfo).HandlerFunc(httpTraceHdrs(server.GetDrivePerfInfosHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetInfo).HandlerFunc(httpTraceHdrs(server.NetInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDispatchNetInfo).HandlerFunc(httpTraceHdrs(server.DispatchNetInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCycleBloom).HandlerFunc(httpTraceHdrs(server.CycleServerBloomFilterHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(httpTraceHdrs(server.GetAllBucketStatsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(httpTraceHdrs(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
@@ -1379,7 +1311,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.Netperf))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.NetSpeedTestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))

cmd/perf-tests.go

@@ -87,7 +87,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 		uploadsCancel()
 	}()
 
-	objNamePrefix := uuid.New().String() + "/"
+	objNamePrefix := uuid.New().String() + SlashSeparator
 
 	userMetadata := make(map[string]string)
 	userMetadata[globalObjectPerfUserMetadata] = "true"
@@ -98,7 +98,11 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 			defer wg.Done()
 			for {
 				reader := newRandomReader(size)
-				info, err := client.PutObject(uploadsCtx, globalObjectPerfBucket, fmt.Sprintf("%s%d.%d", objNamePrefix, i, objCountPerThread[i]), reader, int64(size), minio.PutObjectOptions{UserMetadata: userMetadata, DisableMultipart: true}) // Bypass S3 API freeze
+				tmpObjName := fmt.Sprintf("%s%d.%d", objNamePrefix, i, objCountPerThread[i])
+				info, err := client.PutObject(uploadsCtx, globalObjectPerfBucket, tmpObjName, reader, int64(size), minio.PutObjectOptions{
+					UserMetadata:     userMetadata,
+					DisableMultipart: true,
+				}) // Bypass S3 API freeze
 				if err != nil {
 					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
 						errOnce.Do(func() {

cmd/server-main.go

@@ -138,6 +138,13 @@ func serverCmdArgs(ctx *cli.Context) []string {
 		logger.FatalIf(err, "Unable to validate passed arguments in %s:%s",
 			config.EnvArgs, os.Getenv(config.EnvArgs))
 	}
+	if v == "" {
+		v, _, _, err = env.LookupEnv(config.EnvVolumes)
+		if err != nil {
+			logger.FatalIf(err, "Unable to validate passed arguments in %s:%s",
+				config.EnvVolumes, os.Getenv(config.EnvVolumes))
+		}
+	}
 	if v == "" {
 		// Fall back to older environment value MINIO_ENDPOINTS
 		v, _, _, err = env.LookupEnv(config.EnvEndpoints)
@@ -422,11 +429,14 @@ func serverMain(ctx *cli.Context) {
 	erasureSelfTest()
 	compressSelfTest()
 
+	// Handle all server environment vars.
+	serverHandleEnvVars()
+
 	// Handle all server command args.
 	serverHandleCmdArgs(ctx)
 
-	// Handle all server environment vars.
-	serverHandleEnvVars()
+	// Initialize KMS configuration
+	handleKMSConfig()
 
 	// Set node name, only set for distributed setup.
 	globalConsoleSys.SetNodeName(globalLocalNodeName)
@@ -587,9 +597,6 @@ func serverMain(ctx *cli.Context) {
 			}
 		}()
 
-		// Initialize site replication manager.
-		globalSiteReplicationSys.Init(GlobalContext, newObject)
-
 		// Initialize quota manager.
 		globalBucketQuotaSys.Init(newObject)
 
@@ -612,6 +619,9 @@ func serverMain(ctx *cli.Context) {
 		// Initialize bucket metadata sub-system.
 		globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
 
+		// Initialize site replication manager.
+		globalSiteReplicationSys.Init(GlobalContext, newObject)
+
 		// Initialize bucket notification targets.
 		globalNotificationSys.InitBucketTargets(GlobalContext, newObject)
 

cmd/site-replication.go

@@ -149,7 +149,14 @@ type SRError struct {
 }
 
 func (c SRError) Error() string {
-	return c.Cause.Error()
+	if c.Cause != nil {
+		return c.Cause.Error()
+	}
+	return "<nil>"
+}
+
+func (c SRError) Unwrap() error {
+	return c.Cause
 }
 
 func wrapSRErr(err error) SRError {
@@ -249,13 +256,13 @@ func (c *SiteReplicationSys) saveToDisk(ctx context.Context, state srState) erro
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
-	err = saveConfig(ctx, objAPI, getSRStateFilePath(), buf)
-	if err != nil {
+
+	if err = saveConfig(ctx, objAPI, getSRStateFilePath(), buf); err != nil {
 		return err
 	}
 
-	for _, e := range globalNotificationSys.ReloadSiteReplicationConfig(ctx) {
-		logger.LogIf(ctx, e)
+	for _, err := range globalNotificationSys.ReloadSiteReplicationConfig(ctx) {
+		logger.LogIf(ctx, err)
 	}
 
 	c.Lock()
@@ -606,6 +613,14 @@ func (c *SiteReplicationSys) GetClusterInfo(ctx context.Context) (info madmin.Si
 	return info, nil
 }
 
+const (
+	makeBucketWithVersion   = "MakeBucketWithVersioning"
+	configureReplication    = "ConfigureReplication"
+	deleteBucket            = "DeleteBucket"
+	replicateIAMItem        = "SRPeerReplicateIAMItem"
+	replicateBucketMetadata = "SRPeerReplicateBucketMeta"
+)
+
 // MakeBucketHook - called during a regular make bucket call when cluster
 // replication is enabled. It is responsible for the creation of the same bucket
 // on remote clusters, and creating replication rules on local and peer
@@ -637,9 +652,7 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 	// Create bucket and enable versioning on all peers.
 	makeBucketConcErr := c.concDo(
 		func() error {
-			err := c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
-			logger.LogIf(ctx, c.annotateErr("MakeWithVersioning", err))
-			return err
+			return c.annotateErr(makeBucketWithVersion, c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts))
 		},
 		func(deploymentID string, p madmin.PeerInfo) error {
 			admClient, err := c.getAdminClient(ctx, deploymentID)
@@ -647,27 +660,15 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 				return err
 			}
 
-			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap)
-			logger.LogIf(ctx, c.annotatePeerErr(p.Name, "MakeWithVersioning", err))
-			return err
+			return c.annotatePeerErr(p.Name, makeBucketWithVersion, admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap))
 		},
+		makeBucketWithVersion,
 	)
-	// If all make-bucket-and-enable-versioning operations failed, nothing
-	// more to do.
-	if makeBucketConcErr.allFailed() {
-		return makeBucketConcErr
-	}
 
-	// Log any errors in make-bucket operations.
-	logger.LogIf(ctx, makeBucketConcErr.summaryErr)
-
-	// Create bucket remotes and add replication rules for the bucket on
-	// self and peers.
+	// Create bucket remotes and add replication rules for the bucket on self and peers.
 	makeRemotesConcErr := c.concDo(
 		func() error {
-			err := c.PeerBucketConfigureReplHandler(ctx, bucket)
-			logger.LogIf(ctx, c.annotateErr("ConfigureRepl", err))
-			return err
+			return c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket))
 		},
 		func(deploymentID string, p madmin.PeerInfo) error {
 			admClient, err := c.getAdminClient(ctx, deploymentID)
@@ -675,13 +676,16 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 				return err
 			}
 
-			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil)
-			logger.LogIf(ctx, c.annotatePeerErr(p.Name, "ConfigureRepl", err))
-			return err
+			return c.annotatePeerErr(p.Name, configureReplication, admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil))
 		},
+		configureReplication,
 	)
-	err := makeRemotesConcErr.summaryErr
-	if err != nil {
+
+	if err := errors.Unwrap(makeBucketConcErr); err != nil {
+		return err
+	}
+
+	if err := errors.Unwrap(makeRemotesConcErr); err != nil {
 		return err
 	}
 
@@ -706,17 +710,17 @@ func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string
 	}
 
 	// Send bucket delete to other clusters.
-	cErr := c.concDo(nil, func(deploymentID string, p madmin.PeerInfo) error {
+	cerr := c.concDo(nil, func(deploymentID string, p madmin.PeerInfo) error {
 		admClient, err := c.getAdminClient(ctx, deploymentID)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRPeerBucketOps(ctx, bucket, op, nil)
-		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "DeleteBucket", err))
-		return err
-	})
-	return cErr.summaryErr
+		return c.annotatePeerErr(p.Name, deleteBucket, admClient.SRPeerBucketOps(ctx, bucket, op, nil))
+	},
+		deleteBucket,
+	)
+	return errors.Unwrap(cerr)
 }
 
 // PeerBucketMakeWithVersioningHandler - creates bucket and enables versioning.
@@ -732,8 +736,7 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 		_, ok1 := err.(BucketExists)
 		_, ok2 := err.(BucketAlreadyExists)
 		if !ok1 && !ok2 {
-			logger.LogIf(ctx, c.annotateErr("MakeBucketErr on peer call", err))
-			return wrapSRErr(err)
+			return wrapSRErr(c.annotateErr(makeBucketWithVersion, err))
 		}
 	} else {
 		// Load updated bucket metadata into memory as new
@@ -743,8 +746,7 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 
 	meta, err := globalBucketMetadataSys.Get(bucket)
 	if err != nil {
-		logger.LogIf(ctx, c.annotateErr("MakeBucketErr on peer call", err))
-		return wrapSRErr(err)
+		return wrapSRErr(c.annotateErr(makeBucketWithVersion, err))
 	}
 
 	meta.VersioningConfigXML = enabledBucketVersioningConfig
@@ -805,8 +807,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 			bucketTarget.Arn = globalBucketTargetSys.getRemoteARN(bucket, &bucketTarget)
 			err := globalBucketTargetSys.SetTarget(ctx, bucket, &bucketTarget, false)
 			if err != nil {
-				logger.LogIf(ctx, c.annotatePeerErr(peer.Name, "Bucket target creation error", err))
-				return err
+				return c.annotatePeerErr(peer.Name, "Bucket target creation error", err)
 			}
 			targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 			if err != nil {
@@ -901,11 +902,10 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 		default:
 			err = replicationConfig.AddRule(opts)
 		}
-
 		if err != nil {
-			logger.LogIf(ctx, c.annotatePeerErr(peer.Name, "Error adding bucket replication rule", err))
-			return err
+			return c.annotatePeerErr(peer.Name, "Error adding bucket replication rule", err)
 		}
+
 		// Now convert the configuration back to server's type so we can
 		// do some validation.
 		newReplCfgBytes, err := xml.Marshal(replicationConfig)
@@ -929,9 +929,9 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 		if err != nil {
 			return err
 		}
+
 		err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, replCfgData)
-		logger.LogIf(ctx, c.annotatePeerErr(peer.Name, "Error updating replication configuration", err))
-		return err
+		return c.annotatePeerErr(peer.Name, "Error updating replication configuration", err)
 	}
 
 	c.RLock()
@@ -941,11 +941,9 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 		if d == globalDeploymentID {
 			continue
 		}
-		if err := configurePeerFn(d, peer); err != nil {
-			errMap[d] = err
-		}
+		errMap[d] = configurePeerFn(d, peer)
 	}
-	return c.toErrorFromErrMap(errMap)
+	return c.toErrorFromErrMap(errMap, configureReplication)
 }
 
 // PeerBucketDeleteHandler - deletes bucket on local in response to a delete
@@ -1009,17 +1007,17 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 		return nil
 	}
 
-	cErr := c.concDo(nil, func(d string, p madmin.PeerInfo) error {
+	cerr := c.concDo(nil, func(d string, p madmin.PeerInfo) error {
 		admClient, err := c.getAdminClient(ctx, d)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRPeerReplicateIAMItem(ctx, item)
-		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRPeerReplicateIAMItem", err))
-		return err
-	})
-	return cErr.summaryErr
+		return c.annotatePeerErr(p.Name, replicateIAMItem, admClient.SRPeerReplicateIAMItem(ctx, item))
+	},
+		replicateIAMItem,
+	)
+	return errors.Unwrap(cerr)
 }
 
 // PeerAddPolicyHandler - copies IAM policy to local. A nil policy argument,
@@ -1165,14 +1163,13 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
 	// Verify the session token of the stsCred
 	claims, err := auth.ExtractClaims(stsCred.SessionToken, globalActiveCred.SecretKey)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return fmt.Errorf("STS credential could not be verified")
+		return fmt.Errorf("STS credential could not be verified: %w", err)
 	}
 
 	mapClaims := claims.Map()
 	expiry, err := auth.ExpToInt64(mapClaims["exp"])
 	if err != nil {
-		return fmt.Errorf("Expiry claim was not found: %v", mapClaims)
+		return fmt.Errorf("Expiry claim was not found: %v: %w", mapClaims, err)
 	}
 
 	cred := auth.Credentials{
@@ -1191,7 +1188,7 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
 		// Need to lookup the groups from LDAP.
 		_, ldapGroups, err := globalLDAPConfig.LookupUserDN(ldapUser)
 		if err != nil {
-			return fmt.Errorf("unable to query LDAP server for %s: %v", ldapUser, err)
+			return fmt.Errorf("unable to query LDAP server for %s: %w", ldapUser, err)
 		}
 
 		cred.Groups = ldapGroups
@@ -1199,7 +1196,7 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *mad
 
 	// Set these credentials to IAM.
 	if err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, stsCred.ParentPolicyMapping); err != nil {
-		return fmt.Errorf("unable to save STS credential and/or parent policy mapping: %v", err)
+		return fmt.Errorf("unable to save STS credential and/or parent policy mapping: %w", err)
 	}
 
 	return nil
@@ -1217,17 +1214,17 @@ func (c *SiteReplicationSys) BucketMetaHook(ctx context.Context, item madmin.SRB
 		return nil
 	}
 
-	cErr := c.concDo(nil, func(d string, p madmin.PeerInfo) error {
+	cerr := c.concDo(nil, func(d string, p madmin.PeerInfo) error {
 		admClient, err := c.getAdminClient(ctx, d)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRPeerReplicateBucketMeta(ctx, item)
-		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRPeerReplicateBucketMeta", err))
-		return err
-	})
-	return cErr.summaryErr
+		return c.annotatePeerErr(p.Name, replicateBucketMetadata, admClient.SRPeerReplicateBucketMeta(ctx, item))
+	},
+		replicateBucketMetadata,
+	)
+	return errors.Unwrap(cerr)
 }
 
 // PeerBucketVersioningHandler - updates versioning config to local cluster.
@@ -1795,52 +1792,63 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 // Concurrency helpers
 
 type concErr struct {
-	numActions int
 	errMap     map[string]error
 	summaryErr error
 }
 
 func (c concErr) Error() string {
-	return c.summaryErr.Error()
+	if c.summaryErr != nil {
+		return c.summaryErr.Error()
+	}
+	return "<nil>"
 }
 
-func (c concErr) allFailed() bool {
-	return len(c.errMap) == c.numActions
+func (c concErr) Unwrap() error {
+	return c.summaryErr
 }
 
-func (c *SiteReplicationSys) toErrorFromErrMap(errMap map[string]error) error {
+func (c *SiteReplicationSys) toErrorFromErrMap(errMap map[string]error, actionName string) error {
 	if len(errMap) == 0 {
 		return nil
 	}
 
+	var success int
 	msgs := []string{}
 	for d, err := range errMap {
 		name := c.state.Peers[d].Name
-		msgs = append(msgs, fmt.Sprintf("Site %s (%s): %v", name, d, err))
+		if err == nil {
+			msgs = append(msgs, fmt.Sprintf("'%s' on site %s (%s): succeeded", actionName, name, d))
+			success++
+		} else {
+			msgs = append(msgs, fmt.Sprintf("'%s' on site %s (%s): failed(%v)", actionName, name, d, err))
+		}
 	}
-	return fmt.Errorf("Site replication error(s): %s", strings.Join(msgs, "; "))
+	if success == len(errMap) {
+		return nil
+	}
+	return fmt.Errorf("Site replication error(s): \n%s", strings.Join(msgs, "\n"))
 }
 
-func (c *SiteReplicationSys) newConcErr(numActions int, errMap map[string]error) concErr {
+func (c *SiteReplicationSys) newConcErr(errMap map[string]error, actionName string) error {
 	return concErr{
-		numActions: numActions,
 		errMap:     errMap,
-		summaryErr: c.toErrorFromErrMap(errMap),
+		summaryErr: c.toErrorFromErrMap(errMap, actionName),
 	}
 }
 
 // concDo calls actions concurrently. selfActionFn is run for the current
 // cluster and peerActionFn is run for each peer replication cluster.
-func (c *SiteReplicationSys) concDo(selfActionFn func() error, peerActionFn func(deploymentID string, p madmin.PeerInfo) error) concErr {
+func (c *SiteReplicationSys) concDo(selfActionFn func() error, peerActionFn func(deploymentID string, p madmin.PeerInfo) error, actionName string) error {
 	depIDs := make([]string, 0, len(c.state.Peers))
 	for d := range c.state.Peers {
 		depIDs = append(depIDs, d)
 	}
 	errs := make([]error, len(c.state.Peers))
 	var wg sync.WaitGroup
+	wg.Add(len(depIDs))
 	for i := range depIDs {
-		wg.Add(1)
 		go func(i int) {
+			defer wg.Done()
 			if depIDs[i] == globalDeploymentID {
 				if selfActionFn != nil {
 					errs[i] = selfActionFn()
@@ -1848,35 +1856,28 @@ func (c *SiteReplicationSys) concDo(selfActionFn func() error, peerActionFn func
 			} else {
 				errs[i] = peerActionFn(depIDs[i], c.state.Peers[depIDs[i]])
 			}
-			wg.Done()
 		}(i)
 	}
 	wg.Wait()
 	errMap := make(map[string]error, len(c.state.Peers))
 	for i, depID := range depIDs {
-		if errs[i] != nil {
-			errMap[depID] = errs[i]
-		}
+		errMap[depID] = errs[i]
 	}
-	numActions := len(c.state.Peers) - 1
-	if selfActionFn != nil {
-		numActions++
-	}
-	return c.newConcErr(numActions, errMap)
+	return c.newConcErr(errMap, actionName)
 }
 
 func (c *SiteReplicationSys) annotateErr(annotation string, err error) error {
 	if err == nil {
 		return nil
 	}
-	return fmt.Errorf("%s: %s: %v", c.state.Name, annotation, err)
+	return fmt.Errorf("%s: %s: %w", c.state.Name, annotation, err)
 }
 
 func (c *SiteReplicationSys) annotatePeerErr(dstPeer string, annotation string, err error) error {
 	if err == nil {
 		return nil
 	}
-	return fmt.Errorf("%s->%s: %s: %v", c.state.Name, dstPeer, annotation, err)
+	return fmt.Errorf("%s->%s: %s: %w", c.state.Name, dstPeer, annotation, err)
 }
 
 // isEnabled returns true if site replication is enabled
@@ -2230,6 +2231,10 @@ func (c *SiteReplicationSys) SiteReplicationStatus(ctx context.Context, objAPI O
 	return
 }
 
+const (
+	replicationStatus = "ReplicationStatus"
+)
+
 // siteReplicationStatus returns the site replication status across clusters participating in site replication.
 func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI ObjectLayer, opts madmin.SRStatusOptions) (info srStatusInfo, err error) {
 	c.RLock()
@@ -2269,10 +2274,11 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
 			sris[depIdx[deploymentID]] = srInfo
 			return nil
 		},
+		replicationStatus,
 	)
 
-	if metaInfoConcErr.summaryErr != nil {
-		return info, errSRBackendIssue(metaInfoConcErr.summaryErr)
+	if err := errors.Unwrap(metaInfoConcErr); err != nil {
+		return info, errSRBackendIssue(err)
 	}
 
 	info.Enabled = true
@@ -3520,7 +3526,8 @@ func (c *SiteReplicationSys) healTagMetadata(ctx context.Context, objAPI ObjectL
 			Tags:   latestTaggingConfig,
 		})
 		if err != nil {
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing tagging metadata for peer %s from peer %s : %w", peerName, latestPeerName, err)))
+			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+				fmt.Errorf("Error healing tagging metadata for peer %s from peer %s : %w", peerName, latestPeerName, err)))
 		}
 	}
 	return nil
@@ -3582,8 +3589,9 @@ func (c *SiteReplicationSys) healBucketPolicies(ctx context.Context, objAPI Obje
 			Bucket: bucket,
 			Policy: latestIAMPolicy,
 		}); err != nil {
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing bucket policy metadata for peer %s from peer %s : %w",
-				peerName, latestPeerName, err)))
+			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+				fmt.Errorf("Error healing bucket policy metadata for peer %s from peer %s : %w",
+					peerName, latestPeerName, err)))
 		}
 	}
 	return nil
@@ -3640,7 +3648,7 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 		}
 		if dID == globalDeploymentID {
 			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, latestQuotaConfigBytes); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Error healing quota metadata from peer site %s : %s", latestPeerName, err.Error()))
+				logger.LogIf(ctx, fmt.Errorf("Error healing quota metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
 		}
@@ -3656,8 +3664,9 @@ func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI O
 			Bucket: bucket,
 			Quota:  latestQuotaConfigBytes,
 		}); err != nil {
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing quota config metadata for peer %s from peer %s : %s",
-				peerName, latestPeerName, err.Error())))
+			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+				fmt.Errorf("Error healing quota config metadata for peer %s from peer %s : %w",
+					peerName, latestPeerName, err)))
 		}
 	}
 	return nil
@@ -3712,8 +3721,8 @@ func (c *SiteReplicationSys) healVersioningMetadata(ctx context.Context, objAPI
 			continue
 		}
 		if dID == globalDeploymentID {
-			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, latestVersioningConfigBytes); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Error healing sse metadata from peer site %s : %w", latestPeerName, err))
+			if err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, latestVersioningConfigBytes); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Error healing versioning metadata from peer site %s : %w", latestPeerName, err))
 			}
 			continue
 		}
@@ -3729,8 +3738,9 @@ func (c *SiteReplicationSys) healVersioningMetadata(ctx context.Context, objAPI
 			Versioning: latestVersioningConfig,
 		})
 		if err != nil {
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing versioning config metadata for peer %s from peer %s : %s",
-				peerName, latestPeerName, err.Error())))
+			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+				fmt.Errorf("Error healing versioning config metadata for peer %s from peer %s : %w",
+					peerName, latestPeerName, err)))
 		}
 	}
 	return nil
@@ -3802,8 +3812,9 @@ func (c *SiteReplicationSys) healSSEMetadata(ctx context.Context, objAPI ObjectL
 			SSEConfig: latestSSEConfig,
 		})
 		if err != nil {
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing SSE config metadata for peer %s from peer %s : %s",
-				peerName, latestPeerName, err.Error())))
+			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+				fmt.Errorf("Error healing SSE config metadata for peer %s from peer %s : %w",
+					peerName, latestPeerName, err)))
 		}
 	}
 	return nil
@@ -3875,8 +3886,9 @@ func (c *SiteReplicationSys) healOLockConfigMetadata(ctx context.Context, objAPI
 			Tags:   latestObjLockConfig,
 		})
 		if err != nil {
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "SRPeerReplicateBucketMeta", fmt.Errorf("Error healing object lock config metadata for peer %s from peer %s : %w",
-				peerName, latestPeerName, err)))
+			logger.LogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+				fmt.Errorf("Error healing object lock config metadata for peer %s from peer %s : %w",
+					peerName, latestPeerName, err)))
 		}
 	}
 	return nil
@@ -3939,25 +3951,19 @@ func (c *SiteReplicationSys) healCreateMissingBucket(ctx context.Context, objAPI
 		if dID == globalDeploymentID {
 			err := c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 			if err != nil {
-				logger.LogIf(ctx, c.annotateErr("MakeWithVersioning", fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
-					err, latestPeerName, peerName)))
-				return err
+				return c.annotateErr(makeBucketWithVersion, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
+					err, latestPeerName, peerName))
 			}
 		} else {
 			admClient, err := c.getAdminClient(ctx, dID)
 			if err != nil {
-				logger.LogIf(ctx, c.annotateErr("ConfigureRepl", fmt.Errorf("admin client not found: %w", err)))
-				return err
+				return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
 			}
-			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap)
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "MakeWithVersioning", err))
-			if err != nil {
-				return err
+			if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap); err != nil {
+				return c.annotatePeerErr(peerName, makeBucketWithVersion, err)
 			}
-			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil)
-			logger.LogIf(ctx, c.annotatePeerErr(peerName, "ConfigureRepl", err))
-			if err != nil {
-				return err
+			if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil); err != nil {
+				return c.annotatePeerErr(peerName, configureReplication, err)
 			}
 		}
 	}
@@ -3965,8 +3971,7 @@ func (c *SiteReplicationSys) healCreateMissingBucket(ctx context.Context, objAPI
 		// configure replication from current cluster to other clusters
 		err := c.PeerBucketConfigureReplHandler(ctx, bucket)
 		if err != nil {
-			logger.LogIf(ctx, c.annotateErr("ConfigureRepl", err))
-			return err
+			return c.annotateErr(configureReplication, err)
 		}
 	}
 
@@ -4007,10 +4012,7 @@ func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, ob
 	}
 
 	if replMismatch {
-		err := c.PeerBucketConfigureReplHandler(ctx, bucket)
-		if err != nil {
-			logger.LogIf(ctx, c.annotateErr("ConfigureRepl", err))
-		}
+		logger.LogIf(ctx, c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket)))
 	}
 	return nil
 }

cmd/speedtest.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"net/url"
 	"sort"
 	"time"
 
@@ -60,6 +62,17 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
 				if throughputHighestResults[i].Error != "" {
 					errStr = throughputHighestResults[i].Error
 				}
+
+				// if the default concurrency yields zero results, throw an error.
+				if throughputHighestResults[i].Downloads == 0 && opts.concurrencyStart == concurrency {
+					errStr = fmt.Sprintf("no results for downloads upon first attempt, concurrency %d and duration %s", opts.concurrencyStart, opts.duration)
+				}
+
+				// if the default concurrency yields zero results, throw an error.
+				if throughputHighestResults[i].Uploads == 0 && opts.concurrencyStart == concurrency {
+					errStr = fmt.Sprintf("no results for uploads upon first attempt, concurrency %d and duration %s", opts.concurrencyStart, opts.duration)
+				}
+
 				result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
 					Endpoint:         throughputHighestResults[i].Endpoint,
 					ThroughputPerSec: throughputHighestResults[i].Uploads / uint64(durationSecs),
@@ -171,9 +184,19 @@ func driveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) madmin.
 		return tmpPaths
 	}()
 
+	scheme := "http"
+	if globalIsTLS {
+		scheme = "https"
+	}
+
+	u := &url.URL{
+		Scheme: scheme,
+		Host:   globalLocalNodeName,
+	}
+
 	perfs, err := perf.Run(ctx, paths...)
 	return madmin.DriveSpeedTestResult{
-		Endpoint: globalLocalNodeName,
+		Endpoint: u.String(),
 		Version:  Version,
 		DrivePerf: func() (results []madmin.DrivePerf) {
 			for idx, r := range perfs {

cmd/sts-errors.go

@@ -48,14 +48,10 @@ func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, isErrCode
 	if errCtxt != nil {
 		stsErrorResponse.Error.Message = errCtxt.Error()
 	}
-	var logKind logger.Kind
 	switch errCode {
-	case ErrSTSInternalError, ErrSTSNotInitialized:
-		logKind = logger.Minio
-	default:
-		logKind = logger.All
+	case ErrSTSInternalError, ErrSTSNotInitialized, ErrSTSUpstreamError:
+		logger.LogIf(ctx, errCtxt, logger.Minio)
 	}
-	logger.LogIf(ctx, errCtxt, logKind)
 	encodedErrorResponse := encodeResponse(stsErrorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }

cmd/xl-storage.go

@@ -94,8 +94,8 @@ type xlStorage struct {
 	endpoint Endpoint
 
 	globalSync bool
-
-	rootDisk bool
+	oDirect    bool // indicates if this disk supports ODirect
+	rootDisk   bool
 
 	diskID string
 
@@ -264,15 +264,27 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		filePath := pathJoin(s.diskPath, ".writable-check-"+uuid+".tmp")
 		w, err := s.openFileDirect(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL)
 		if err != nil {
-			return s, err
+			// relax single drive mode without O_DIRECT support
+			if globalIsErasureSD && errors.Is(err, errUnsupportedDisk) {
+				s.oDirect = false
+			} else {
+				return s, err
+			}
 		}
 		_, err = w.Write(alignedBuf)
 		w.Close()
 		if err != nil {
 			if isSysErrInvalidArg(err) {
-				return s, errUnsupportedDisk
+				err = errUnsupportedDisk
 			}
-			return s, err
+			// relax single drive mode without O_DIRECT support
+			if globalIsErasureSD && errors.Is(err, errUnsupportedDisk) {
+				s.oDirect = false
+			} else {
+				return s, err
+			}
+		} else {
+			s.oDirect = true
 		}
 		renameAll(filePath, pathJoin(s.diskPath, minioMetaTmpDeletedBucket, uuid))
 
@@ -289,6 +301,7 @@ func newXLStorage(ep Endpoint) (s *xlStorage, err error) {
 		s.diskID = format.Erasure.This
 		s.formatLastCheck = time.Now()
 		s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1
+		s.oDirect = true
 	}
 
 	// Success.
@@ -1372,7 +1385,13 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 		return nil, time.Time{}, ctx.Err()
 	}
 
-	f, err := OpenFileDirectIO(filePath, readMode, 0o666)
+	odirectEnabled := !globalAPIConfig.isDisableODirect() && s.oDirect
+	var f *os.File
+	if odirectEnabled {
+		f, err = OpenFileDirectIO(filePath, readMode, 0o666)
+	} else {
+		f, err = OpenFile(filePath, readMode, 0o666)
+	}
 	if err != nil {
 		if osIsNotExist(err) {
 			// Check if the object doesn't exist because its bucket
@@ -1662,7 +1681,14 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		return nil, err
 	}
 
-	file, err := OpenFileDirectIO(filePath, readMode, 0o666)
+	odirectEnabled := !globalAPIConfig.isDisableODirect() && s.oDirect
+
+	var file *os.File
+	if odirectEnabled {
+		file, err = OpenFileDirectIO(filePath, readMode, 0o666)
+	} else {
+		file, err = OpenFile(filePath, readMode, 0o666)
+	}
 	if err != nil {
 		switch {
 		case osIsNotExist(err):
@@ -1706,7 +1732,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	}
 
 	alignment := offset%xioutil.DirectioAlignSize == 0
-	if !alignment || globalAPIConfig.isDisableODirect() {
+	if !alignment && odirectEnabled {
 		if err = disk.DisableDirectIO(file); err != nil {
 			file.Close()
 			return nil, err
@@ -1736,13 +1762,11 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		io.Reader
 		io.Closer
 	}{Reader: io.LimitReader(diskHealthReader(ctx, or), length), Closer: closeWrapper(func() error {
-		if !alignment || offset+length%xioutil.DirectioAlignSize != 0 {
+		if (!alignment || offset+length%xioutil.DirectioAlignSize != 0) && odirectEnabled {
 			// invalidate page-cache for unaligned reads.
-			if !globalAPIConfig.isDisableODirect() {
-				// skip removing from page-cache only
-				// if O_DIRECT was disabled.
-				disk.FadviseDontNeed(file)
-			}
+			// skip removing from page-cache only
+			// if O_DIRECT was disabled.
+			disk.FadviseDontNeed(file)
 		}
 		return or.Close()
 	})}
@@ -1814,7 +1838,13 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		return osErrToFileErr(err)
 	}
 
-	w, err := OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	odirectEnabled := s.oDirect
+	var w *os.File
+	if odirectEnabled {
+		w, err = OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	} else {
+		w, err = OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
+	}
 	if err != nil {
 		return osErrToFileErr(err)
 	}
@@ -1834,7 +1864,12 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		defer xioutil.ODirectPoolLarge.Put(bufp)
 	}
 
-	written, err := xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
+	var written int64
+	if odirectEnabled {
+		written, err = xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
+	} else {
+		written, err = io.CopyBuffer(diskHealthWriter(ctx, w), r, *bufp)
+	}
 	if err != nil {
 		return err
 	}

docs/config/README.md

@@ -2,7 +2,7 @@
 
 ## Configuration Directory
 
-MinIO stores all its config as part of the server deployment, config is erasure coded on MinIO. On a fresh deployment MinIO automatically generates a new `config` and this config is available to be configured via `mc admin config` command. MinIO also encrypts all the config, IAM and policies content if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md)
+MinIO stores all its config as part of the server deployment, config is erasure coded on MinIO. On a fresh deployment MinIO automatically generates a new `config` and this config is available to be configured via `mc admin config` command. MinIO also encrypts all the config, IAM and policies content if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md).
 
 ### Certificate Directory
 
@@ -123,7 +123,7 @@ MINIO_CACHE_COMMENT  (sentence)  optionally add a comment to this setting
 
 #### Etcd
 
-MinIO supports storing encrypted IAM assets in etcd, if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md)
+MinIO supports storing encrypted IAM assets in etcd, if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md).
 
 > NOTE: if *path_prefix* is set then MinIO will not federate your buckets, namespaced IAM assets are assumed as isolated tenants, only buckets are considered globally unique but performing a lookup with a *bucket* which belongs to a different tenant will fail unlike federated setups where MinIO would port-forward and route the request to relevant cluster accordingly. This is a special feature, federated deployments should not need to set *path_prefix*.
 
@@ -181,7 +181,7 @@ MINIO_API_REMOTE_TRANSPORT_DEADLINE  (duration)  set the deadline for API reques
 
 #### Notifications
 
-Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://docs.min.io/docs/minio-bucket-notification-guide.html)
+Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://docs.min.io/docs/minio-bucket-notification-guide.html).
 
 ```
 notify_webhook        publish bucket notifications to webhook endpoints
@@ -244,7 +244,7 @@ MINIO_ETCD_CLIENT_CERT_KEY  (path)      client cert key for mTLS authentication
 MINIO_ETCD_COMMENT          (sentence)  optionally add a comment to this setting
 ```
 
-This behavior is consistent across all keys, each key self documents itself with valid examples.
+This behavior is consistent across all keys; each key self-documents itself with valid examples.
 
 ## Dynamic systems without restarting server
 
@@ -274,7 +274,7 @@ delay     (float)     scanner delay multiplier, defaults to '10.0'
 max_wait  (duration)  maximum wait time between operations, defaults to '15s'
 ```
 
-Example: Following setting will decrease the scanner speed by a factor of 3, reducing the system resource use, but increasing the latency of updates being reflected.
+Example: the following setting will decrease the scanner speed by a factor of 3, reducing the system resource use, but increasing the latency of updates being reflected.
 
 ```sh
 ~ mc admin config set alias/ scanner delay=30.0
@@ -326,7 +326,8 @@ minio server /data
 
 ### Domain
 
-By default, MinIO supports path-style requests that are of the format <http://mydomain.com/bucket/object>. `MINIO_DOMAIN` environment variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the matched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html)
+By default, MinIO supports path-style requests that are of the format <http://mydomain.com/bucket/object>. `MINIO_DOMAIN` environment variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the matched pattern `$1` is used as bucket and the path is used as object. Read more about path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html).
+
 Example:
 
 ```sh

docs/debugging/s3-verify/main.go

@@ -0,0 +1,268 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"flag"
+	"fmt"
+	"hash"
+	"io"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+var (
+	sourceEndpoint, sourceAccessKey, sourceSecretKey string
+	sourceBucket, sourcePrefix                       string
+	targetEndpoint, targetAccessKey, targetSecretKey string
+	targetBucket, targetPrefix                       string
+	debug                                            bool
+	insecure                                         bool
+)
+
+func main() {
+	flag.StringVar(&sourceEndpoint, "source-endpoint", "https://play.min.io", "S3 endpoint URL")
+	flag.StringVar(&sourceAccessKey, "source-access-key", "Q3AM3UQ867SPQQA43P2F", "S3 Access Key")
+	flag.StringVar(&sourceSecretKey, "source-secret-key", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "S3 Secret Key")
+	flag.StringVar(&sourceBucket, "source-bucket", "", "Select a specific bucket")
+	flag.StringVar(&sourcePrefix, "source-prefix", "", "Select a prefix")
+
+	flag.StringVar(&targetEndpoint, "target-endpoint", "https://play.min.io", "S3 endpoint URL")
+	flag.StringVar(&targetAccessKey, "target-access-key", "Q3AM3UQ867SPQQA43P2F", "S3 Access Key")
+	flag.StringVar(&targetSecretKey, "target-secret-key", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", "S3 Secret Key")
+	flag.StringVar(&targetBucket, "target-bucket", "", "Select a specific bucket")
+	flag.StringVar(&targetPrefix, "target-prefix", "", "Select a prefix")
+
+	flag.BoolVar(&debug, "debug", false, "Prints HTTP network calls to S3 endpoint")
+	flag.BoolVar(&insecure, "insecure", false, "Disable TLS verification")
+	flag.Parse()
+
+	if sourceEndpoint == "" {
+		log.Fatalln("source Endpoint is not provided")
+	}
+
+	if sourceAccessKey == "" {
+		log.Fatalln("source Access key is not provided")
+	}
+
+	if sourceSecretKey == "" {
+		log.Fatalln("source Secret key is not provided")
+	}
+
+	if sourceBucket == "" && sourcePrefix != "" {
+		log.Fatalln("--source-prefix is specified without --source-bucket.")
+	}
+
+	if targetEndpoint == "" {
+		log.Fatalln("target Endpoint is not provided")
+	}
+
+	if targetAccessKey == "" {
+		log.Fatalln("target Access key is not provided")
+	}
+
+	if targetSecretKey == "" {
+		log.Fatalln("target Secret key is not provided")
+	}
+
+	if targetBucket == "" && targetPrefix != "" {
+		log.Fatalln("--target-prefix is specified without --target-bucket.")
+	}
+
+	u, err := url.Parse(sourceEndpoint)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	secure := strings.EqualFold(u.Scheme, "https")
+	transport, err := minio.DefaultTransport(secure)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	if insecure {
+		// skip TLS verification
+		transport.TLSClientConfig.InsecureSkipVerify = true
+	}
+
+	sclnt, err := minio.New(u.Host, &minio.Options{
+		Creds:     credentials.NewStaticV4(sourceAccessKey, sourceSecretKey, ""),
+		Secure:    secure,
+		Transport: transport,
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	tclnt, err := minio.New(u.Host, &minio.Options{
+		Creds:     credentials.NewStaticV4(targetAccessKey, targetSecretKey, ""),
+		Secure:    secure,
+		Transport: transport,
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	if debug {
+		sclnt.TraceOn(os.Stderr)
+		tclnt.TraceOn(os.Stderr)
+	}
+
+	sopts := minio.ListObjectsOptions{
+		Recursive: true,
+		Prefix:    sourcePrefix,
+	}
+
+	topts := minio.ListObjectsOptions{
+		Recursive: true,
+		Prefix:    targetPrefix,
+	}
+
+	srcCh := sclnt.ListObjects(context.Background(), sourceBucket, sopts)
+	tgtCh := tclnt.ListObjects(context.Background(), targetBucket, topts)
+
+	srcCtnt, srcOk := <-srcCh
+	tgtCtnt, tgtOk := <-tgtCh
+
+	var srcEOF, tgtEOF bool
+
+	srcSha256 := sha256.New()
+	tgtSha256 := sha256.New()
+
+	for {
+		srcSha256.Reset()
+		tgtSha256.Reset()
+
+		srcEOF = !srcOk
+		tgtEOF = !tgtOk
+
+		// No objects from source AND target: Finish
+		if srcEOF && tgtEOF {
+			break
+		}
+
+		if !srcEOF && srcCtnt.Err != nil {
+			log.Fatal(srcCtnt.Err)
+		}
+
+		if !tgtEOF && tgtCtnt.Err != nil {
+			log.Fatal(tgtCtnt.Err)
+		}
+
+		// If source doesn't have objects anymore, comparison becomes obvious
+		if srcEOF {
+			fmt.Printf("only in target: %s\n", tgtCtnt.Key)
+			tgtCtnt, tgtOk = <-tgtCh
+			continue
+		}
+
+		// The same for target
+		if tgtEOF {
+			fmt.Printf("only in source: %s\n", srcCtnt.Key)
+			srcCtnt, srcOk = <-srcCh
+			continue
+		}
+
+		if srcCtnt.Key == tgtCtnt.Key {
+			if verifyChecksum(sclnt, srcSha256, tgtSha256, srcCtnt, tgtCtnt) {
+				fmt.Printf("all readable source and target: %s -> %s\n", srcCtnt.Key, tgtCtnt.Key)
+			}
+
+			srcCtnt, srcOk = <-srcCh
+			tgtCtnt, tgtOk = <-tgtCh
+			continue
+		}
+
+		fmt.Printf("only in target: %s (%s)\n", tgtCtnt.Key, tgtCtnt.VersionID)
+		tgtCtnt, tgtOk = <-tgtCh
+	}
+}
+
+func verifyChecksum(sclnt *minio.Client, srcSha256, tgtSha256 hash.Hash, srcCtnt, tgtCtnt minio.ObjectInfo) (allgood bool) {
+	opts := minio.GetObjectOptions{}
+	if srcCtnt.Size != tgtCtnt.Size {
+		fmt.Printf("differ in size sourceSize: %d, targetSize: %d\n", srcCtnt.Size, tgtCtnt.Size)
+		return false
+	} else if srcCtnt.ContentType != tgtCtnt.ContentType {
+		fmt.Printf("differ in contentType source: %s, target: %s\n", srcCtnt.ContentType, tgtCtnt.ContentType)
+		return false
+	}
+
+	core := minio.Core{Client: sclnt}
+	sobj, _, _, err := core.GetObject(context.Background(), sourceBucket, srcCtnt.Key, opts)
+	if err != nil {
+		fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
+		return false
+	}
+
+	tobj, _, _, err := core.GetObject(context.Background(), targetBucket, tgtCtnt.Key, opts)
+	if err != nil {
+		fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
+		return false
+	}
+
+	var sourceFailed, targetFailed bool
+	var wg sync.WaitGroup
+
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		srcSize, err := io.Copy(srcSha256, sobj)
+		if err != nil {
+			fmt.Printf("unreadable on source: %s (%s)\n", srcCtnt.Key, err)
+			sourceFailed = true
+		} else if srcSize != srcCtnt.Size {
+			fmt.Printf("unreadable on source - size differs upon read: %s\n", srcCtnt.Key)
+			sourceFailed = true
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		tgtSize, err := io.Copy(tgtSha256, tobj)
+		if err != nil {
+			fmt.Printf("unreadable on target: %s (%s)\n", tgtCtnt.Key, err)
+			targetFailed = true
+		} else if tgtSize != tgtCtnt.Size {
+			fmt.Printf("unreadable on target - size differs upon read: %s\n", tgtCtnt.Key)
+			targetFailed = true
+		}
+	}()
+	wg.Wait()
+
+	sobj.Close()
+	tobj.Close()
+
+	if !sourceFailed && !targetFailed {
+		ssum := srcSha256.Sum(nil)
+		tsum := tgtSha256.Sum(nil)
+		allgood = bytes.Equal(ssum, tsum)
+		if !allgood {
+			fmt.Printf("sha256 sum mismatch: %s -> Expected(%x), Found(%x)\n", srcCtnt.Key, ssum, tsum)
+		}
+	}
+
+	return allgood
+}

docs/gateway/nas.md

@@ -43,7 +43,7 @@ MinIO Gateway comes with an embedded web based object browser. Point your web br
 
 ## Test using MinIO Client `mc`
 
-`mc` provides a modern alternative to UNIX commands such as ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services.
+`mc` provides a modern alternative to UNIX commands such as ls, cat, cp, mirror, diff, etc. It supports filesystems and Amazon S3 compatible cloud storage services.
 
 ### Configure `mc`
 
@@ -88,25 +88,24 @@ export MINIO_NOTIFY_WEBHOOK_ENDPOINT_1=http://localhost:8080/
 export MINIO_NOTIFY_WEBHOOK_QUEUE_DIR_1=/tmp/webhk
 ```
 
-> NOTE: Please check the docs for the corresponding ENV setting. Alternatively, We can obtain other ENVs in the form `mc admin config set alias/ <sub-sys> --env`
+> NOTE: Please check the docs for the corresponding ENV setting. Alternatively, we can obtain other ENVs in the form `mc admin config set alias/ <sub-sys> --env`
 
 ## Symlink support
 
-NAS gateway implementation allows symlinks on regular files,
+NAS gateway implementation allows symlinks on regular files.
 
 ### Behavior
 
-- For reads symlink resolves to file symlink points to.
+- For reads symlinks resolve to the file the symlink points to. 
 - For deletes
-  - Delete of symlink deletes the symlink but not the real file to which the symlink points.
-  - Delete of actual file automatically makes symlink'ed file invisible, dangling symlinks won't be visible.
+  - Deleting a symlink deletes the symlink but not the real file to which the symlink points.
+  - Deleting the real file a symlink points to automatically makes the dangling symlink invisible.
 
 #### Caveats
 
 - Disallows follow of directory symlinks to avoid security issues, and leaving them as is on namespace makes them very inconsistent.
-- Dangling symlinks are ignored automatically.
 
-*Directory symlinks is not and will not be supported as there are no safe ways to handle them.*
+*Directory symlinks are not and will not be supported as there are no safe ways to handle them.*
 
 ## Explore Further
 

docs/metrics/prometheus/grafana/minio-dashboard.json

@@ -45,6 +45,12 @@
       "id": "stat",
       "name": "Stat",
       "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
     }
   ],
   "annotations": {
@@ -72,7 +78,7 @@
   "gnetId": 13502,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1653756543524,
+  "iteration": 1655183014775,
   "links": [
     {
       "icon": "external link",
@@ -266,7 +272,38 @@
           ],
           "unit": "bytes"
         },
-        "overrides": []
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Free"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Used"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "orange",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
       },
       "gridPos": {
         "h": 6,
@@ -327,96 +364,129 @@
       "type": "piechart"
     },
     {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
       "datasource": "${DS_PROMETHEUS}",
-      "fill": 1,
-      "fillGradient": 0,
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Objects"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "blue",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "__systemRef": "hideSeriesFrom",
+            "matcher": {
+              "id": "byNames",
+              "options": {
+                "mode": "exclude",
+                "names": [
+                  "Usage"
+                ],
+                "prefix": "All except:",
+                "readOnly": true
+              }
+            },
+            "properties": [
+              {
+                "id": "custom.hideFrom",
+                "value": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": true
+                }
+              }
+            ]
+          }
+        ]
+      },
       "gridPos": {
         "h": 6,
         "w": 6,
         "x": 10,
         "y": 0
       },
-      "hiddenSeries": false,
       "id": 68,
-      "legend": {
-        "avg": false,
-        "current": true,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "nullPointMode": "null",
       "options": {
-        "alertThreshold": true
+        "legend": {
+          "calcs": [],
+          "displayMode": "hidden",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single"
+        }
       },
-      "percentage": false,
       "pluginVersion": "8.2.1",
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
       "targets": [
         {
           "exemplar": true,
           "expr": "sum(minio_bucket_usage_total_bytes{job=\"$scrape_jobs\"}) by (instance)",
           "interval": "",
-          "legendFormat": "Used Capacity",
+          "legendFormat": "Usage",
           "refId": "A"
         }
       ],
-      "thresholds": [],
       "timeFrom": null,
-      "timeRegions": [],
       "timeShift": null,
       "title": "Data Usage Growth",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:419",
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:420",
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
+      "type": "timeseries"
     },
     {
       "datasource": "${DS_PROMETHEUS}",
@@ -505,7 +575,7 @@
               },
               {
                 "color": "red",
-                "value": 80
+                "value": 2000
               }
             ]
           },
@@ -661,7 +731,7 @@
               },
               {
                 "color": "red",
-                "value": 80
+                "value": 2000
               }
             ]
           },
@@ -1711,6 +1781,198 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {
+        "S3 Errors": "light-red",
+        "S3 Requests": "light-green"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 22
+      },
+      "hiddenSeries": false,
+      "id": 86,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
+          "interval": "1m",
+          "intervalFactor": 2,
+          "legendFormat": "{{server,api}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Request Error Rate (5xx)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:331",
+          "format": "none",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:332",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {
+        "S3 Errors": "light-red",
+        "S3 Requests": "light-green"
+      },
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 22
+      },
+      "hiddenSeries": false,
+      "id": 88,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
+          "interval": "1m",
+          "intervalFactor": 2,
+          "legendFormat": "{{server,api}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "S3 API Request Error Rate (4xx)",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:331",
+          "format": "none",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:332",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
     {
       "aliasColors": {
         "10.13.1.25:9000 DELETE": "red",
@@ -1734,7 +1996,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 22
+        "y": 32
       },
       "hiddenSeries": false,
       "id": 17,
@@ -1839,7 +2101,7 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 22
+        "y": 32
       },
       "hiddenSeries": false,
       "id": 84,
@@ -1947,7 +2209,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 31
+        "y": 41
       },
       "hiddenSeries": false,
       "id": 77,
@@ -2039,7 +2301,7 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 31
+        "y": 41
       },
       "hiddenSeries": false,
       "id": 76,
@@ -2131,7 +2393,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 40
+        "y": 50
       },
       "hiddenSeries": false,
       "id": 74,
@@ -2225,7 +2487,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 40
+        "y": 50
       },
       "hiddenSeries": false,
       "id": 82,
@@ -2330,7 +2592,7 @@
         "h": 9,
         "w": 12,
         "x": 0,
-        "y": 48
+        "y": 58
       },
       "hiddenSeries": false,
       "id": 11,
@@ -2447,7 +2709,7 @@
         "h": 9,
         "w": 12,
         "x": 12,
-        "y": 48
+        "y": 58
       },
       "hiddenSeries": false,
       "id": 8,
@@ -2542,7 +2804,7 @@
         "h": 7,
         "w": 24,
         "x": 0,
-        "y": 57
+        "y": 67
       },
       "hiddenSeries": false,
       "id": 73,
@@ -2632,7 +2894,7 @@
       }
     }
   ],
-  "refresh": false,
+  "refresh": "",
   "schemaVersion": 31,
   "style": "dark",
   "tags": [
@@ -2666,7 +2928,7 @@
     ]
   },
   "time": {
-    "from": "now-3h",
+    "from": "now-6h",
     "to": "now"
   },
   "timepicker": {
@@ -2696,5 +2958,5 @@
   "timezone": "",
   "title": "MinIO Dashboard",
   "uid": "TgmJnqnnk",
-  "version": 7
+  "version": 22
 }

docs/metrics/prometheus/list.md

@@ -15,6 +15,8 @@ These metrics can be from any MinIO server once per collection.
 | `minio_bucket_usage_object_total`            | Total number of objects                                                                                             |
 | `minio_bucket_usage_total_bytes`             | Total bucket size in bytes                                                                                          |
 | `minio_bucket_quota_total_bytes`             | Total bucket quota size in bytes                                                                                    |
+| `minio_bucket_traffic_sent_bytes`            | Total s3 bytes sent per bucket                                                                                      |
+| `minio_bucket_traffic_received_bytes`        | Total s3 bytes received per bucket                                                                                  |
 | `minio_cache_hits_total`                     | Total number of disk cache hits                                                                                     |
 | `minio_cache_missed_total`                   | Total number of disk cache misses                                                                                   |
 | `minio_cache_sent_bytes`                     | Total number of bytes served from cache                                                                             |
@@ -52,7 +54,9 @@ These metrics can be from any MinIO server once per collection.
 | `minio_node_process_uptime_seconds`          | Uptime for MinIO process per node in seconds.                                                                       |
 | `minio_node_syscall_read_total`              | Total read SysCalls to the kernel. /proc/[pid]/io syscr                                                             |
 | `minio_node_syscall_write_total`             | Total write SysCalls to the kernel. /proc/[pid]/io syscw                                                            |
-| `minio_s3_requests_error_total`              | Total number S3 requests with errors                                                                                |
+| `minio_s3_requests_errors_total`             | Total number S3 requests with 4xx and 5xx errors                                                                    |
+| `minio_s3_requests_4xx_errors_total`         | Total number S3 requests with 4xx errors                                                                            |
+| `minio_s3_requests_5xx_errors_total`         | Total number S3 requests with 5xx errors                                                                            |
 | `minio_s3_requests_inflight_total`           | Total number of S3 requests currently in flight                                                                     |
 | `minio_s3_requests_total`                    | Total number S3 requests                                                                                            |
 | `minio_s3_time_ttfb_seconds_distribution`    | Distribution of the time to first byte across API calls.                                                            |

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-06-03T01-40-53Z
+  image: quay.io/minio/minio:RELEASE.2022-06-17T02-00-35Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/sts/assume-role.go

@@ -24,8 +24,10 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/url"
+	"os"
 	"time"
 
 	"github.com/minio/minio-go/v7"
@@ -60,7 +62,7 @@ func init() {
 	flag.BoolVar(&displayCreds, "d", false, "Only show generated credentials")
 	flag.DurationVar(&expiryDuration, "e", 0, "Request a duration of validity for the generated credential")
 	flag.StringVar(&bucketToList, "b", "", "Bucket to list (defaults to username)")
-	// flag.StringVar(&sessionPolicyFile, "s", "", "File containing session policy to apply to the STS request")
+	flag.StringVar(&sessionPolicyFile, "s", "", "File containing session policy to apply to the STS request")
 }
 
 func main() {
@@ -77,21 +79,20 @@ func main() {
 	var stsOpts cr.STSAssumeRoleOptions
 	stsOpts.AccessKey = minioUsername
 	stsOpts.SecretKey = minioPassword
-	// FIXME: add support for passing this in minio-go
-	// if sessionPolicyFile != "" {
-	// 	var policy string
-	// 	if f, err := os.Open(sessionPolicyFile); err != nil {
-	// 		log.Fatalf("Unable to open session policy file: %v", sessionPolicyFile, err)
-	// 	} else {
-	// 		bs, err := ioutil.ReadAll(f)
-	// 		if err != nil {
-	// 			log.Fatalf("Error reading session policy file: %v", err)
-	// 		}
-	// 		policy = string(bs)
-	// 	}
-	// 	opts
-	// 	ldapOpts = append(ldapOpts, cr.LDAPIdentityPolicyOpt(policy))
-	// }
+
+	if sessionPolicyFile != "" {
+		var policy string
+		if f, err := os.Open(sessionPolicyFile); err != nil {
+			log.Fatalf("Unable to open session policy file: %v", err)
+		} else {
+			bs, err := ioutil.ReadAll(f)
+			if err != nil {
+				log.Fatalf("Error reading session policy file: %v", err)
+			}
+			policy = string(bs)
+		}
+		stsOpts.Policy = policy
+	}
 	if expiryDuration != 0 {
 		stsOpts.DurationSeconds = int(expiryDuration.Seconds())
 	}
@@ -126,7 +127,7 @@ func main() {
 	// Use generated credentials to authenticate with MinIO server
 	minioClient, err := minio.New(stsEndpointURL.Host, opts)
 	if err != nil {
-		log.Fatalf("Error initializing client: ", err)
+		log.Fatalf("Error initializing client: %v", err)
 	}
 
 	// Use minIO Client object normally like the regular client.

go.mod

@@ -43,21 +43,20 @@ require (
 	github.com/lib/pq v1.10.4
 	github.com/miekg/dns v1.1.48
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.18.0
+	github.com/minio/console v0.19.0
 	github.com/minio/csvparser v1.0.0
-	github.com/minio/dperf v0.4.0
+	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.19.2
 	github.com/minio/madmin-go v1.3.14
-	github.com/minio/minio-go/v7 v7.0.27
-	github.com/minio/pkg v1.1.24
+	github.com/minio/minio-go/v7 v7.0.29
+	github.com/minio/pkg v1.1.25
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.4.2
 	github.com/minio/sio v0.3.0
 	github.com/minio/zipindex v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/montanaflynn/stats v0.6.6
 	github.com/nats-io/nats-server/v2 v2.7.4
 	github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d
 	github.com/nats-io/stan.go v0.10.2
@@ -72,7 +71,7 @@ require (
 	github.com/rs/cors v1.7.0
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/secure-io/sio-go v0.3.1
-	github.com/shirou/gopsutil/v3 v3.22.2
+	github.com/shirou/gopsutil/v3 v3.22.5
 	github.com/streadway/amqp v1.0.0
 	github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e
 	github.com/valyala/bytebufferpool v1.0.0
@@ -170,6 +169,7 @@ require (
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
 	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
 	github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect

go.sum

@@ -611,12 +611,12 @@ github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.2 h1:Em3IM68MTm3h+Oxa0nxrV9VQqDgbxvC5iq5A+pqzDeI=
 github.com/minio/colorjson v1.0.2/go.mod h1:JWxcL2n8T8JVf+NY6awl6kn5nK49aAzHOeQEM33dL0k=
-github.com/minio/console v0.18.0 h1:6SDDYDBqwGPhw930yTSlYWWLoH9Qj/O1mKTXBG51MhM=
-github.com/minio/console v0.18.0/go.mod h1:yfF8zcgJtD+KPPFHHD1PMpFbhMnQRY8C7jmsrHBdoRU=
+github.com/minio/console v0.19.0 h1:GorxVfHaXBR8eKapVupRpvI53UB7/f5p/hf6NKI8dJ4=
+github.com/minio/console v0.19.0/go.mod h1:yfF8zcgJtD+KPPFHHD1PMpFbhMnQRY8C7jmsrHBdoRU=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
-github.com/minio/dperf v0.4.0 h1:iUiHOO96S/hcRzIaqWao6NJPkbuNQTEndrq3ht+vXOQ=
-github.com/minio/dperf v0.4.0/go.mod h1:mo7rPmlIl3Bgn8CrM0Yjeu3AL3xXVd7sH7Hpj4OGK1E=
+github.com/minio/dperf v0.4.2 h1:u2fcdXViX3+alZcEYSOh1crNbGtVU5drQ24lP3Xx6j8=
+github.com/minio/dperf v0.4.2/go.mod h1:mo7rPmlIl3Bgn8CrM0Yjeu3AL3xXVd7sH7Hpj4OGK1E=
 github.com/minio/filepath v1.0.0 h1:fvkJu1+6X+ECRA6G3+JJETj4QeAYO9sV43I79H8ubDY=
 github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEXx9T/Bw=
 github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
@@ -633,11 +633,11 @@ github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77Z
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.23/go.mod h1:ei5JjmxwHaMrgsMrn4U/+Nmg+d8MKS1U2DAn1ou4+Do=
-github.com/minio/minio-go/v7 v7.0.27 h1:yJCvm78B+2+ll1PqO9eSD1as6Ibw3IYnnD8PyBEB2zo=
-github.com/minio/minio-go/v7 v7.0.27/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg=
+github.com/minio/minio-go/v7 v7.0.29 h1:7md6lIq1s6zPzUiDRX1BVLHolA4pDM8RMQqIszaJbY0=
+github.com/minio/minio-go/v7 v7.0.29/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg=
 github.com/minio/pkg v1.1.20/go.mod h1:Xo7LQshlxGa9shKwJ7NzQbgW4s8T/Wc1cOStR/eUiMY=
-github.com/minio/pkg v1.1.24 h1:a2RCb6LgsCi9DvrripuvlFQRCNb5Hp1HIssnsUqLoZY=
-github.com/minio/pkg v1.1.24/go.mod h1:z9PfmEI804KFkF6eY4LoGe8IDVvTCsYGVuaf58Dr0WI=
+github.com/minio/pkg v1.1.25 h1:QYLzmTFUV5D3bY9qXKzDj7eW2C+HOPcdtIZft9q2Azo=
+github.com/minio/pkg v1.1.25/go.mod h1:z9PfmEI804KFkF6eY4LoGe8IDVvTCsYGVuaf58Dr0WI=
 github.com/minio/selfupdate v0.4.0 h1:A7t07pN4Ch1tBTIRStW0KhUVyykz+2muCqFsITQeEW8=
 github.com/minio/selfupdate v0.4.0/go.mod h1:mcDkzMgq8PRcpCRJo/NlPY7U45O5dfYl2Y0Rg7IustY=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -805,8 +805,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc=
 github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs=
 github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88=
-github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks=
-github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY=
+github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA=
+github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA=
 github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -848,6 +848,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w=
 github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -861,11 +862,9 @@ github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk1
 github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e h1:P5tyWbssToKowBPTA1/EzqPXwrZNc8ZeNPdjgpcDEoI=
 github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e/go.mod h1:g7jEyb18KPe65d9RRhGw+ThaJr5duyBH8eaFgBUor7Y=
 github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
-github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
 github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
 github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
 github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM=
-github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
 github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
 github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
@@ -1158,7 +1157,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/color/color.go

@@ -73,6 +73,13 @@ var (
 		return fmt.Sprint
 	}()
 
+	Greenf = func() func(format string, a ...interface{}) string {
+		if IsTerminal() {
+			return color.New(color.FgGreen).SprintfFunc()
+		}
+		return fmt.Sprintf
+	}()
+
 	GreenBold = func() func(a ...interface{}) string {
 		if IsTerminal() {
 			return color.New(color.FgGreen, color.Bold).SprintFunc()

internal/config/config.go

@@ -940,3 +940,197 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err e
 	c[subSys][tgt] = currKVS
 	return dynamic, nil
 }
+
+// CheckValidKeys - checks if the config parameters for the given subsystem and
+// target are valid. It checks both the configuration store as well as
+// environment variables.
+func (c Config) CheckValidKeys(subSys string, deprecatedKeys []string) error {
+	defKVS, ok := DefaultKVS[subSys]
+	if !ok {
+		return fmt.Errorf("Subsystem %s does not exist", subSys)
+	}
+
+	// Make a list of valid keys for the subsystem including the `comment`
+	// key.
+	validKeys := make([]string, 0, len(defKVS)+1)
+	for _, param := range defKVS {
+		validKeys = append(validKeys, param.Key)
+	}
+	validKeys = append(validKeys, Comment)
+
+	subSysEnvVars := env.List(fmt.Sprintf("%s%s", EnvPrefix, strings.ToUpper(subSys)))
+
+	// Set of env vars for the sub-system to validate.
+	candidates := set.CreateStringSet(subSysEnvVars...)
+
+	// Remove all default target env vars from the candidates set (as they
+	// are valid).
+	for _, param := range validKeys {
+		paramEnvName := getEnvVarName(subSys, Default, param)
+		candidates.Remove(paramEnvName)
+	}
+
+	isSingleTarget := SubSystemsSingleTargets.Contains(subSys)
+	if isSingleTarget && len(candidates) > 0 {
+		return fmt.Errorf("The following environment variables are unknown: %s",
+			strings.Join(candidates.ToSlice(), ", "))
+	}
+
+	if !isSingleTarget {
+		// Validate other env vars for all targets.
+		envVars := candidates.ToSlice()
+		for _, envVar := range envVars {
+			for _, param := range validKeys {
+				pEnvName := getEnvVarName(subSys, Default, param) + Default
+				if len(envVar) > len(pEnvName) && strings.HasPrefix(envVar, pEnvName) {
+					// This envVar is valid - it has a
+					// non-empty target.
+					candidates.Remove(envVar)
+				}
+			}
+		}
+
+		// Whatever remains are invalid env vars - return an error.
+		if len(candidates) > 0 {
+			return fmt.Errorf("The following environment variables are unknown: %s",
+				strings.Join(candidates.ToSlice(), ", "))
+		}
+	}
+
+	validKeysSet := set.CreateStringSet(validKeys...)
+	validKeysSet = validKeysSet.Difference(set.CreateStringSet(deprecatedKeys...))
+	kvsMap := c[subSys]
+	for tgt, kvs := range kvsMap {
+		invalidKV := KVS{}
+		for _, kv := range kvs {
+			if !validKeysSet.Contains(kv.Key) {
+				invalidKV = append(invalidKV, kv)
+			}
+		}
+		if len(invalidKV) > 0 {
+			return Errorf(
+				"found invalid keys (%s) for '%s:%s' sub-system, use 'mc admin config reset myminio %s:%s' to fix invalid keys",
+				invalidKV.String(), subSys, tgt, subSys, tgt)
+		}
+	}
+	return nil
+}
+
+// GetAvailableTargets - returns a list of targets configured for the given
+// subsystem (whether they are enabled or not). A target could be configured via
+// environment variables or via the configuration store. The default target is
+// `_` and is always returned.
+func (c Config) GetAvailableTargets(subSys string) ([]string, error) {
+	if SubSystemsSingleTargets.Contains(subSys) {
+		return []string{Default}, nil
+	}
+
+	defKVS, ok := DefaultKVS[subSys]
+	if !ok {
+		return nil, fmt.Errorf("Subsystem %s does not exist", subSys)
+	}
+
+	kvsMap := c[subSys]
+	s := set.NewStringSet()
+
+	// Add all targets that are configured in the config store.
+	for k := range kvsMap {
+		s.Add(k)
+	}
+
+	// Add targets that are configured via environment variables.
+	for _, param := range defKVS {
+		envVarPrefix := getEnvVarName(subSys, Default, param.Key) + Default
+		envsWithPrefix := env.List(envVarPrefix)
+		for _, k := range envsWithPrefix {
+			tgtName := strings.TrimPrefix(k, envVarPrefix)
+			if tgtName != "" {
+				s.Add(tgtName)
+			}
+		}
+	}
+
+	return s.ToSlice(), nil
+}
+
+func getEnvVarName(subSys, target, param string) string {
+	if target == Default {
+		return fmt.Sprintf("%s%s_%s", EnvPrefix, strings.ToUpper(subSys), strings.ToUpper(param))
+	}
+
+	return fmt.Sprintf("%s%s_%s_%s", EnvPrefix, strings.ToUpper(subSys), strings.ToUpper(param), target)
+}
+
+var resolvableSubsystems = set.CreateStringSet(IdentityOpenIDSubSys)
+
+// ValueSource represents the source of a config parameter value.
+type ValueSource uint8
+
+// Constants for ValueSource
+const (
+	ValueSourceAbsent ValueSource = iota // this is an error case
+	ValueSourceDef
+	ValueSourceCfg
+	ValueSourceEnv
+)
+
+// ResolveConfigParam returns the effective value of a configuration parameter,
+// within a subsystem and subsystem target. The effective value is, in order of
+// decreasing precedence:
+//
+// 1. the value of the corresponding environment variable if set,
+// 2. the value of the parameter in the config store if set,
+// 3. the default value,
+//
+// This function only works for a subset of sub-systems, others return
+// `ValueSourceAbsent`. FIXME: some parameters have custom environment
+// variables for which support needs to be added.
+func (c Config) ResolveConfigParam(subSys, target, cfgParam string) (value string, cs ValueSource) {
+	// cs = ValueSourceAbsent initially as it is iota by default.
+
+	// Initially only support OpenID
+	if !resolvableSubsystems.Contains(subSys) {
+		return
+	}
+
+	// Check if config param requested is valid.
+	defKVS, ok := DefaultKVS[subSys]
+	if !ok {
+		return
+	}
+
+	defValue, isFound := defKVS.Lookup(cfgParam)
+	if !isFound {
+		return
+	}
+
+	if target == "" {
+		target = Default
+	}
+
+	envVar := getEnvVarName(subSys, target, cfgParam)
+
+	// Lookup Env var.
+	value = env.Get(envVar, "")
+	if value != "" {
+		cs = ValueSourceEnv
+		return
+	}
+
+	// Lookup config store.
+	if subSysStore, ok := c[subSys]; ok {
+		if kvs, ok2 := subSysStore[target]; ok2 {
+			var ok3 bool
+			value, ok3 = kvs.Lookup(cfgParam)
+			if ok3 {
+				cs = ValueSourceCfg
+				return
+			}
+		}
+	}
+
+	// Return the default value.
+	value = defValue
+	cs = ValueSourceDef
+	return
+}

internal/config/constants.go

@@ -48,6 +48,7 @@ const (
 	EnvPublicIPs  = "MINIO_PUBLIC_IPS"
 	EnvFSOSync    = "MINIO_FS_OSYNC"
 	EnvArgs       = "MINIO_ARGS"
+	EnvVolumes    = "MINIO_VOLUMES"
 	EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
 	EnvSiteName   = "MINIO_SITE_NAME"

internal/config/identity/openid/jwt_test.go

@@ -243,7 +243,7 @@ func TestKeycloakProviderInitialization(t *testing.T) {
 	testKvs.Set(Vendor, "keycloak")
 	testKvs.Set(KeyCloakRealm, "TestRealm")
 	testKvs.Set(KeyCloakAdminURL, "http://keycloak.test/auth/admin")
-	cfgGet := func(env, param string) string {
+	cfgGet := func(param string) string {
 		return testKvs.Get(param)
 	}
 

internal/config/identity/openid/openid.go

@@ -36,49 +36,32 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/identity/openid/provider"
 	"github.com/minio/minio/internal/hash/sha256"
-	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
 )
 
 // OpenID keys and envs.
 const (
-	JwksURL       = "jwks_url"
+	ClientID      = "client_id"
+	ClientSecret  = "client_secret"
 	ConfigURL     = "config_url"
 	ClaimName     = "claim_name"
 	ClaimUserinfo = "claim_userinfo"
-	ClaimPrefix   = "claim_prefix"
-	ClientID      = "client_id"
-	ClientSecret  = "client_secret"
 	RolePolicy    = "role_policy"
 	DisplayName   = "display_name"
 
-	Vendor             = "vendor"
 	Scopes             = "scopes"
 	RedirectURI        = "redirect_uri"
 	RedirectURIDynamic = "redirect_uri_dynamic"
+	Vendor             = "vendor"
 
 	// Vendor specific ENV only enabled if the Vendor matches == "vendor"
 	KeyCloakRealm    = "keycloak_realm"
 	KeyCloakAdminURL = "keycloak_admin_url"
 
-	EnvIdentityOpenIDEnable             = "MINIO_IDENTITY_OPENID_ENABLE"
-	EnvIdentityOpenIDVendor             = "MINIO_IDENTITY_OPENID_VENDOR"
-	EnvIdentityOpenIDClientID           = "MINIO_IDENTITY_OPENID_CLIENT_ID"
-	EnvIdentityOpenIDClientSecret       = "MINIO_IDENTITY_OPENID_CLIENT_SECRET"
-	EnvIdentityOpenIDURL                = "MINIO_IDENTITY_OPENID_CONFIG_URL"
-	EnvIdentityOpenIDClaimName          = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
-	EnvIdentityOpenIDClaimUserInfo      = "MINIO_IDENTITY_OPENID_CLAIM_USERINFO"
-	EnvIdentityOpenIDClaimPrefix        = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
-	EnvIdentityOpenIDRolePolicy         = "MINIO_IDENTITY_OPENID_ROLE_POLICY"
-	EnvIdentityOpenIDRedirectURI        = "MINIO_IDENTITY_OPENID_REDIRECT_URI"
-	EnvIdentityOpenIDRedirectURIDynamic = "MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC"
-	EnvIdentityOpenIDScopes             = "MINIO_IDENTITY_OPENID_SCOPES"
-	EnvIdentityOpenIDDisplayName        = "MINIO_IDENTITY_OPENID_DISPLAY_NAME"
-
-	// Vendor specific ENVs only enabled if the Vendor matches == "vendor"
-	EnvIdentityOpenIDKeyCloakRealm    = "MINIO_IDENTITY_OPENID_KEYCLOAK_REALM"
-	EnvIdentityOpenIDKeyCloakAdminURL = "MINIO_IDENTITY_OPENID_KEYCLOAK_ADMIN_URL"
+	// Removed params
+	JwksURL     = "jwks_url"
+	ClaimPrefix = "claim_prefix"
 )
 
 // DefaultKVS - default config for OpenID config
@@ -191,7 +174,7 @@ func (r *Config) Clone() Config {
 }
 
 // LookupConfig lookup jwks from config, override with any ENVs.
-func LookupConfig(kvsMap map[string]config.KVS, transport http.RoundTripper, closeRespFn func(io.ReadCloser), serverRegion string) (c Config, err error) {
+func LookupConfig(s config.Config, transport http.RoundTripper, closeRespFn func(io.ReadCloser), serverRegion string) (c Config, err error) {
 	openIDClientTransport := http.DefaultTransport
 	if transport != nil {
 		openIDClientTransport = transport
@@ -209,48 +192,28 @@ func LookupConfig(kvsMap map[string]config.KVS, transport http.RoundTripper, clo
 		closeRespFn:      closeRespFn,
 	}
 
-	// Make a copy of the config we received so we can mutate it safely.
-	kvsMap2 := make(map[string]config.KVS, len(kvsMap))
-	for k, v := range kvsMap {
-		kvsMap2[k] = v
-	}
-
-	// Add in each configuration name found from environment variables, i.e.
-	// if we see MINIO_IDENTITY_OPENID_CONFIG_URL_2, we add the key "2" to
-	// `kvsMap2` if it does not already exist.
-	envs := env.List(EnvIdentityOpenIDURL + config.Default)
-	for _, k := range envs {
-		cfgName := strings.TrimPrefix(k, EnvIdentityOpenIDURL+config.Default)
-		if cfgName == "" {
-			return c, config.Errorf("Environment variable must have a non-empty config name: %s", k)
-		}
-
-		// It is possible that some variables were specified via config
-		// commands and some variables are intended to be overridden
-		// from the environment, so we ensure that the key is not
-		// overwritten in `kvsMap2` as it may have existing config.
-		if _, ok := kvsMap2[cfgName]; !ok {
-			kvsMap2[cfgName] = DefaultKVS
-		}
-	}
-
 	var (
 		hasLegacyPolicyMapping = false
 		seenClientIDs          = set.NewStringSet()
 	)
-	for cfgName, kvs := range kvsMap2 {
-		// remove this since we have removed support for this already.
-		kvs.Delete(JwksURL)
 
-		if err = config.CheckValidKeys(config.IdentityOpenIDSubSys, kvs, DefaultKVS); err != nil {
-			return c, err
-		}
+	// remove this since we have removed support for this already.
+	deprecatedKeys := []string{JwksURL}
+	if err := s.CheckValidKeys(config.IdentityOpenIDSubSys, deprecatedKeys); err != nil {
+		return c, err
+	}
 
-		getCfgVal := func(envVar, cfgParam string) string {
-			if cfgName != config.Default {
-				envVar += config.Default + cfgName
-			}
-			return env.Get(envVar, kvs.Get(cfgParam))
+	openIDTargets, err := s.GetAvailableTargets(config.IdentityOpenIDSubSys)
+	if err != nil {
+		return c, err
+	}
+
+	for _, cfgName := range openIDTargets {
+		getCfgVal := func(cfgParam string) string {
+			// As parameters are already validated, we skip checking
+			// if the config param was found.
+			val, _ := s.ResolveConfigParam(config.IdentityOpenIDSubSys, cfgName, cfgParam)
+			return val
 		}
 
 		// In the past, when only one openID provider was allowed, there
@@ -261,7 +224,7 @@ func LookupConfig(kvsMap map[string]config.KVS, transport http.RoundTripper, clo
 		// setting, otherwise we treat it as enabled if some important
 		// parameters are non-empty.
 		var (
-			cfgEnableVal        = getCfgVal(EnvIdentityOpenIDEnable, config.Enable)
+			cfgEnableVal        = getCfgVal(config.Enable)
 			isExplicitlyEnabled = false
 		)
 		if cfgEnableVal != "" {
@@ -281,7 +244,7 @@ func LookupConfig(kvsMap map[string]config.KVS, transport http.RoundTripper, clo
 		}
 
 		p := newProviderCfgFromConfig(getCfgVal)
-		configURL := getCfgVal(EnvIdentityOpenIDURL, ConfigURL)
+		configURL := getCfgVal(ConfigURL)
 
 		if !isExplicitlyEnabled {
 			enabled = true
@@ -315,7 +278,7 @@ func LookupConfig(kvsMap map[string]config.KVS, transport http.RoundTripper, clo
 			return c, errors.New("please specify config_url to enable fetching claims from UserInfo endpoint")
 		}
 
-		if scopeList := getCfgVal(EnvIdentityOpenIDScopes, Scopes); scopeList != "" {
+		if scopeList := getCfgVal(Scopes); scopeList != "" {
 			var scopes []string
 			for _, scope := range strings.Split(scopeList, ",") {
 				scope = strings.TrimSpace(scope)

internal/config/identity/openid/providercfg.go

@@ -52,17 +52,17 @@ type providerCfg struct {
 	provider provider.Provider
 }
 
-func newProviderCfgFromConfig(getCfgVal func(env, cfgName string) string) providerCfg {
+func newProviderCfgFromConfig(getCfgVal func(cfgName string) string) providerCfg {
 	return providerCfg{
-		DisplayName:        getCfgVal(EnvIdentityOpenIDDisplayName, DisplayName),
-		ClaimName:          getCfgVal(EnvIdentityOpenIDClaimName, ClaimName),
-		ClaimUserinfo:      getCfgVal(EnvIdentityOpenIDClaimUserInfo, ClaimUserinfo) == config.EnableOn,
-		ClaimPrefix:        getCfgVal(EnvIdentityOpenIDClaimPrefix, ClaimPrefix),
-		RedirectURI:        getCfgVal(EnvIdentityOpenIDRedirectURI, RedirectURI),
-		RedirectURIDynamic: getCfgVal(EnvIdentityOpenIDRedirectURIDynamic, RedirectURIDynamic) == config.EnableOn,
-		ClientID:           getCfgVal(EnvIdentityOpenIDClientID, ClientID),
-		ClientSecret:       getCfgVal(EnvIdentityOpenIDClientSecret, ClientSecret),
-		RolePolicy:         getCfgVal(EnvIdentityOpenIDRolePolicy, RolePolicy),
+		DisplayName:        getCfgVal(DisplayName),
+		ClaimName:          getCfgVal(ClaimName),
+		ClaimUserinfo:      getCfgVal(ClaimUserinfo) == config.EnableOn,
+		ClaimPrefix:        getCfgVal(ClaimPrefix),
+		RedirectURI:        getCfgVal(RedirectURI),
+		RedirectURIDynamic: getCfgVal(RedirectURIDynamic) == config.EnableOn,
+		ClientID:           getCfgVal(ClientID),
+		ClientSecret:       getCfgVal(ClientSecret),
+		RolePolicy:         getCfgVal(RolePolicy),
 	}
 }
 
@@ -72,16 +72,16 @@ const (
 
 // initializeProvider initializes if any additional vendor specific information
 // was provided, initialization will return an error initial login fails.
-func (p *providerCfg) initializeProvider(cfgGet func(string, string) string, transport http.RoundTripper) error {
-	vendor := cfgGet(EnvIdentityOpenIDVendor, Vendor)
+func (p *providerCfg) initializeProvider(cfgGet func(string) string, transport http.RoundTripper) error {
+	vendor := cfgGet(Vendor)
 	if vendor == "" {
 		return nil
 	}
 	var err error
 	switch vendor {
 	case keyCloakVendor:
-		adminURL := cfgGet(EnvIdentityOpenIDKeyCloakAdminURL, KeyCloakAdminURL)
-		realm := cfgGet(EnvIdentityOpenIDKeyCloakRealm, KeyCloakRealm)
+		adminURL := cfgGet(KeyCloakAdminURL)
+		realm := cfgGet(KeyCloakRealm)
 		p.provider, err = provider.KeyCloak(
 			provider.WithAdminURL(adminURL),
 			provider.WithOpenIDConfig(provider.DiscoveryDoc(p.DiscoveryDoc)),

internal/disk/health.go

@@ -1,145 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package disk
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"time"
-
-	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go"
-	"github.com/montanaflynn/stats"
-)
-
-// GetHealthInfo about the drive
-func GetHealthInfo(ctx context.Context, drive, fsPath string) (madmin.DiskLatency, madmin.DiskThroughput, error) {
-	// Create a file with O_DIRECT flag, choose default umask and also make sure
-	// we are exclusively writing to a new file using O_EXCL.
-	w, err := OpenFileDirectIO(fsPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0o666)
-	if err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-
-	defer func() {
-		w.Close()
-		os.Remove(fsPath)
-	}()
-
-	blockSize := 4 * humanize.MiByte
-	fileSize := 256 * humanize.MiByte
-
-	latencies := make([]float64, fileSize/blockSize)
-	throughputs := make([]float64, fileSize/blockSize)
-
-	data := AlignedBlock(blockSize)
-
-	for i := 0; i < (fileSize / blockSize); i++ {
-		if ctx.Err() != nil {
-			return madmin.DiskLatency{}, madmin.DiskThroughput{}, ctx.Err()
-		}
-		startTime := time.Now()
-		if n, err := w.Write(data); err != nil {
-			return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-		} else if n != blockSize {
-			return madmin.DiskLatency{}, madmin.DiskThroughput{}, fmt.Errorf("Expected to write %d, but only wrote %d", blockSize, n)
-		}
-		latencyInSecs := time.Since(startTime).Seconds()
-		latencies[i] = latencyInSecs
-	}
-
-	// Sync every full writes fdatasync
-	Fdatasync(w)
-
-	for i := range latencies {
-		throughput := float64(blockSize) / latencies[i]
-		throughputs[i] = throughput
-	}
-
-	var avgLatency float64
-	var percentile50Latency float64
-	var percentile90Latency float64
-	var percentile99Latency float64
-	var minLatency float64
-	var maxLatency float64
-
-	var avgThroughput float64
-	var percentile50Throughput float64
-	var percentile90Throughput float64
-	var percentile99Throughput float64
-	var minThroughput float64
-	var maxThroughput float64
-
-	if avgLatency, err = stats.Mean(latencies); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if percentile50Latency, err = stats.Percentile(latencies, 50); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if percentile90Latency, err = stats.Percentile(latencies, 90); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if percentile99Latency, err = stats.Percentile(latencies, 99); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if maxLatency, err = stats.Max(latencies); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if minLatency, err = stats.Min(latencies); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	l := madmin.DiskLatency{
-		Avg:          avgLatency,
-		Percentile50: percentile50Latency,
-		Percentile90: percentile90Latency,
-		Percentile99: percentile99Latency,
-		Min:          minLatency,
-		Max:          maxLatency,
-	}
-
-	if avgThroughput, err = stats.Mean(throughputs); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if percentile50Throughput, err = stats.Percentile(throughputs, 50); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if percentile90Throughput, err = stats.Percentile(throughputs, 90); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if percentile99Throughput, err = stats.Percentile(throughputs, 99); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if maxThroughput, err = stats.Max(throughputs); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-	if minThroughput, err = stats.Min(throughputs); err != nil {
-		return madmin.DiskLatency{}, madmin.DiskThroughput{}, err
-	}
-
-	t := madmin.DiskThroughput{
-		Avg:          avgThroughput,
-		Percentile50: percentile50Throughput,
-		Percentile90: percentile90Throughput,
-		Percentile99: percentile99Throughput,
-		Min:          minThroughput,
-		Max:          maxThroughput,
-	}
-
-	return l, t, nil
-}