fix: dex getting started guide URL (#10701)

Before this commit, the crawler ILM will always send object delete event
notification though this is wrong.

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14-alpine as builder
+FROM golang:1.15-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

README.md

@@ -88,7 +88,7 @@ service minio start
 ```
 
 ## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.14](https://golang.org/dl/#stable)
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio

cmd/admin-handlers.go

@@ -35,7 +35,6 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -1425,6 +1424,31 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 
 }
 
+// BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
+// ----------
+// Get bandwidth consumption information
+func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "BandwidthMonitor")
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.BandwidthMonitorAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	setEventStreamHeaders(w)
+	bucketsRequestedString := r.URL.Query().Get("buckets")
+	bucketsRequested := strings.Split(bucketsRequestedString, ",")
+	consolidatedReport := globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
+	enc := json.NewEncoder(w)
+	err := enc.Encode(consolidatedReport)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+	}
+	w.(http.Flusher).Flush()
+}
+
 // ServerInfoHandler - GET /minio/admin/v3/info
 // ----------
 // Get server information

cmd/admin-handlers_test.go

@@ -105,8 +105,8 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer := &erasureZones{zones: make([]*erasureSets, 1)}
-	objLayer.zones[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
+	objLayer := &erasureServerSets{serverSets: make([]*erasureSets, 1)}
+	objLayer.serverSets[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}

cmd/admin-router.go

@@ -214,6 +214,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			// -- OBD API --
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
 				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
+				HandlerFunc(httpTraceHdrs(adminAPI.BandwidthMonitorHandler))
 		}
 	}
 

cmd/admin-server-info.go

@@ -24,13 +24,13 @@ import (
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerSets EndpointServerSets, r *http.Request) madmin.ServerProperties {
 	addr := r.Host
 	if globalIsDistErasure {
-		addr = GetLocalPeer(endpointZones)
+		addr = GetLocalPeer(endpointServerSets)
 	}
 	network := make(map[string]string)
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			nodeName := endpoint.Host
 			if nodeName == "" {

cmd/auth-handler.go

@@ -334,7 +334,8 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	if cred.AccessKey == "" {
+	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
+		// Anonymous checks are not meant for ListBuckets action
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Action:          action,
@@ -378,12 +379,13 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Request is allowed return the appropriate access key.
 		return cred.AccessKey, owner, ErrNone
 	}
+
 	if action == policy.ListBucketVersionsAction {
 		// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
 		// verify as a fallback.
 		if globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          iampolicy.Action(policy.ListBucketAction),
+			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			ObjectName:      objectName,
@@ -554,7 +556,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		if retMode == objectlock.RetGovernance && byPassSet {
 			byPassSet = globalPolicySys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
-				Action:          policy.Action(policy.BypassGovernanceRetentionAction),
+				Action:          policy.BypassGovernanceRetentionAction,
 				BucketName:      bucketName,
 				ConditionValues: conditions,
 				IsOwner:         false,
@@ -563,7 +565,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		}
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.Action(policy.PutObjectRetentionAction),
+			Action:          policy.PutObjectRetentionAction,
 			BucketName:      bucketName,
 			ConditionValues: conditions,
 			IsOwner:         false,
@@ -586,7 +588,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	if retMode == objectlock.RetGovernance && byPassSet {
 		byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.BypassGovernanceRetentionAction,
+			Action:          iampolicy.BypassGovernanceRetentionAction,
 			BucketName:      bucketName,
 			ObjectName:      objectName,
 			ConditionValues: conditions,
@@ -596,7 +598,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	}
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
-		Action:          policy.PutObjectRetentionAction,
+		Action:          iampolicy.PutObjectRetentionAction,
 		BucketName:      bucketName,
 		ConditionValues: conditions,
 		ObjectName:      objectName,

cmd/background-newdisks-heal-ops.go

@@ -39,7 +39,7 @@ type healingTracker struct {
 }
 
 func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	z, ok := objAPI.(*erasureZones)
+	z, ok := objAPI.(*erasureServerSets)
 	if !ok {
 		return
 	}
@@ -107,7 +107,7 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healSequence) {
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		select {
@@ -129,8 +129,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
 
-				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.zones))
-				for i := range z.zones {
+				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverSets))
+				for i := range z.serverSets {
 					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
 				}
 			}
@@ -149,7 +149,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				}
 
 				// Calculate the set index where the current endpoint belongs
-				setIndex, _, err := findDiskIndex(z.zones[zoneIdx].format, format)
+				setIndex, _, err := findDiskIndex(z.serverSets[zoneIdx].format, format)
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					continue
@@ -164,7 +164,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 					for _, disk := range disks {
 						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
 
-						lbDisks := z.zones[i].sets[setIndex].getLoadBalancedNDisks(z.zones[i].listTolerancePerSet)
+						lbDisks := z.serverSets[i].sets[setIndex].getOnlineDisks()
 						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
 							logger.LogIf(ctx, err)
 							continue

cmd/bootstrap-peer-server.go

@@ -54,7 +54,7 @@ type bootstrapRESTServer struct{}
 type ServerSystemConfig struct {
 	MinioPlatform  string
 	MinioRuntime   string
-	MinioEndpoints EndpointZones
+	MinioEndpoints EndpointServerSets
 }
 
 // Diff - returns error on first difference found in two configs.
@@ -161,9 +161,9 @@ func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSyst
 	return srcCfg.Diff(recvCfg)
 }
 
-func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones) error {
+func verifyServerSystemConfig(ctx context.Context, endpointServerSets EndpointServerSets) error {
 	srcCfg := getServerSystemCfg()
-	clnts := newBootstrapRESTClients(endpointZones)
+	clnts := newBootstrapRESTClients(endpointServerSets)
 	var onlineServers int
 	var offlineEndpoints []string
 	var retries int
@@ -198,10 +198,10 @@ func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones)
 	return nil
 }
 
-func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient {
+func newBootstrapRESTClients(endpointServerSets EndpointServerSets) []*bootstrapRESTClient {
 	seenHosts := set.NewStringSet()
 	var clnts []*bootstrapRESTClient
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if seenHosts.Contains(endpoint.Host) {
 				continue

cmd/bucket-handlers.go

@@ -487,19 +487,21 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
+		eventName := event.ObjectRemovedDelete
+
 		objInfo := ObjectInfo{
 			Name:      dobj.ObjectName,
 			VersionID: dobj.VersionID,
 		}
+
 		if dobj.DeleteMarker {
-			objInfo = ObjectInfo{
-				Name:         dobj.ObjectName,
-				DeleteMarker: dobj.DeleteMarker,
-				VersionID:    dobj.DeleteMarkerVersionID,
-			}
+			objInfo.DeleteMarker = dobj.DeleteMarker
+			objInfo.VersionID = dobj.DeleteMarkerVersionID
+			eventName = event.ObjectRemovedDeleteMarkerCreated
 		}
+
 		sendEvent(eventArgs{
-			EventName:    event.ObjectRemovedDelete,
+			EventName:    eventName,
 			BucketName:   bucket,
 			Object:       objInfo,
 			ReqParams:    extractReqParams(r),

cmd/bucket-metadata-sys.go

@@ -49,6 +49,7 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
 	}
 	sys.Lock()
 	delete(sys.metadataMap, bucket)
+	globalBucketMonitor.DeleteBucket(bucket)
 	sys.Unlock()
 }
 
@@ -373,6 +374,20 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	return meta.bucketTargetConfig, nil
 }
 
+// GetBucketTarget returns the target for the bucket and arn.
+func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) {
+	targets, err := sys.GetBucketTargetsConfig(bucket)
+	if err != nil {
+		return madmin.BucketTarget{}, err
+	}
+	for _, t := range targets.Targets {
+		if t.Arn == arn {
+			return t, nil
+		}
+	}
+	return madmin.BucketTarget{}, errConfigNotFound
+}
+
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {

cmd/bucket-replication.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/bandwidth"
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/event"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -119,7 +120,7 @@ func mustReplicater(ctx context.Context, r *http.Request, bucket, object string,
 	return cfg.Replicate(opts)
 }
 
-func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
 		if k == xhttp.AmzBucketReplicationStatus {
@@ -168,6 +169,7 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		putOpts.ServerSideEncryption = encrypt.NewSSE()
 	}
+
 	return
 }
 
@@ -184,16 +186,15 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, cfg.RoleArn))
 		return
 	}
-
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
 		VersionID: objInfo.VersionID,
 	})
 	if err != nil {
 		return
 	}
-
 	objInfo = gr.ObjInfo
 	size, err := objInfo.GetActualSize()
 	if err != nil {
@@ -224,11 +225,26 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 			return
 		}
 	}
-	putOpts := putReplicationOpts(dest, objInfo)
 
+	target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
+		return
+	}
+	putOpts := putReplicationOpts(ctx, dest, objInfo)
 	replicationStatus := replication.Complete
-	_, err = tgt.PutObject(ctx, dest.Bucket, object, gr, size, "", "", putOpts)
-	gr.Close()
+
+	// Setup bandwidth throttling
+	totalNodesCount := len(GetRemotePeers(globalEndpoints)) + 1
+	b := target.BandwidthLimit / int64(totalNodesCount)
+	var headerSize int
+	for k, v := range putOpts.Header() {
+		headerSize += len(k) + len(v)
+	}
+	r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+
+	_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
+	r.Close()
 	if err != nil {
 		replicationStatus = replication.Failed
 	}

cmd/bucket-targets.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"net/http"
 	"sync"
+	"time"
 
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
@@ -207,14 +208,14 @@ func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objA
 	return nil
 }
 
-// UpdateTarget updates target to reflect metadata updates
-func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTargets) {
+// UpdateAllTargets updates target to reflect metadata updates
+func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
 	if sys == nil {
 		return
 	}
 	sys.Lock()
 	defer sys.Unlock()
-	if cfg == nil || cfg.Empty() {
+	if tgts == nil || tgts.Empty() {
 		// remove target and arn association
 		if tgts, ok := sys.targetsMap[bucket]; ok {
 			for _, t := range tgts {
@@ -225,10 +226,10 @@ func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTarget
 		return
 	}
 
-	if len(cfg.Targets) > 0 {
-		sys.targetsMap[bucket] = cfg.Targets
+	if len(tgts.Targets) > 0 {
+		sys.targetsMap[bucket] = tgts.Targets
 	}
-	for _, tgt := range cfg.Targets {
+	for _, tgt := range tgts.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {
 			continue
@@ -238,7 +239,7 @@ func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTarget
 			sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
 		}
 	}
-	sys.targetsMap[bucket] = cfg.Targets
+	sys.targetsMap[bucket] = tgts.Targets
 }
 
 // create minio-go clients for buckets having remote targets
@@ -281,8 +282,9 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*m
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
 
 	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = NewGatewayHTTPTransport()
+		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour)
 	})
+
 	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,

cmd/config-current.go

@@ -25,9 +25,9 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
-	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
+	"github.com/minio/minio/cmd/config/heal"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
@@ -56,7 +56,7 @@ func initHelp() {
 		config.KmsKesSubSys:         crypto.DefaultKesKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
-		config.CrawlerSubSys:        crawler.DefaultKVS,
+		config.HealSubSys:           heal.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -109,8 +109,8 @@ func initHelp() {
 			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
 		},
 		config.HelpKV{
-			Key:         config.CrawlerSubSys,
-			Description: "manage continuous disk crawling for bucket disk usage, lifecycle, quota and data integrity checks",
+			Key:         config.HealSubSys,
+			Description: "manage object healing frequency and bitrot verification checks",
 		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
@@ -191,7 +191,7 @@ func initHelp() {
 		config.EtcdSubSys:           etcd.Help,
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
-		config.CrawlerSubSys:        crawler.Help,
+		config.HealSubSys:           heal.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -253,7 +253,7 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
-	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -438,9 +438,9 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 	}
-	globalCrawlerConfig, err = crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	globalHealConfig, err = heal.LookupConfig(s[config.HealSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to read crawler config: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to read heal config: %w", err))
 	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())

cmd/config/config.go

@@ -76,7 +76,7 @@ const (
 	KmsKesSubSys         = "kms_kes"
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
-	CrawlerSubSys        = "crawler"
+	HealSubSys           = "heal"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -113,7 +113,7 @@ var SubSystems = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
-	CrawlerSubSys,
+	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
 	NotifyKafkaSubSys,
@@ -140,7 +140,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
-	CrawlerSubSys,
+	HealSubSys,
 }...)
 
 // Constant separators

cmd/config/heal/heal.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package crawler
+package heal
 
 import (
 	"errors"
@@ -24,20 +24,20 @@ import (
 
 // Compression environment variables
 const (
-	BitrotScan = "bitrotscan"
+	Bitrot = "bitrotscan"
 )
 
-// Config represents the crawler settings.
+// Config represents the heal settings.
 type Config struct {
 	// Bitrot will perform bitrot scan on local disk when checking objects.
 	Bitrot bool `json:"bitrotscan"`
 }
 
 var (
-	// DefaultKVS - default KV config for crawler settings
+	// DefaultKVS - default KV config for heal settings
 	DefaultKVS = config.KVS{
 		config.KV{
-			Key:   BitrotScan,
+			Key:   Bitrot,
 			Value: config.EnableOff,
 		},
 	}
@@ -45,7 +45,7 @@ var (
 	// Help provides help for config values
 	Help = config.HelpKVS{
 		config.HelpKV{
-			Key:         BitrotScan,
+			Key:         Bitrot,
 			Description: `perform bitrot scan on disks when checking objects during crawl`,
 			Optional:    true,
 			Type:        "on|off",
@@ -55,12 +55,12 @@ var (
 
 // LookupConfig - lookup config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
-	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+	if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
-	bitrot := kvs.Get(BitrotScan)
+	bitrot := kvs.Get(Bitrot)
 	if bitrot != config.EnableOn && bitrot != config.EnableOff {
-		return cfg, errors.New(BitrotScan + ": must be 'on' or 'off'")
+		return cfg, errors.New(Bitrot + ": must be 'on' or 'off'")
 	}
 	cfg.Bitrot = bitrot == config.EnableOn
 	return cfg, nil

cmd/consolelogger.go

@@ -40,8 +40,8 @@ type HTTPConsoleLoggerSys struct {
 	logBuf   *ring.Ring
 }
 
-func mustGetNodeName(endpointZones EndpointZones) (nodeName string) {
-	host, err := xnet.ParseHost(GetLocalPeer(endpointZones))
+func mustGetNodeName(endpointServerSets EndpointServerSets) (nodeName string) {
+	host, err := xnet.ParseHost(GetLocalPeer(endpointServerSets))
 	if err != nil {
 		logger.FatalIf(err, "Unable to start console logging subsystem")
 	}
@@ -63,8 +63,8 @@ func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
 }
 
 // SetNodeName - sets the node name if any after distributed setup has initialized
-func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointZones EndpointZones) {
-	sys.nodeName = mustGetNodeName(endpointZones)
+func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointServerSets EndpointServerSets) {
+	sys.nodeName = mustGetNodeName(endpointServerSets)
 }
 
 // HasLogListeners returns true if console log listeners are registered

cmd/data-crawler.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/crawler"
+	"github.com/minio/minio/cmd/config/heal"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
@@ -53,7 +53,7 @@ const (
 )
 
 var (
-	globalCrawlerConfig          crawler.Config
+	globalHealConfig             heal.Config
 	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 )
 
@@ -135,12 +135,11 @@ type cachedFolder struct {
 }
 
 type folderScanner struct {
-	root               string
-	getSize            getSizeFn
-	oldCache           dataUsageCache
-	newCache           dataUsageCache
-	withFilter         *bloomFilter
-	waitForLowActiveIO func()
+	root       string
+	getSize    getSizeFn
+	oldCache   dataUsageCache
+	newCache   dataUsageCache
+	withFilter *bloomFilter
 
 	dataUsageCrawlMult  float64
 	dataUsageCrawlDebug bool
@@ -155,7 +154,7 @@ type folderScanner struct {
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
 // If the supplied context is canceled the function will return at the first chance.
-func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
@@ -183,7 +182,6 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		getSize:             getSize,
 		oldCache:            cache,
 		newCache:            dataUsageCache{Info: cache.Info},
-		waitForLowActiveIO:  waitForLowActiveIO,
 		newFolders:          nil,
 		existingFolders:     nil,
 		dataUsageCrawlMult:  delayMult,
@@ -376,7 +374,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				}
 			}
 		}
-		f.waitForLowActiveIO()
 		sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
 
 		cache := dataUsageEntry{}
@@ -424,7 +421,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				}
 				return nil
 			}
-			f.waitForLowActiveIO()
 			// Dynamic time delay.
 			t := UTCNow()
 
@@ -484,7 +480,9 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		// If that doesn't bring it back we remove the folder and assume it was deleted.
 		// This means that the next run will not look for it.
 		for k := range existing {
-			f.waitForLowActiveIO()
+			// Dynamic time delay.
+			t := UTCNow()
+
 			bucket, prefix := path2BucketObject(k)
 			if f.dataUsageCrawlDebug {
 				logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix)
@@ -498,6 +496,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 						versionID: versionID,
 					}, madmin.HealItemObject)
 				})
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
 
 			if f.dataUsageCrawlDebug && err != nil {
 				logger.Info(color.Green("healObjects:")+" checking returned value %v", err)
@@ -535,7 +534,6 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 		default:
 		}
 
-		f.waitForLowActiveIO()
 		if typ&os.ModeDir != 0 {
 			dirStack = append(dirStack, entName)
 			err := readDirFn(path.Join(dirStack...), addDir)
@@ -749,9 +747,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		return size
 	}
 
+	eventName := event.ObjectRemovedDelete
+	if obj.DeleteMarker {
+		eventName = event.ObjectRemovedDeleteMarkerCreated
+	}
+
 	// Notify object deleted event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectRemovedDelete,
+		EventName:  eventName,
 		BucketName: i.bucket,
 		Object:     obj,
 		Host:       "Internal: [ILM-EXPIRY]",

cmd/data-usage_test.go

@@ -62,7 +62,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return 0, nil
 	}
 
-	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -183,7 +183,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		},
 	}
 	createUsageTestFiles(t, base, bucket, files)
-	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+	got, err = crawlDataFolder(context.Background(), base, got, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -268,7 +268,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -355,7 +355,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return 0, nil
 	}
-	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -465,7 +465,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		},
 	}
 	createUsageTestFiles(t, base, "", files)
-	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+	got, err = crawlDataFolder(context.Background(), base, got, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -548,7 +548,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -652,7 +652,7 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return 0, nil
 	}
-	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
+	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/endpoint-ellipses.go

@@ -329,7 +329,7 @@ var (
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
 func createServerEndpoints(serverAddr string, args ...string) (
-	endpointZones EndpointZones, setupType SetupType, err error) {
+	endpointServerSets EndpointServerSets, setupType SetupType, err error) {
 
 	if len(args) == 0 {
 		return nil, -1, errInvalidArgument
@@ -352,13 +352,13 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, err
 		}
-		endpointZones = append(endpointZones, ZoneEndpoints{
+		endpointServerSets = append(endpointServerSets, ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
 		})
 		setupType = newSetupType
-		return endpointZones, setupType, nil
+		return endpointServerSets, setupType, nil
 	}
 
 	var prevSetupType SetupType
@@ -374,12 +374,12 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			return nil, -1, err
 		}
 		if setDriveCount != 0 && setDriveCount != len(setArgs[0]) {
-			return nil, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
+			return nil, -1, fmt.Errorf("All serverSets should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
 		}
 		if prevSetupType != UnknownSetupType && prevSetupType != setupType {
-			return nil, -1, fmt.Errorf("All zones should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
+			return nil, -1, fmt.Errorf("All serverSets should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
 		}
-		if err = endpointZones.Add(ZoneEndpoints{
+		if err = endpointServerSets.Add(ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
@@ -393,5 +393,5 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		prevSetupType = setupType
 	}
 
-	return endpointZones, setupType, nil
+	return endpointServerSets, setupType, nil
 }

cmd/endpoint.go

@@ -201,12 +201,12 @@ type ZoneEndpoints struct {
 	Endpoints    Endpoints
 }
 
-// EndpointZones - list of list of endpoints
-type EndpointZones []ZoneEndpoints
+// EndpointServerSets - list of list of endpoints
+type EndpointServerSets []ZoneEndpoints
 
 // GetLocalZoneIdx returns the zone which endpoint belongs to locally.
 // if ep is remote this code will return -1 zoneIndex
-func (l EndpointZones) GetLocalZoneIdx(ep Endpoint) int {
+func (l EndpointServerSets) GetLocalZoneIdx(ep Endpoint) int {
 	for i, zep := range l {
 		for _, cep := range zep.Endpoints {
 			if cep.IsLocal && ep.IsLocal {
@@ -220,14 +220,14 @@ func (l EndpointZones) GetLocalZoneIdx(ep Endpoint) int {
 }
 
 // Add add zone endpoints
-func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
+func (l *EndpointServerSets) Add(zeps ZoneEndpoints) error {
 	existSet := set.NewStringSet()
 	for _, zep := range *l {
 		for _, ep := range zep.Endpoints {
 			existSet.Add(ep.String())
 		}
 	}
-	// Validate if there are duplicate endpoints across zones
+	// Validate if there are duplicate endpoints across serverSets
 	for _, ep := range zeps.Endpoints {
 		if existSet.Contains(ep.String()) {
 			return fmt.Errorf("duplicate endpoints found")
@@ -238,17 +238,17 @@ func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
 }
 
 // FirstLocal returns true if the first endpoint is local.
-func (l EndpointZones) FirstLocal() bool {
+func (l EndpointServerSets) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
 }
 
 // HTTPS - returns true if secure for URLEndpointType.
-func (l EndpointZones) HTTPS() bool {
+func (l EndpointServerSets) HTTPS() bool {
 	return l[0].Endpoints.HTTPS()
 }
 
 // NEndpoints - returns all nodes count
-func (l EndpointZones) NEndpoints() (count int) {
+func (l EndpointServerSets) NEndpoints() (count int) {
 	for _, ep := range l {
 		count += len(ep.Endpoints)
 	}
@@ -256,7 +256,7 @@ func (l EndpointZones) NEndpoints() (count int) {
 }
 
 // Hostnames - returns list of unique hostnames
-func (l EndpointZones) Hostnames() []string {
+func (l EndpointServerSets) Hostnames() []string {
 	foundSet := set.NewStringSet()
 	for _, ep := range l {
 		for _, endpoint := range ep.Endpoints {
@@ -688,9 +688,9 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 // the first element from the set of peers which indicate that
 // they are local. There is always one entry that is local
 // even with repeated server endpoints.
-func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
+func GetLocalPeer(endpointServerSets EndpointServerSets) (localPeer string) {
 	peerSet := set.NewStringSet()
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.Type() != URLEndpointType {
 				continue
@@ -713,9 +713,9 @@ func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
 }
 
 // GetRemotePeers - get hosts information other than this minio service.
-func GetRemotePeers(endpointZones EndpointZones) []string {
+func GetRemotePeers(endpointServerSets EndpointServerSets) []string {
 	peerSet := set.NewStringSet()
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.Type() != URLEndpointType {
 				continue
@@ -745,12 +745,12 @@ func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 }
 
 // GetProxyEndpoints - get all endpoints that can be used to proxy list request.
-func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
+func GetProxyEndpoints(endpointServerSets EndpointServerSets) ([]ProxyEndpoint, error) {
 	var proxyEps []ProxyEndpoint
 
 	proxyEpSet := set.NewStringSet()
 
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.Type() != URLEndpointType {
 				continue
@@ -771,11 +771,7 @@ func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
 			}
 
 			// allow transport to be HTTP/1.1 for proxying.
-			tr := newCustomHTTP11Transport(tlsConfig, rest.DefaultTimeout)()
-
-			// Allow more requests to be in flight with higher response header timeout.
-			tr.ResponseHeaderTimeout = 30 * time.Minute
-			tr.MaxIdleConnsPerHost = 64
+			tr := newCustomHTTPProxyTransport(tlsConfig, rest.DefaultTimeout)()
 
 			proxyEps = append(proxyEps, ProxyEndpoint{
 				Endpoint:  endpoint,

cmd/erasure-bucket.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
@@ -49,7 +50,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 		g.Go(func() error {
 			if storageDisks[index] != nil {
 				if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
-					if err != errVolumeExists {
+					if !errors.Is(err, errVolumeExists) {
 						logger.LogIf(ctx, err)
 					}
 					return err

cmd/erasure-common.go

@@ -37,12 +37,47 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
+func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
+	disks := er.getDisks()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if disks[i-1] == nil {
+				return
+			}
+			di, err := disks[i-1].DiskInfo(context.Background())
+			if err != nil || di.Healing {
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				// - Do not consume disks which are being healed
+				//
+				// - Future: skip busy disks
+				return
+			}
+
+			mu.Lock()
+			newDisks = append(newDisks, disks[i-1])
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+	return newDisks
+}
+
 // getLoadBalancedNDisks - fetches load balanced (sufficiently randomized) disk slice
 // with N disks online. If ndisks is zero or negative, then it will returns all disks,
 // same if ndisks is greater than the number of all disks.
 func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAPI) {
 	disks := er.getLoadBalancedDisks(ndisks != -1)
 	for _, disk := range disks {
+		if disk == nil {
+			continue
+		}
 		newDisks = append(newDisks, disk)
 		ndisks--
 		if ndisks == 0 {
@@ -89,8 +124,8 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 			}
 
 			mu.Lock()
-			// Capture disks usage wise
-			newDisks[di.Used] = append(newDisks[di.Used], disks[i-1])
+			// Capture disks usage wise upto resolution of MiB
+			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i-1])
 			mu.Unlock()
 		}()
 	}

cmd/erasure-common_test.go

@@ -55,8 +55,8 @@ func TestErasureParentDirIsObject(t *testing.T) {
 		t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName)
 	}
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 	testCases := []struct {
 		parentIsObject bool
 		objectName     string

cmd/erasure-healing-common_test.go

@@ -178,8 +178,8 @@ func TestListOnlineDisks(t *testing.T) {
 
 	object := "object"
 	data := bytes.Repeat([]byte("a"), 1024)
-	z := obj.(*erasureZones)
-	erasureDisks := z.zones[0].sets[0].getDisks()
+	z := obj.(*erasureServerSets)
+	erasureDisks := z.serverSets[0].sets[0].getDisks()
 	for i, test := range testCases {
 		_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
 		if err != nil {
@@ -274,8 +274,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	// make data with more than one part
 	partCount := 3
 	data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount)
-	z := obj.(*erasureZones)
-	s := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	s := z.serverSets[0].sets[0]
 	erasureDisks := s.getDisks()
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
 	if err != nil {

cmd/erasure-healing.go

@@ -449,7 +449,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 	}
 
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tmpID, len(storageDisks)/2+1)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpID, len(storageDisks)/2+1)
 
 	// Generate and write `xl.meta` generated from other disks.
 	outDatedDisks, err = writeUniqueFileInfo(ctx, outDatedDisks, minioMetaTmpBucket, tmpID,
@@ -711,7 +711,8 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	}
 
 	if validMeta.Deleted {
-		return validMeta, false
+		// notFoundParts is ignored since a delete marker does not have any parts
+		return validMeta, corruptedErasureMeta+notFoundErasureMeta > len(errs)/2
 	}
 
 	// We couldn't find any valid meta we are indeed corrupted, return true right away.

cmd/erasure-healing_test.go

@@ -42,8 +42,8 @@ func TestHealing(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -197,8 +197,8 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	// Test 1: Remove the object backend files from the first disk.
-	z := objLayer.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := objLayer.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
 	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
@@ -342,8 +342,8 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	// Remove the object backend files from the first disk.
-	z := obj.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 	firstDisk := er.getDisks()[0]
 
 	_, err = obj.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
@@ -366,7 +366,7 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	erasureDisks := er.getDisks()
-	z.zones[0].erasureDisksMu.Lock()
+	z.serverSets[0].erasureDisksMu.Lock()
 	er.getDisks = func() []StorageAPI {
 		// Nil more than half the disks, to remove write quorum.
 		for i := 0; i <= len(erasureDisks)/2; i++ {
@@ -374,7 +374,7 @@ func TestHealObjectErasure(t *testing.T) {
 		}
 		return erasureDisks
 	}
-	z.zones[0].erasureDisksMu.Unlock()
+	z.serverSets[0].erasureDisksMu.Unlock()
 
 	// Try healing now, expect to receive errDiskNotFound.
 	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
@@ -419,8 +419,8 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	}
 
 	// Remove the object backend files from the first disk.
-	z := obj.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 	firstDisk := er.getDisks()[0]
 	err = firstDisk.DeleteVol(context.Background(), pathJoin(bucket, encodeDirObject(object)), true)
 	if err != nil {

cmd/erasure-metadata-utils_test.go

@@ -148,13 +148,13 @@ func TestShuffleDisks(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer removeRoots(disks)
-	z := objLayer.(*erasureZones)
+	z := objLayer.(*erasureServerSets)
 	testShuffleDisks(t, z)
 }
 
 // Test shuffleDisks which returns shuffled slice of disks for their actual distribution.
-func testShuffleDisks(t *testing.T, z *erasureZones) {
-	disks := z.zones[0].GetDisks(0)()
+func testShuffleDisks(t *testing.T, z *erasureServerSets) {
+	disks := z.serverSets[0].GetDisks(0)()
 	distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15}
 	shuffledDisks := shuffleDisks(disks, distribution)
 	// From the "distribution" above you can notice that:
@@ -196,6 +196,6 @@ func TestEvalDisks(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer removeRoots(disks)
-	z := objLayer.(*erasureZones)
+	z := objLayer.(*erasureServerSets)
 	testShuffleDisks(t, z)
 }

cmd/erasure-multipart.go

@@ -277,7 +277,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	// Delete the tmp path later in case we fail to commit (ignore
 	// returned errors) - this will be a no-op in case of a commit
 	// success.
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tempUploadIDPath, writeQuorum)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempUploadIDPath, writeQuorum)
 
 	var partsMetadata = make([]FileInfo, len(onlineDisks))
 	for i := range onlineDisks {
@@ -396,7 +396,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	tmpPartPath := pathJoin(tmpPart, partSuffix)
 
 	// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tmpPart, writeQuorum)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpPart, writeQuorum)
 
 	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
 	if err != nil {

cmd/erasure-object_test.go

@@ -132,8 +132,8 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 		for _, dir := range fsDirs {
 			defer os.RemoveAll(dir)
 		}
-		z := obj.(*erasureZones)
-		xl := z.zones[0].sets[0]
+		z := obj.(*erasureServerSets)
+		xl := z.serverSets[0].sets[0]
 		objs = append(objs, xl)
 	}
 
@@ -205,8 +205,8 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -225,7 +225,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	// for a 16 disk setup, quorum is 9. To simulate disks not found yet
 	// quorum is available, we remove disks leaving quorum disks behind.
 	erasureDisks := xl.getDisks()
-	z.zones[0].erasureDisksMu.Lock()
+	z.serverSets[0].erasureDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		for i := range erasureDisks[:7] {
 			erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk)
@@ -233,7 +233,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 		return erasureDisks
 	}
 
-	z.zones[0].erasureDisksMu.Unlock()
+	z.serverSets[0].erasureDisksMu.Unlock()
 	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
@@ -247,14 +247,14 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 
 	// Remove one more disk to 'lose' quorum, by setting it to nil.
 	erasureDisks = xl.getDisks()
-	z.zones[0].erasureDisksMu.Lock()
+	z.serverSets[0].erasureDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		erasureDisks[7] = nil
 		erasureDisks[8] = nil
 		return erasureDisks
 	}
 
-	z.zones[0].erasureDisksMu.Unlock()
+	z.serverSets[0].erasureDisksMu.Unlock()
 	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})
 	// since majority of disks are not available, metaquorum is not achieved and hence errErasureWriteQuorum error
 	if err != toObjectErr(errErasureWriteQuorum, bucket, object) {
@@ -275,8 +275,8 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -311,11 +311,11 @@ func TestGetObjectNoQuorum(t *testing.T) {
 				erasureDisks[i] = newNaughtyDisk(erasureDisks[i], diskErrors, errFaultyDisk)
 			}
 		}
-		z.zones[0].erasureDisksMu.Lock()
+		z.serverSets[0].erasureDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI {
 			return erasureDisks
 		}
-		z.zones[0].erasureDisksMu.Unlock()
+		z.serverSets[0].erasureDisksMu.Unlock()
 		// Fetch object from store.
 		err = xl.GetObject(ctx, bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts)
 		if err != toObjectErr(errErasureReadQuorum, bucket, object) {
@@ -338,8 +338,8 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -374,11 +374,11 @@ func TestPutObjectNoQuorum(t *testing.T) {
 				erasureDisks[i] = newNaughtyDisk(erasureDisks[i], diskErrors, errFaultyDisk)
 			}
 		}
-		z.zones[0].erasureDisksMu.Lock()
+		z.serverSets[0].erasureDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI {
 			return erasureDisks
 		}
-		z.zones[0].erasureDisksMu.Unlock()
+		z.serverSets[0].erasureDisksMu.Unlock()
 		// Upload new content to same object "object"
 		_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
 		if err != toObjectErr(errErasureWriteQuorum, bucket, object) {
@@ -404,8 +404,8 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	partCount := 3
 	data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 	erasureDisks := xl.getDisks()
 
 	ctx, cancel := context.WithCancel(GlobalContext)

cmd/erasure-sets.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
+	"math/rand"
 	"net/http"
 	"sort"
 	"sync"
@@ -235,7 +236,7 @@ func (s *erasureSets) connectDisks() {
 			disk.SetDiskID(format.Erasure.This)
 			if endpoint.IsLocal && disk.Healing() {
 				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
-				logger.Info(fmt.Sprintf("Found the drive %s which needs healing, attempting to heal...", disk))
+				logger.Info(fmt.Sprintf("Found the drive %s that needs healing, attempting to heal...", disk))
 			}
 
 			s.erasureDisksMu.Lock()
@@ -261,6 +262,13 @@ func (s *erasureSets) connectDisks() {
 // endpoints by reconnecting them and making sure to place them into right position in
 // the set topology, this monitoring happens at a given monitoring interval.
 func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInterval time.Duration) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
+
+	// Pre-emptively connect the disks if possible.
+	s.connectDisks()
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -271,6 +279,17 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 	}
 }
 
+// GetAllLockers return a list of all lockers for all sets.
+func (s *erasureSets) GetAllLockers() []dsync.NetLocker {
+	allLockers := make([]dsync.NetLocker, s.setDriveCount*s.setCount)
+	for i, lockers := range s.erasureLockers {
+		for j, locker := range lockers {
+			allLockers[i*s.setDriveCount+j] = locker
+		}
+	}
+	return allLockers
+}
+
 func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) {
 	return func() ([]dsync.NetLocker, string) {
 		lockers := make([]dsync.NetLocker, s.setDriveCount)
@@ -332,7 +351,7 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 		sets:                make([]*erasureObjects, setCount),
 		erasureDisks:        make([][]StorageAPI, setCount),
 		erasureLockers:      make([][]dsync.NetLocker, setCount),
-		erasureLockOwner:    mustGetUUID(),
+		erasureLockOwner:    GetLocalPeer(globalEndpoints),
 		endpoints:           endpoints,
 		endpointStrings:     endpointStrings,
 		setCount:            setCount,
@@ -416,7 +435,7 @@ func (s *erasureSets) SetDriveCount() int {
 }
 
 // StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets.
-// This only returns disk usage info for Zones to perform placement decision, this call
+// This only returns disk usage info for ServerSets to perform placement decision, this call
 // is not implemented in Object interface and is not meant to be used by other object
 // layer implementations.
 func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
@@ -1007,6 +1026,7 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
 			wg.Add(1)
 			go func(disk StorageAPI) {
 				defer wg.Done()
+
 				entryCh, err := disk.WalkVersions(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
 				if err != nil {
 					return

cmd/erasure-sets_test.go

@@ -213,8 +213,8 @@ func TestHashedLayer(t *testing.T) {
 			defer os.RemoveAll(dir)
 		}
 
-		z := obj.(*erasureZones)
-		objs = append(objs, z.zones[0].sets[0])
+		z := obj.(*erasureServerSets)
+		objs = append(objs, z.serverSets[0].sets[0])
 	}
 
 	sets := &erasureSets{sets: objs, distributionAlgo: "CRCMOD"}

cmd/erasure.go

@@ -81,14 +81,6 @@ func (er erasureObjects) SetDriveCount() int {
 func (er erasureObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
 	closeStorageDisks(er.getDisks())
-	select {
-	case _, ok := <-er.mrfOpCh:
-		if ok {
-			close(er.mrfOpCh)
-		}
-	default:
-		close(er.mrfOpCh)
-	}
 	return nil
 }
 
@@ -252,7 +244,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 	}
 
 	// Collect disks we can use.
-	disks := er.getLoadBalancedDisks(true)
+	disks := er.getOnlineDisks()
 	if len(disks) == 0 {
 		logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl")
 		return nil

cmd/fs-v1.go

@@ -327,7 +327,7 @@ func (fs *FSObjects) crawlBucket(ctx context.Context, bucket string, cache dataU
 	}
 
 	// Load bucket info.
-	cache, err = crawlDataFolder(ctx, fs.fsPath, cache, fs.waitForLowActiveIO, func(item crawlItem) (int64, error) {
+	cache, err = crawlDataFolder(ctx, fs.fsPath, cache, func(item crawlItem) (int64, error) {
 		bucket, object := item.bucket, item.objectPath()
 		fsMetaBytes, err := ioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile))
 		if err != nil && !os.IsNotExist(err) {

cmd/gateway-main.go

@@ -19,12 +19,14 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"net"
 	"net/url"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
@@ -153,6 +155,11 @@ func ValidateGatewayArguments(serverAddr, endpointAddr string) error {
 
 // StartGateway - handler for 'minio gateway <name>'.
 func StartGateway(ctx *cli.Context, gw Gateway) {
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
+	defer globalDNSCache.Stop()
+
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
 	logger.SetDeploymentID(globalDeploymentID)
@@ -289,7 +296,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalHTTPServer = httpServer
 	globalObjLayerMutex.Unlock()
 
-	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
+	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 
 	newObject, err := gw.NewGatewayLayer(globalActiveCred)
 	if err != nil {
@@ -323,8 +330,8 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	if enableIAMOps {
-		// Initialize IAM sys.
-		startBackgroundIAMLoad(GlobalContext, newObject)
+		// Initialize users credentials and policies in background.
+		go globalIAMSys.Init(GlobalContext, newObject)
 	}
 
 	if globalCacheConfig.Enabled {

cmd/gateway/azure/gateway-azure.go

@@ -559,7 +559,8 @@ func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.Storag
 
 // MakeBucketWithLocation - Create a new container on azure backend.
 func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
-	if opts.LockEnabled || opts.VersioningEnabled {
+	// Filter out unsupported features in Azure and return immediately with NotImplemented error
+	if opts.LockEnabled || opts.VersioningEnabled || strings.ContainsAny(bucket, ".") {
 		return minio.NotImplemented{}
 	}
 

cmd/gateway/nas/gateway-nas.go

@@ -52,7 +52,6 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
-
      {{.Prompt}} {{.HelpName}} /shared/nasvol
 `
 

cmd/globals.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/pkg/bucket/bandwidth"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/config/cache"
@@ -149,6 +150,7 @@ var (
 	globalEnvTargetList *event.TargetList
 
 	globalBucketMetadataSys *BucketMetadataSys
+	globalBucketMonitor     *bandwidth.Monitor
 	globalPolicySys         *PolicySys
 	globalIAMSys            *IAMSys
 
@@ -186,7 +188,7 @@ var (
 	// registered listeners
 	globalConsoleSys *HTTPConsoleLoggerSys
 
-	globalEndpoints EndpointZones
+	globalEndpoints EndpointServerSets
 
 	// Global server's network statistics
 	globalConnStats = newConnStats()
@@ -272,6 +274,8 @@ var (
 	globalFSOSync bool
 
 	globalProxyEndpoints []ProxyEndpoint
+
+	globalDNSCache *xhttp.DNSCache
 	// Add new variable global values here.
 )
 

cmd/http/dial_dnscache.go

@@ -0,0 +1,197 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"context"
+	"log"
+	"math/rand"
+	"net"
+	"sync"
+	"time"
+)
+
+var randPerm = func(n int) []int {
+	return rand.Perm(n)
+}
+
+// DialContextWithDNSCache is a helper function which returns `net.DialContext` function.
+// It randomly fetches an IP from the DNS cache and dials it by the given dial
+// function. It dials one by one and returns first connected `net.Conn`.
+// If it fails to dial all IPs from cache it returns first error. If no baseDialFunc
+// is given, it sets default dial function.
+//
+// You can use returned dial function for `http.Transport.DialContext`.
+//
+// In this function, it uses functions from `rand` package. To make it really random,
+// you MUST call `rand.Seed` and change the value from the default in your application
+func DialContextWithDNSCache(cache *DNSCache, baseDialCtx DialContext) DialContext {
+	if baseDialCtx == nil {
+		// This is same as which `http.DefaultTransport` uses.
+		baseDialCtx = (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext
+	}
+	return func(ctx context.Context, network, host string) (net.Conn, error) {
+		h, p, err := net.SplitHostPort(host)
+		if err != nil {
+			return nil, err
+		}
+
+		// Fetch DNS result from cache.
+		//
+		// ctxLookup is only used for canceling DNS Lookup.
+		ctxLookup, cancelF := context.WithTimeout(ctx, cache.lookupTimeout)
+		defer cancelF()
+		addrs, err := cache.Fetch(ctxLookup, h)
+		if err != nil {
+			return nil, err
+		}
+
+		var firstErr error
+		for _, randomIndex := range randPerm(len(addrs)) {
+			conn, err := baseDialCtx(ctx, "tcp", net.JoinHostPort(addrs[randomIndex], p))
+			if err == nil {
+				return conn, nil
+			}
+			if firstErr == nil {
+				firstErr = err
+			}
+		}
+
+		return nil, firstErr
+	}
+}
+
+const (
+	// cacheSize is initial size of addr and IP list cache map.
+	cacheSize = 64
+)
+
+// defaultFreq is default frequency a resolver refreshes DNS cache.
+var (
+	defaultFreq          = 3 * time.Second
+	defaultLookupTimeout = 10 * time.Second
+)
+
+// DNSCache is DNS cache resolver which cache DNS resolve results in memory.
+type DNSCache struct {
+	sync.RWMutex
+
+	lookupHostFn  func(ctx context.Context, host string) ([]string, error)
+	lookupTimeout time.Duration
+
+	cache  map[string][]string
+	closer func()
+}
+
+// NewDNSCache initializes DNS cache resolver and starts auto refreshing
+// in a new goroutine. To stop auto refreshing, call `Stop()` function.
+// Once `Stop()` is called auto refreshing cannot be resumed.
+func NewDNSCache(freq time.Duration, lookupTimeout time.Duration) *DNSCache {
+	if freq <= 0 {
+		freq = defaultFreq
+	}
+
+	if lookupTimeout <= 0 {
+		lookupTimeout = defaultLookupTimeout
+	}
+
+	ticker := time.NewTicker(freq)
+	ch := make(chan struct{})
+	closer := func() {
+		ticker.Stop()
+		close(ch)
+	}
+
+	r := &DNSCache{
+		lookupHostFn:  net.DefaultResolver.LookupHost,
+		lookupTimeout: lookupTimeout,
+		cache:         make(map[string][]string, cacheSize),
+		closer:        closer,
+	}
+
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				r.Refresh()
+			case <-ch:
+				return
+			}
+		}
+	}()
+
+	return r
+}
+
+// LookupHost lookups address list from DNS server, persist the results
+// in-memory cache. `Fetch` is used to obtain the values for a given host.
+func (r *DNSCache) LookupHost(ctx context.Context, host string) ([]string, error) {
+	addrs, err := r.lookupHostFn(ctx, host)
+	if err != nil {
+		return nil, err
+	}
+
+	r.Lock()
+	r.cache[host] = addrs
+	r.Unlock()
+
+	return addrs, nil
+}
+
+// Fetch fetches IP list from the cache. If IP list of the given addr is not in the cache,
+// then it lookups from DNS server by `Lookup` function.
+func (r *DNSCache) Fetch(ctx context.Context, host string) ([]string, error) {
+	r.RLock()
+	addrs, ok := r.cache[host]
+	r.RUnlock()
+	if ok {
+		return addrs, nil
+	}
+	return r.LookupHost(ctx, host)
+}
+
+// Refresh refreshes IP list cache, automatically.
+func (r *DNSCache) Refresh() {
+	r.RLock()
+	hosts := make([]string, 0, len(r.cache))
+	for host := range r.cache {
+		hosts = append(hosts, host)
+	}
+	r.RUnlock()
+
+	for _, host := range hosts {
+		ctx, cancelF := context.WithTimeout(context.Background(), r.lookupTimeout)
+		if _, err := r.LookupHost(ctx, host); err != nil {
+			log.Println("failed to refresh DNS cache, resolver is unavailable", err)
+		}
+		cancelF()
+	}
+}
+
+// Stop stops auto refreshing.
+func (r *DNSCache) Stop() {
+	r.Lock()
+	defer r.Unlock()
+	if r.closer != nil {
+		r.closer()
+		r.closer = nil
+	}
+}

cmd/http/dial_dnscache_test.go

@@ -0,0 +1,204 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"net"
+	"testing"
+	"time"
+)
+
+var (
+	testFreq                 = 1 * time.Second
+	testDefaultLookupTimeout = 1 * time.Second
+)
+
+func testDNSCache(t *testing.T) *DNSCache {
+	t.Helper() // skip printing file and line information from this function
+	return NewDNSCache(testFreq, testDefaultLookupTimeout)
+}
+
+func TestDialContextWithDNSCache(t *testing.T) {
+	resolver := &DNSCache{
+		cache: map[string][]string{
+			"play.min.io": {
+				"127.0.0.1",
+				"127.0.0.2",
+				"127.0.0.3",
+			},
+		},
+	}
+
+	cases := []struct {
+		permF func(n int) []int
+		dialF DialContext
+	}{
+		{
+			permF: func(n int) []int {
+				return []int{0}
+			},
+			dialF: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				if got, want := addr, net.JoinHostPort("127.0.0.1", "443"); got != want {
+					t.Fatalf("got addr %q, want %q", got, want)
+				}
+				return nil, nil
+			},
+		},
+		{
+			permF: func(n int) []int {
+				return []int{1}
+			},
+			dialF: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				if got, want := addr, net.JoinHostPort("127.0.0.2", "443"); got != want {
+					t.Fatalf("got addr %q, want %q", got, want)
+				}
+				return nil, nil
+			},
+		},
+		{
+			permF: func(n int) []int {
+				return []int{2}
+			},
+			dialF: func(ctx context.Context, network, addr string) (net.Conn, error) {
+				if got, want := addr, net.JoinHostPort("127.0.0.3", "443"); got != want {
+					t.Fatalf("got addr %q, want %q", got, want)
+				}
+				return nil, nil
+			},
+		},
+	}
+
+	origFunc := randPerm
+	defer func() {
+		randPerm = origFunc
+	}()
+
+	for _, tc := range cases {
+		t.Run("", func(t *testing.T) {
+			randPerm = tc.permF
+			if _, err := DialContextWithDNSCache(resolver, tc.dialF)(context.Background(), "tcp", "play.min.io:443"); err != nil {
+				t.Fatalf("err: %s", err)
+			}
+		})
+	}
+
+}
+
+func TestDialContextWithDNSCacheRand(t *testing.T) {
+	rand.Seed(time.Now().UTC().UnixNano())
+	defer func() {
+		rand.Seed(1)
+	}()
+
+	resolver := &DNSCache{
+		cache: map[string][]string{
+			"play.min.io": {
+				"127.0.0.1",
+				"127.0.0.2",
+				"127.0.0.3",
+			},
+		},
+	}
+
+	count := make(map[string]int)
+	dialF := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		count[addr]++
+		return nil, nil
+	}
+
+	for i := 0; i < 100; i++ {
+		if _, err := DialContextWithDNSCache(resolver, dialF)(context.Background(), "tcp", "play.min.io:443"); err != nil {
+			t.Fatalf("err: %s", err)
+		}
+	}
+
+	for _, c := range count {
+		got := float32(c) / float32(100)
+		if got < float32(0.2) {
+			t.Fatalf("expected 0.2 rate got %f", got)
+		}
+	}
+}
+
+// Verify without port Dial fails, Go stdlib net.Dial expects port
+func TestDialContextWithDNSCacheScenario1(t *testing.T) {
+	resolver := testDNSCache(t)
+	if _, err := DialContextWithDNSCache(resolver, nil)(context.Background(), "tcp", "play.min.io"); err == nil {
+		t.Fatalf("expect to fail") // expected port
+	}
+}
+
+// Verify if the host lookup function failed to return addresses
+func TestDialContextWithDNSCacheScenario2(t *testing.T) {
+	res := testDNSCache(t)
+	originalFunc := res.lookupHostFn
+	defer func() {
+		res.lookupHostFn = originalFunc
+	}()
+
+	res.lookupHostFn = func(ctx context.Context, host string) ([]string, error) {
+		return nil, fmt.Errorf("err")
+	}
+
+	if _, err := DialContextWithDNSCache(res, nil)(context.Background(), "tcp", "min.io:443"); err == nil {
+		t.Fatalf("exect to fail")
+	}
+}
+
+// Verify we always return the first error from net.Dial failure
+func TestDialContextWithDNSCacheScenario3(t *testing.T) {
+	resolver := &DNSCache{
+		cache: map[string][]string{
+			"min.io": {
+				"1.1.1.1",
+				"2.2.2.2",
+				"3.3.3.3",
+			},
+		},
+	}
+
+	origFunc := randPerm
+	randPerm = func(n int) []int {
+		return []int{0, 1, 2}
+	}
+	defer func() {
+		randPerm = origFunc
+	}()
+
+	want := errors.New("error1")
+	dialF := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		if addr == net.JoinHostPort("1.1.1.1", "443") {
+			return nil, want // first error should be returned
+		}
+		if addr == net.JoinHostPort("2.2.2.2", "443") {
+			return nil, fmt.Errorf("error2")
+		}
+		if addr == net.JoinHostPort("3.3.3.3", "443") {
+			return nil, fmt.Errorf("error3")
+		}
+		return nil, nil
+	}
+
+	_, got := DialContextWithDNSCache(resolver, dialF)(context.Background(), "tcp", "min.io:443")
+	if got != want {
+		t.Fatalf("got error %v, want %v", got, want)
+	}
+}

cmd/http/listen_nix.go

@@ -27,9 +27,9 @@ import (
 var cfg = &tcplisten.Config{
 	DeferAccept: true,
 	FastOpen:    true,
-	// Bump up the soMaxConn value from 128 to 2048 to
+	// Bump up the soMaxConn value from 128 to 4096 to
 	// handle large incoming concurrent requests.
-	Backlog: 2048,
+	Backlog: 4096,
 }
 
 // Unix listener with special TCP options.

cmd/iam.go

@@ -408,13 +408,6 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
 	return sys.store.migrateBackendFormat(ctx)
 }
 
-// Loads IAM users and policies in background, any un-handled
-// error means this code can potentially crash the server
-// in such a situation manual intervention is necessary.
-func startBackgroundIAMLoad(ctx context.Context, objAPI ObjectLayer) {
-	go globalIAMSys.Init(ctx, objAPI)
-}
-
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	if objAPI == nil {

cmd/lock-rest-server.go

@@ -247,6 +247,11 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 		return nil
 	}
 
+	z, ok := objAPI.(*erasureServerSets)
+	if !ok {
+		return nil
+	}
+
 	type nlock struct {
 		locks  int
 		writer bool
@@ -265,6 +270,8 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 		}
 	}
 
+	allLockersFn := z.GetAllLockers
+
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
 	for lendpoint, nlrips := range getLongLivedLocks(interval) {
@@ -273,9 +280,8 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 			// Locks are only held on first zone, make sure that
 			// we only look for ownership of locks from endpoints
 			// on first zone.
-			for _, endpoint := range globalEndpoints[0].Endpoints {
-				c := newLockAPI(endpoint)
-				if !c.IsOnline() {
+			for _, c := range allLockersFn() {
+				if !c.IsOnline() || c == nil {
 					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
 					continue
 				}
@@ -292,16 +298,12 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 				cancel()
 				if err != nil {
 					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
-					c.Close()
 					continue
 				}
 
 				if !expired {
 					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
 				}
-
-				// Close the connection regardless of the call response.
-				c.Close()
 			}
 
 			// Read locks we assume quorum for be N/2 success
@@ -366,8 +368,8 @@ func startLockMaintenance(ctx context.Context) {
 }
 
 // registerLockRESTHandlers - register lock rest router.
-func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
-	for _, ep := range endpointZones {
+func registerLockRESTHandlers(router *mux.Router, endpointServerSets EndpointServerSets) {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if !endpoint.IsLocal {
 				continue

cmd/logger/logger.go

@@ -19,6 +19,7 @@ package logger
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"go/build"
 	"hash"
@@ -60,11 +61,6 @@ var globalDeploymentID string
 // TimeFormat - logging time format.
 const TimeFormat string = "15:04:05 MST 01/02/2006"
 
-// List of error strings to be ignored by LogIf
-const (
-	diskNotFoundError = "disk not found"
-)
-
 var matchingFuncNames = [...]string{
 	"http.HandlerFunc.ServeHTTP",
 	"cmd.serverMain",
@@ -303,7 +299,7 @@ func LogIf(ctx context.Context, err error, errKind ...interface{}) {
 		return
 	}
 
-	if err.Error() != diskNotFoundError {
+	if !errors.Is(err, context.Canceled) {
 		logIf(ctx, err, errKind...)
 	}
 }

cmd/notification.go

@@ -33,6 +33,8 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
+	bandwidth "github.com/minio/minio/pkg/bandwidth"
+	bucketBandwidth "github.com/minio/minio/pkg/bucket/bandwidth"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
@@ -1185,7 +1187,7 @@ func (sys *NotificationSys) GetLocalDiskIDs(ctx context.Context) (localDiskIDs [
 }
 
 // NewNotificationSys - creates new notification system object.
-func NewNotificationSys(endpoints EndpointZones) *NotificationSys {
+func NewNotificationSys(endpoints EndpointServerSets) *NotificationSys {
 	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.Init()
 	return &NotificationSys{
 		targetList:                 event.NewTargetList(),
@@ -1283,3 +1285,50 @@ func sendEvent(args eventArgs) {
 
 	globalNotificationSys.Send(args)
 }
+
+// GetBandwidthReports - gets the bandwidth report from all nodes including self.
+func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...string) bandwidth.Report {
+	reports := make([]*bandwidth.Report, len(sys.peerClients))
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	for index := range sys.peerClients {
+		if sys.peerClients[index] == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			var err error
+			reports[index], err = sys.peerClients[index].MonitorBandwidth(ctx, buckets)
+			return err
+		}, index)
+	}
+
+	for index, err := range g.Wait() {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
+			sys.peerClients[index].host.String())
+		ctx := logger.SetReqInfo(ctx, reqInfo)
+		logger.LogOnceIf(ctx, err, sys.peerClients[index].host.String())
+	}
+	reports = append(reports, globalBucketMonitor.GetReport(bucketBandwidth.SelectBuckets(buckets...)))
+	consolidatedReport := bandwidth.Report{
+		BucketStats: make(map[string]bandwidth.Details),
+	}
+	for _, report := range reports {
+		if report == nil || report.BucketStats == nil {
+			continue
+		}
+		for bucket := range report.BucketStats {
+			d, ok := consolidatedReport.BucketStats[bucket]
+			if !ok {
+				consolidatedReport.BucketStats[bucket] = bandwidth.Details{}
+				d = consolidatedReport.BucketStats[bucket]
+				d.LimitInBytesPerSecond = report.BucketStats[bucket].LimitInBytesPerSecond
+			}
+			if d.LimitInBytesPerSecond < report.BucketStats[bucket].LimitInBytesPerSecond {
+				d.LimitInBytesPerSecond = report.BucketStats[bucket].LimitInBytesPerSecond
+			}
+			d.CurrentBandwidthInBytesPerSecond += report.BucketStats[bucket].CurrentBandwidthInBytesPerSecond
+			consolidatedReport.BucketStats[bucket] = d
+		}
+	}
+	return consolidatedReport
+}

cmd/obdinfo.go

@@ -61,10 +61,10 @@ func getLocalCPUOBDInfo(ctx context.Context, r *http.Request) madmin.ServerCPUOB
 
 }
 
-func getLocalDrivesOBD(ctx context.Context, parallel bool, endpointZones EndpointZones, r *http.Request) madmin.ServerDrivesOBDInfo {
+func getLocalDrivesOBD(ctx context.Context, parallel bool, endpointServerSets EndpointServerSets, r *http.Request) madmin.ServerDrivesOBDInfo {
 	var drivesOBDInfo []madmin.DriveOBDInfo
 	var wg sync.WaitGroup
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			// Only proceed for local endpoints
 			if endpoint.IsLocal {
@@ -105,7 +105,7 @@ func getLocalDrivesOBD(ctx context.Context, parallel bool, endpointZones Endpoin
 
 	addr := r.Host
 	if globalIsDistErasure {
-		addr = GetLocalPeer(endpointZones)
+		addr = GetLocalPeer(endpointServerSets)
 	}
 	if parallel {
 		return madmin.ServerDrivesOBDInfo{

cmd/object-handlers-common.go

@@ -291,30 +291,20 @@ func deleteObject(ctx context.Context, obj ObjectLayer, cache CacheObjectLayer,
 	// Proceed to delete the object.
 	objInfo, err = deleteObject(ctx, bucket, object, opts)
 	if objInfo.Name != "" {
-		// Requesting only a delete marker which was successfully attempted.
+		eventName := event.ObjectRemovedDelete
 		if objInfo.DeleteMarker {
-			// Notify object deleted marker event.
-			sendEvent(eventArgs{
-				EventName:    event.ObjectRemovedDeleteMarkerCreated,
-				BucketName:   bucket,
-				Object:       objInfo,
-				ReqParams:    extractReqParams(r),
-				RespElements: extractRespElements(w),
-				UserAgent:    r.UserAgent(),
-				Host:         handlers.GetSourceIP(r),
-			})
-		} else {
-			// Notify object deleted event.
-			sendEvent(eventArgs{
-				EventName:    event.ObjectRemovedDelete,
-				BucketName:   bucket,
-				Object:       objInfo,
-				ReqParams:    extractReqParams(r),
-				RespElements: extractRespElements(w),
-				UserAgent:    r.UserAgent(),
-				Host:         handlers.GetSourceIP(r),
-			})
+			eventName = event.ObjectRemovedDeleteMarkerCreated
 		}
+		// Notify object deleted marker event.
+		sendEvent(eventArgs{
+			EventName:    eventName,
+			BucketName:   bucket,
+			Object:       objInfo,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         handlers.GetSourceIP(r),
+		})
 	}
 	return objInfo, err
 }

cmd/peer-rest-client.go

@@ -26,6 +26,7 @@ import (
 	"math"
 	"net/url"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -35,6 +36,7 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
+	"github.com/minio/minio/pkg/bandwidth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
@@ -820,8 +822,8 @@ func (client *peerRESTClient) ConsoleLog(logCh chan interface{}, doneCh <-chan s
 	}()
 }
 
-func getRemoteHosts(endpointZones EndpointZones) []*xnet.Host {
-	peers := GetRemotePeers(endpointZones)
+func getRemoteHosts(endpointServerSets EndpointServerSets) []*xnet.Host {
+	peers := GetRemotePeers(endpointServerSets)
 	remoteHosts := make([]*xnet.Host, 0, len(peers))
 	for _, hostStr := range peers {
 		host, err := xnet.ParseHost(hostStr)
@@ -836,7 +838,7 @@ func getRemoteHosts(endpointZones EndpointZones) []*xnet.Host {
 }
 
 // newPeerRestClients creates new peer clients.
-func newPeerRestClients(endpoints EndpointZones) []*peerRESTClient {
+func newPeerRestClients(endpoints EndpointServerSets) []*peerRESTClient {
 	peerHosts := getRemoteHosts(endpoints)
 	restClients := make([]*peerRESTClient, len(peerHosts))
 	for i, host := range peerHosts {
@@ -884,3 +886,19 @@ func newPeerRESTClient(peer *xnet.Host) *peerRESTClient {
 
 	return &peerRESTClient{host: peer, restClient: restClient}
 }
+
+// MonitorBandwidth - send http trace request to peer nodes
+func (client *peerRESTClient) MonitorBandwidth(ctx context.Context, buckets []string) (*bandwidth.Report, error) {
+	values := make(url.Values)
+	values.Set(peerRESTBuckets, strings.Join(buckets, ","))
+	respBody, err := client.callWithContext(ctx, peerRESTMethodGetBandwidth, values, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+	defer http.DrainBody(respBody)
+
+	dec := gob.NewDecoder(respBody)
+	var bandwidthReport bandwidth.Report
+	err = dec.Decode(&bandwidthReport)
+	return &bandwidthReport, err
+}

cmd/peer-rest-common.go

@@ -57,10 +57,12 @@ const (
 	peerRESTMethodListen                = "/listen"
 	peerRESTMethodLog                   = "/log"
 	peerRESTMethodGetLocalDiskIDs       = "/getlocaldiskids"
+	peerRESTMethodGetBandwidth          = "/bandwidth"
 )
 
 const (
 	peerRESTBucket      = "bucket"
+	peerRESTBuckets     = "buckets"
 	peerRESTUser        = "user"
 	peerRESTGroup       = "group"
 	peerRESTUserTemp    = "user-temp"

cmd/peer-rest-server.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
+	b "github.com/minio/minio/pkg/bucket/bandwidth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
 	trace "github.com/minio/minio/pkg/trace"
@@ -628,7 +629,7 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
 	}
 
 	if meta.bucketTargetConfig != nil {
-		globalBucketTargetSys.UpdateTarget(bucketName, meta.bucketTargetConfig)
+		globalBucketTargetSys.UpdateAllTargets(bucketName, meta.bucketTargetConfig)
 	}
 }
 
@@ -726,11 +727,11 @@ func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *
 }
 
 // Return disk IDs of all the local disks.
-func getLocalDiskIDs(z *erasureZones) []string {
+func getLocalDiskIDs(z *erasureServerSets) []string {
 	var ids []string
 
-	for zoneIdx := range z.zones {
-		for _, set := range z.zones[zoneIdx].sets {
+	for zoneIdx := range z.serverSets {
+		for _, set := range z.serverSets[zoneIdx].sets {
 			disks := set.getDisks()
 			for _, disk := range disks {
 				if disk == nil {
@@ -775,7 +776,7 @@ func (s *peerRESTServer) GetLocalDiskIDs(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	z, ok := objLayer.(*erasureZones)
+	z, ok := objLayer.(*erasureServerSets)
 	if !ok {
 		s.writeErrorResponse(w, errServerNotInitialized)
 		return
@@ -1047,6 +1048,30 @@ func (s *peerRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+// GetBandwidth gets the bandwidth for the buckets requested.
+func (s *peerRESTServer) GetBandwidth(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+	bucketsString := r.URL.Query().Get("buckets")
+	w.WriteHeader(http.StatusOK)
+	w.(http.Flusher).Flush()
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	selectBuckets := b.SelectBuckets(strings.Split(bucketsString, ",")...)
+	report := globalBucketMonitor.GetReport(selectBuckets)
+
+	enc := gob.NewEncoder(w)
+	if err := enc.Encode(report); err != nil {
+		s.writeErrorResponse(w, errors.New("Encoding report failed: "+err.Error()))
+		return
+	}
+	w.(http.Flusher).Flush()
+}
+
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
 	server := &peerRESTServer{}
@@ -1085,4 +1110,5 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLog).HandlerFunc(server.ConsoleLogHandler)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocalDiskIDs).HandlerFunc(httpTraceHdrs(server.GetLocalDiskIDs))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBandwidth).HandlerFunc(httpTraceHdrs(server.GetBandwidth))
 }

cmd/rest/client.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"sync/atomic"
@@ -74,11 +75,10 @@ type Client struct {
 	// Should only be modified before any calls are made.
 	MaxErrResponseSize int64
 
-	httpClient          *http.Client
-	httpIdleConnsCloser func()
-	url                 *url.URL
-	newAuthToken        func(audience string) string
-	connected           int32
+	httpClient   *http.Client
+	url          *url.URL
+	newAuthToken func(audience string) string
+	connected    int32
 }
 
 // URL query separator constants
@@ -157,9 +157,6 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
 // Close closes all idle connections of the underlying http client
 func (c *Client) Close() {
 	atomic.StoreInt32(&c.connected, closed)
-	if c.httpIdleConnsCloser != nil {
-		c.httpIdleConnsCloser()
-	}
 }
 
 // NewClient - returns new REST client.
@@ -169,7 +166,6 @@ func NewClient(url *url.URL, newCustomTransport func() *http.Transport, newAuthT
 	tr := newCustomTransport()
 	return &Client{
 		httpClient:          &http.Client{Transport: tr},
-		httpIdleConnsCloser: tr.CloseIdleConnections,
 		url:                 url,
 		newAuthToken:        newAuthToken,
 		connected:           online,
@@ -190,18 +186,18 @@ func (c *Client) MarkOffline() {
 	// Start goroutine that will attempt to reconnect.
 	// If server is already trying to reconnect this will have no effect.
 	if c.HealthCheckFn != nil && atomic.CompareAndSwapInt32(&c.connected, online, offline) {
-		go func(healthFunc func() bool) {
-			ticker := time.NewTicker(c.HealthCheckInterval)
-			defer ticker.Stop()
-			for range ticker.C {
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+		go func() {
+			for {
 				if atomic.LoadInt32(&c.connected) == closed {
 					return
 				}
-				if healthFunc() {
+				if c.HealthCheckFn() {
 					atomic.CompareAndSwapInt32(&c.connected, offline, online)
 					return
 				}
+				time.Sleep(time.Duration(r.Float64() * float64(c.HealthCheckInterval)))
 			}
-		}(c.HealthCheckFn)
+		}()
 	}
 }

cmd/routers.go

@@ -23,9 +23,9 @@ import (
 )
 
 // Composed function registering routers for only distributed Erasure setup.
-func registerDistErasureRouters(router *mux.Router, endpointZones EndpointZones) {
+func registerDistErasureRouters(router *mux.Router, endpointServerSets EndpointServerSets) {
 	// Register storage REST router only if its a distributed setup.
-	registerStorageRESTHandlers(router, endpointZones)
+	registerStorageRESTHandlers(router, endpointServerSets)
 
 	// Register peer REST router only if its a distributed setup.
 	registerPeerRESTHandlers(router)
@@ -34,7 +34,7 @@ func registerDistErasureRouters(router *mux.Router, endpointZones EndpointZones)
 	registerBootstrapRESTHandlers(router)
 
 	// Register distributed namespace lock routers.
-	registerLockRESTHandlers(router, endpointZones)
+	registerLockRESTHandlers(router, endpointServerSets)
 }
 
 // List of some generic handlers which are applied for all incoming requests.
@@ -79,14 +79,14 @@ var globalHandlers = []MiddlewareFunc{
 }
 
 // configureServer handler returns final handler for the http server.
-func configureServerHandler(endpointZones EndpointZones) (http.Handler, error) {
+func configureServerHandler(endpointServerSets EndpointServerSets) (http.Handler, error) {
 	// Initialize router. `SkipClean(true)` stops gorilla/mux from
 	// normalizing URL path minio/minio#3256
 	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
 
 	// Initialize distributed NS lock.
 	if globalIsDistErasure {
-		registerDistErasureRouters(router, endpointZones)
+		registerDistErasureRouters(router, endpointServerSets)
 	}
 
 	// Add STS router always.

cmd/server-main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"os"
 	"os/signal"
@@ -32,6 +33,7 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/bucket/bandwidth"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/env"
@@ -97,6 +99,9 @@ func serverCmdArgs(ctx *cli.Context) []string {
 		v = env.Get(config.EnvEndpoints, "")
 	}
 	if v == "" {
+		if !ctx.Args().Present() || ctx.Args().First() == "help" {
+			cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1)
+		}
 		return ctx.Args()
 	}
 	return strings.Fields(v)
@@ -158,6 +163,9 @@ func newAllSubsystems() {
 	// Create new bucket metadata system.
 	globalBucketMetadataSys = NewBucketMetadataSys()
 
+	// Create the bucket bandwidth monitor
+	globalBucketMonitor = bandwidth.NewMonitor(GlobalServiceDoneCh)
+
 	// Create a new config system.
 	globalConfigSys = NewConfigSys()
 
@@ -186,7 +194,7 @@ func newAllSubsystems() {
 	globalBucketTargetSys = NewBucketTargetSys()
 }
 
-func initServer(ctx context.Context, newObject ObjectLayer) (err error) {
+func initServer(ctx context.Context, newObject ObjectLayer) error {
 	// Create cancel context to control 'newRetryTimer' go routine.
 	retryCtx, cancel := context.WithCancel(ctx)
 
@@ -199,39 +207,6 @@ func initServer(ctx context.Context, newObject ObjectLayer) (err error) {
 	// appropriately. This is also true for rotation of encrypted
 	// content.
 	txnLk := newObject.NewNSLock(retryCtx, minioMetaBucket, minioConfigPrefix+"/transaction.lock")
-	defer func() {
-		if err != nil {
-			var cerr config.Err
-			// For any config error, we don't need to drop into safe-mode
-			// instead its a user error and should be fixed by user.
-			if errors.As(err, &cerr) {
-				logger.FatalIf(err, "Unable to initialize the server")
-				return
-			}
-
-			// If context was canceled
-			if errors.Is(err, context.Canceled) {
-				logger.FatalIf(err, "Server startup canceled upon user request")
-				return
-			}
-		}
-
-		// Prints the formatted startup message, if err is not nil then it prints additional information as well.
-		printStartupMessage(getAPIEndpoints(), err)
-
-		if globalActiveCred.Equal(auth.DefaultCredentials) {
-			msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ACCESS_KEY' and 'MINIO_SECRET_KEY'", globalActiveCred)
-			logger.StartupMessage(color.RedBold(msg))
-		}
-
-		<-globalOSSignalCh
-	}()
-
-	// Enable background operations for erasure coding
-	if globalIsErasure {
-		initAutoHeal(ctx, newObject)
-		initBackgroundReplication(ctx, newObject)
-	}
 
 	// allocate dynamic timeout once before the loop
 	configLockTimeout := newDynamicTimeout(5*time.Second, 3*time.Second)
@@ -248,7 +223,9 @@ func initServer(ctx context.Context, newObject ObjectLayer) (err error) {
 	//    version is needed, migration is needed etc.
 	rquorum := InsufficientReadQuorum{}
 	wquorum := InsufficientWriteQuorum{}
-	for range retry.NewTimerWithJitter(retryCtx, 250*time.Millisecond, 500*time.Millisecond, retry.MaxJitter) {
+
+	var err error
+	for range retry.NewTimerWithJitter(retryCtx, 500*time.Millisecond, time.Second, retry.MaxJitter) {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
 		if err = txnLk.GetLock(configLockTimeout); err != nil {
@@ -385,7 +362,12 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 
 // serverMain handler called for 'minio server' command.
 func serverMain(ctx *cli.Context) {
-	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
+	defer globalDNSCache.Stop()
+
+	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 
 	go handleSignals()
 
@@ -505,19 +487,47 @@ func serverMain(ctx *cli.Context) {
 
 	go initDataCrawler(GlobalContext, newObject)
 
-	// Initialize users credentials and policies in background.
-	go startBackgroundIAMLoad(GlobalContext, newObject)
+	// Enable background operations for erasure coding
+	if globalIsErasure {
+		initAutoHeal(GlobalContext, newObject)
+		initBackgroundReplication(GlobalContext, newObject)
+	}
 
-	initServer(GlobalContext, newObject)
+	if err = initServer(GlobalContext, newObject); err != nil {
+		var cerr config.Err
+		// For any config error, we don't need to drop into safe-mode
+		// instead its a user error and should be fixed by user.
+		if errors.As(err, &cerr) {
+			logger.FatalIf(err, "Unable to initialize the server")
+		}
+
+		// If context was canceled
+		if errors.Is(err, context.Canceled) {
+			logger.FatalIf(err, "Server startup canceled upon user request")
+		}
+	}
+
+	// Initialize users credentials and policies in background right after config has initialized.
+	go globalIAMSys.Init(GlobalContext, newObject)
+
+	// Prints the formatted startup message, if err is not nil then it prints additional information as well.
+	printStartupMessage(getAPIEndpoints(), err)
+
+	if globalActiveCred.Equal(auth.DefaultCredentials) {
+		msg := fmt.Sprintf("Detected default credentials '%s', please change the credentials immediately using 'MINIO_ACCESS_KEY' and 'MINIO_SECRET_KEY'", globalActiveCred)
+		logger.StartupMessage(color.RedBold(msg))
+	}
+
+	<-globalOSSignalCh
 }
 
 // Initialize object layer with the supplied disks, objectLayer is nil upon any error.
-func newObjectLayer(ctx context.Context, endpointZones EndpointZones) (newObject ObjectLayer, err error) {
+func newObjectLayer(ctx context.Context, endpointServerSets EndpointServerSets) (newObject ObjectLayer, err error) {
 	// For FS only, directly use the disk.
-	if endpointZones.NEndpoints() == 1 {
+	if endpointServerSets.NEndpoints() == 1 {
 		// Initialize new FS object layer.
-		return NewFSObjectLayer(endpointZones[0].Endpoints[0].Path)
+		return NewFSObjectLayer(endpointServerSets[0].Endpoints[0].Path)
 	}
 
-	return newErasureZones(ctx, endpointZones)
+	return newErasureServerSets(ctx, endpointServerSets)
 }

cmd/server-main_test.go

@@ -58,7 +58,7 @@ func TestNewObjectLayer(t *testing.T) {
 		t.Fatal("Unexpected object layer initialization error", err)
 	}
 
-	_, ok = obj.(*erasureZones)
+	_, ok = obj.(*erasureServerSets)
 	if !ok {
 		t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj))
 	}

cmd/storage-rest-client.go

@@ -121,9 +121,6 @@ type storageRESTClient struct {
 // permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *storageRESTClient) call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) {
-	if !client.IsOnline() {
-		return nil, errDiskNotFound
-	}
 	if values == nil {
 		values = make(url.Values)
 	}
@@ -134,7 +131,6 @@ func (client *storageRESTClient) call(ctx context.Context, method string, values
 	}
 
 	err = toStorageErr(err)
-
 	return nil, err
 }
 

cmd/storage-rest-server.go

@@ -890,8 +890,8 @@ func logFatalErrs(err error, endpoint Endpoint, exit bool) {
 }
 
 // registerStorageRPCRouter - register storage rpc router.
-func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
-	for _, ep := range endpointZones {
+func registerStorageRESTHandlers(router *mux.Router, endpointServerSets EndpointServerSets) {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if !endpoint.IsLocal {
 				continue

cmd/test-utils_test.go

@@ -58,6 +58,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bucket/policy"
@@ -99,6 +100,8 @@ func init() {
 
 	logger.Disable = true
 
+	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
+
 	initHelp()
 
 	resetTestGlobals()
@@ -286,7 +289,7 @@ func isSameType(obj1, obj2 interface{}) bool {
 //   defer s.Stop()
 type TestServer struct {
 	Root      string
-	Disks     EndpointZones
+	Disks     EndpointServerSets
 	AccessKey string
 	SecretKey string
 	Server    *httptest.Server
@@ -403,7 +406,7 @@ func resetGlobalConfig() {
 }
 
 func resetGlobalEndpoints() {
-	globalEndpoints = EndpointZones{}
+	globalEndpoints = EndpointServerSets{}
 }
 
 func resetGlobalIsErasure() {
@@ -1546,14 +1549,14 @@ func getRandomDisks(N int) ([]string, error) {
 }
 
 // Initialize object layer with the supplied disks, objectLayer is nil upon any error.
-func newTestObjectLayer(ctx context.Context, endpointZones EndpointZones) (newObject ObjectLayer, err error) {
+func newTestObjectLayer(ctx context.Context, endpointServerSets EndpointServerSets) (newObject ObjectLayer, err error) {
 	// For FS only, directly use the disk.
-	if endpointZones.NEndpoints() == 1 {
+	if endpointServerSets.NEndpoints() == 1 {
 		// Initialize new FS object layer.
-		return NewFSObjectLayer(endpointZones[0].Endpoints[0].Path)
+		return NewFSObjectLayer(endpointServerSets[0].Endpoints[0].Path)
 	}
 
-	z, err := newErasureZones(ctx, endpointZones)
+	z, err := newErasureServerSets(ctx, endpointServerSets)
 	if err != nil {
 		return nil, err
 	}
@@ -1566,16 +1569,16 @@ func newTestObjectLayer(ctx context.Context, endpointZones EndpointZones) (newOb
 }
 
 // initObjectLayer - Instantiates object layer and returns it.
-func initObjectLayer(ctx context.Context, endpointZones EndpointZones) (ObjectLayer, []StorageAPI, error) {
-	objLayer, err := newTestObjectLayer(ctx, endpointZones)
+func initObjectLayer(ctx context.Context, endpointServerSets EndpointServerSets) (ObjectLayer, []StorageAPI, error) {
+	objLayer, err := newTestObjectLayer(ctx, endpointServerSets)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	var formattedDisks []StorageAPI
 	// Should use the object layer tests for validating cache.
-	if z, ok := objLayer.(*erasureZones); ok {
-		formattedDisks = z.zones[0].GetDisks(0)()
+	if z, ok := objLayer.(*erasureServerSets); ok {
+		formattedDisks = z.serverSets[0].GetDisks(0)()
 	}
 
 	// Success.
@@ -2212,7 +2215,7 @@ func generateTLSCertKey(host string) ([]byte, []byte, error) {
 	return certOut.Bytes(), keyOut.Bytes(), nil
 }
 
-func mustGetZoneEndpoints(args ...string) EndpointZones {
+func mustGetZoneEndpoints(args ...string) EndpointServerSets {
 	endpoints := mustGetNewEndpoints(args...)
 	return []ZoneEndpoints{{
 		SetCount:     1,
@@ -2227,8 +2230,8 @@ func mustGetNewEndpoints(args ...string) (endpoints Endpoints) {
 	return endpoints
 }
 
-func getEndpointsLocalAddr(endpointZones EndpointZones) string {
-	for _, endpoints := range endpointZones {
+func getEndpointsLocalAddr(endpointServerSets EndpointServerSets) string {
+	for _, endpoints := range endpointServerSets {
 		for _, endpoint := range endpoints.Endpoints {
 			if endpoint.IsLocal && endpoint.Type() == URLEndpointType {
 				return endpoint.Host

cmd/utils.go

@@ -468,9 +468,9 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
 	// https://golang.org/pkg/net/http/#Transport documentation
 	tr := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewInternodeDialContext(dialTimeout),
-		MaxIdleConnsPerHost:   16,
-		IdleConnTimeout:       30 * time.Second,
+		DialContext:           xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
+		MaxIdleConnsPerHost:   1024,
+		IdleConnTimeout:       15 * time.Second,
 		ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
 		TLSHandshakeTimeout:   15 * time.Second,
 		ExpectContinueTimeout: 15 * time.Second,
@@ -490,15 +490,16 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
 	}
 }
 
-func newCustomHTTP11Transport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
+// Used by only proxied requests, specifically only supports HTTP/1.1
+func newCustomHTTPProxyTransport(tlsConfig *tls.Config, dialTimeout time.Duration) func() *http.Transport {
 	// For more details about various values used here refer
 	// https://golang.org/pkg/net/http/#Transport documentation
 	tr := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewCustomDialContext(dialTimeout),
-		MaxIdleConnsPerHost:   16,
-		IdleConnTimeout:       1 * time.Minute,
-		ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
+		DialContext:           xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
+		MaxIdleConnsPerHost:   1024,
+		IdleConnTimeout:       15 * time.Second,
+		ResponseHeaderTimeout: 30 * time.Minute, // Set larger timeouts for proxied requests.
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 10 * time.Second,
 		TLSClientConfig:       tlsConfig,
@@ -518,9 +519,9 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
 	// https://golang.org/pkg/net/http/#Transport documentation
 	tr := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewCustomDialContext(dialTimeout),
-		MaxIdleConnsPerHost:   16,
-		IdleConnTimeout:       1 * time.Minute,
+		DialContext:           xhttp.DialContextWithDNSCache(globalDNSCache, xhttp.NewInternodeDialContext(dialTimeout)),
+		MaxIdleConnsPerHost:   1024,
+		IdleConnTimeout:       15 * time.Second,
 		ResponseHeaderTimeout: 3 * time.Minute, // Set conservative timeouts for MinIO internode.
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 10 * time.Second,
@@ -553,9 +554,8 @@ func newGatewayHTTPTransport(timeout time.Duration) *http.Transport {
 		RootCAs: globalRootCAs,
 	}, defaultDialTimeout)()
 
-	// Allow more requests to be in flight.
+	// Customize response header timeout for gateway transport.
 	tr.ResponseHeaderTimeout = timeout
-	tr.MaxIdleConnsPerHost = 16
 	return tr
 }
 

cmd/web-handlers_test.go

@@ -1224,17 +1224,17 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
 	}
 
 	// Set faulty disks to Erasure backend
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 	erasureDisks := xl.getDisks()
-	z.zones[0].erasureDisksMu.Lock()
+	z.serverSets[0].erasureDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		for i, d := range erasureDisks {
 			erasureDisks[i] = newNaughtyDisk(d, nil, errFaultyDisk)
 		}
 		return erasureDisks
 	}
-	z.zones[0].erasureDisksMu.Unlock()
+	z.serverSets[0].erasureDisksMu.Unlock()
 
 	// Initialize web rpc endpoint.
 	apiRouter := initTestWebRPCEndPoint(obj)

cmd/xl-storage.go

@@ -370,9 +370,9 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
 	if objAPI == nil {
 		return cache, errServerNotInitialized
 	}
-	opts := globalCrawlerConfig
+	opts := globalHealConfig
 
-	dataUsageInfo, err := crawlDataFolder(ctx, s.diskPath, cache, s.waitForLowActiveIO, func(item crawlItem) (int64, error) {
+	dataUsageInfo, err := crawlDataFolder(ctx, s.diskPath, cache, func(item crawlItem) (int64, error) {
 		// Look for `xl.meta/xl.json' at the leaf.
 		if !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFile) &&
 			!strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFileV1) {
@@ -414,7 +414,10 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
 					err := s.VerifyFile(ctx, item.bucket, item.objectPath(), version)
 					switch err {
 					case errFileCorrupt:
-						res, err := objAPI.HealObject(ctx, item.bucket, item.objectPath(), oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling, ScanMode: madmin.HealDeepScan})
+						res, err := objAPI.HealObject(ctx, item.bucket, item.objectPath(), oi.VersionID, madmin.HealOpts{
+							Remove:   healDeleteDangling,
+							ScanMode: madmin.HealDeepScan,
+						})
 						if err != nil {
 							if !errors.Is(err, NotImplemented{}) {
 								logger.LogIf(ctx, err)

docs/config/README.md

@@ -179,18 +179,19 @@ KEY:
 api  manage global HTTP API call specific features, such as throttling, authentication types, etc.
 
 ARGS:
-requests_max       (number)    set the maximum number of concurrent requests, e.g. "1600"
-requests_deadline  (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
-ready_deadline     (duration)  set the deadline for health check API /minio/health/ready e.g. "1m"
-cors_allow_origin  (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
+requests_max               (number)    set the maximum number of concurrent requests, e.g. "1600"
+requests_deadline          (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
+cors_allow_origin          (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
+remote_transport_deadline  (duration)  set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"
 ```
 
 or environment variables
 
 ```
-MINIO_API_REQUESTS_MAX       (number)    set the maximum number of concurrent requests, e.g. "1600"
-MINIO_API_REQUESTS_DEADLINE  (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
-MINIO_API_CORS_ALLOW_ORIGIN  (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
+MINIO_API_REQUESTS_MAX               (number)    set the maximum number of concurrent requests, e.g. "1600"
+MINIO_API_REQUESTS_DEADLINE          (duration)  set the deadline for API requests waiting to be processed e.g. "1m"
+MINIO_API_CORS_ALLOW_ORIGIN          (csv)       set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"
+MINIO_API_REMOTE_TRANSPORT_DEADLINE  (duration)  set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"
 ```
 
 #### Notifications

docs/distributed/DESIGN.md

@@ -94,37 +94,37 @@ Input for the key is the object name specified in `PutObject()`, returns a uniqu
 
 - MinIO does erasure coding at the object level not at the volume level, unlike other object storage vendors. This allows applications to choose different storage class by setting `x-amz-storage-class=STANDARD/REDUCED_REDUNDANCY` for each object uploads so effectively utilizing the capacity of the cluster. Additionally these can also be enforced using IAM policies to make sure the client uploads with correct HTTP headers.
 
-- MinIO also supports expansion of existing clusters in zones. Each zone is a self contained entity with same SLA's (read/write quorum) for each object as original cluster. By using the existing namespace for lookup validation MinIO ensures conflicting objects are not created. When no such object exists then MinIO simply uses the least used zone.
+- MinIO also supports expansion of existing clusters in server sets. Each zone is a self contained entity with same SLA's (read/write quorum) for each object as original cluster. By using the existing namespace for lookup validation MinIO ensures conflicting objects are not created. When no such object exists then MinIO simply uses the least used zone.
 
-__There are no limits on how many zones can be combined__
+__There are no limits on how many server sets can be combined__
 
 ```
 minio server http://host{1...32}/export{1...32} http://host{5...6}/export{1...8}
 ```
 
-In above example there are two zones
+In above example there are two server sets
 
 - 32 * 32 = 1024 drives zone1
 - 2 * 8 = 16 drives zone2
 
 > Notice the requirement of common SLA here original cluster had 1024 drives with 16 drives per erasure set, second zone is expected to have a minimum of 16 drives to match the original cluster SLA or it should be in multiples of 16.
 
-MinIO places new objects in zones based on proportionate free space, per zone. Following pseudo code demonstrates this behavior.
+MinIO places new objects in server sets based on proportionate free space, per zone. Following pseudo code demonstrates this behavior.
 ```go
 func getAvailableZoneIdx(ctx context.Context) int {
-        zones := z.getZonesAvailableSpace(ctx)
-        total := zones.TotalAvailable()
+        serverSets := z.getServerSetsAvailableSpace(ctx)
+        total := serverSets.TotalAvailable()
         // choose when we reach this many
         choose := rand.Uint64() % total
         atTotal := uint64(0)
-        for _, zone := range zones {
+        for _, zone := range serverSets {
                 atTotal += zone.Available
                 if atTotal > choose && zone.Available > 0 {
                         return zone.Index
                 }
         }
         // Should not happen, but print values just in case.
-        panic(fmt.Errorf("reached end of zones (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
+        panic(fmt.Errorf("reached end of serverSets (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
 }
 ```
 

docs/distributed/README.md

@@ -77,10 +77,10 @@ For example:
 minio server http://host{1...4}/export{1...16} http://host{5...12}/export{1...16}
 ```
 
-Now the server has expanded total storage by _(newly_added_servers\*m)_ more disks, taking the total count to _(existing_servers\*m)+(newly_added_servers\*m)_ disks. New object upload requests automatically start using the least used cluster. This expansion strategy works endlessly, so you can perpetually expand your clusters as needed.  When you restart, it is immediate and non-disruptive to the applications. Each group of servers in the command-line is called a zone. There are 2 zones in this example. New objects are placed in zones in proportion to the amount of free space in each zone. Within each zone, the location of the erasure-set of drives is determined based on a deterministic hashing algorithm.
+Now the server has expanded total storage by _(newly_added_servers\*m)_ more disks, taking the total count to _(existing_servers\*m)+(newly_added_servers\*m)_ disks. New object upload requests automatically start using the least used cluster. This expansion strategy works endlessly, so you can perpetually expand your clusters as needed.  When you restart, it is immediate and non-disruptive to the applications. Each group of servers in the command-line is called a zone. There are 2 server sets in this example. New objects are placed in server sets in proportion to the amount of free space in each zone. Within each zone, the location of the erasure-set of drives is determined based on a deterministic hashing algorithm.
 
 > __NOTE:__ __Each zone you add must have the same erasure coding set size as the original zone, so the same data redundancy SLA is maintained.__
-> For example, if your first zone was 8 drives, you could add further zones of 16, 32 or 1024 drives each. All you have to make sure is deployment SLA is multiples of original data redundancy SLA i.e 8.
+> For example, if your first zone was 8 drives, you could add further server sets of 16, 32 or 1024 drives each. All you have to make sure is deployment SLA is multiples of original data redundancy SLA i.e 8.
 
 ## 3. Test your setup
 To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/docs/minio-client-quickstart-guide).

docs/metrics/healthcheck/README.md

@@ -1,22 +1,22 @@
 ## MinIO Healthcheck
 
-MinIO server exposes three un-authenticated, healthcheck endpoints liveness probe and a cluster probe at `/minio/health/live`, `/minio/health/ready` and `/minio/health/cluster` respectively.
+MinIO server exposes three un-authenticated, healthcheck endpoints liveness probe and a cluster probe at `/minio/health/live` and `/minio/health/cluster` respectively.
 
 ### Liveness probe
 
 This probe always responds with '200 OK'. When liveness probe fails, Kubernetes like platforms restart the container.
 
 ```
-          livenessProbe:
-            httpGet:
-              path: /minio/health/live
-              port: 9000
-              scheme: HTTP
-            initialDelaySeconds: 120
-            periodSeconds: 15
-            timeoutSeconds: 10
-            successThreshold: 1
-            failureThreshold: 3
+livenessProbe:
+  httpGet:
+    path: /minio/health/live
+    port: 9000
+    scheme: HTTP
+  initialDelaySeconds: 120
+  periodSeconds: 15
+  timeoutSeconds: 10
+  successThreshold: 1
+  failureThreshold: 3
 ```
 
 ### Cluster probe

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # it through port 9000.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2020-10-12T21-53-21Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/sts/dex.md

@@ -4,7 +4,7 @@ Dex is an identity service that uses OpenID Connect to drive authentication for
 
 ## Prerequisites
 
-Install Dex by following [Dex Getting Started Guide](https://github.com/dexidp/dex/blob/master/Documentation/getting-started.md)
+Install Dex by following [Dex Getting Started Guide](https://dexidp.io/docs/getting-started/)
 
 ### Start Dex
 

docs/zh_CN/distributed/DESIGN.md

@@ -113,19 +113,19 @@ minio server http://host{1...32}/export{1...32} http://host{5...6}/export{1...8}
 MinIO根据每个区域的可用空间比例将新对象放置在区域中。以下伪代码演示了此行为。
 ```go
 func getAvailableZoneIdx(ctx context.Context) int {
-        zones := z.getZonesAvailableSpace(ctx)
-        total := zones.TotalAvailable()
+        serverSets := z.getServerSetsAvailableSpace(ctx)
+        total := serverSets.TotalAvailable()
         // choose when we reach this many
         choose := rand.Uint64() % total
         atTotal := uint64(0)
-        for _, zone := range zones {
+        for _, zone := range serverSets {
                 atTotal += zone.Available
                 if atTotal > choose && zone.Available > 0 {
                         return zone.Index
                 }
         }
         // Should not happen, but print values just in case.
-        panic(fmt.Errorf("reached end of zones (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
+        panic(fmt.Errorf("reached end of serverSets (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
 }
 ```
 

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/alecthomas/participle v0.2.1
 	github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2
-	github.com/beevik/ntp v0.2.0
+	github.com/beevik/ntp v0.3.0
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/cheggaaa/pb v1.0.28
 	github.com/colinmarc/hdfs/v2 v2.1.1

go.sum

@@ -49,8 +49,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
 github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM=
 github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M=
-github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw=
-github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
+github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw=
+github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=

mint/preinstall.sh

@@ -38,7 +38,7 @@ $APT update
 $APT install gnupg ca-certificates
 
 # download and install golang
-GO_VERSION="1.14.7"
+GO_VERSION="1.15.2"
 GO_INSTALL_PATH="/usr/local"
 download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
 if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then

pkg/bandwidth/bandwidth.go

@@ -0,0 +1,28 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bandwidth
+
+// Details for the measured bandwidth
+type Details struct {
+	LimitInBytesPerSecond            int64   `json:"limitInBits"`
+	CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"`
+}
+
+// Report captures the details for all buckets.
+type Report struct {
+	BucketStats map[string]Details `json:"bucketStats,omitempty"`
+}

pkg/bucket/bandwidth/measurement.go

@@ -0,0 +1,87 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bandwidth
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	// betaBucket is the weight used to calculate exponential moving average
+	betaBucket = 0.1 // Number of averages considered = 1/(1-betaObject)
+)
+
+// bucketMeasurement captures the bandwidth details for one bucket
+type bucketMeasurement struct {
+	lock                 sync.Mutex
+	bytesSinceLastWindow uint64    // Total bytes since last window was processed
+	startTime            time.Time // Start time for window
+	expMovingAvg         float64   // Previously calculate sliding window
+}
+
+// newBucketMeasurement creates a new instance of the measurement with the initial start time.
+func newBucketMeasurement(initTime time.Time) *bucketMeasurement {
+	return &bucketMeasurement{
+		startTime: initTime,
+	}
+}
+
+// incrementBytes add bytes reported for a bucket.
+func (m *bucketMeasurement) incrementBytes(bytes uint64) {
+	atomic.AddUint64(&m.bytesSinceLastWindow, bytes)
+}
+
+// updateExponentialMovingAverage processes the measurements captured so far.
+func (m *bucketMeasurement) updateExponentialMovingAverage(endTime time.Time) {
+	// Calculate aggregate avg bandwidth and exp window avg
+	m.lock.Lock()
+	defer func() {
+		m.startTime = endTime
+		m.lock.Unlock()
+	}()
+
+	if endTime.Before(m.startTime) {
+		return
+	}
+
+	duration := endTime.Sub(m.startTime)
+
+	bytesSinceLastWindow := atomic.SwapUint64(&m.bytesSinceLastWindow, 0)
+
+	if m.expMovingAvg == 0 {
+		// Should address initial calculation and should be fine for resuming from 0
+		m.expMovingAvg = float64(bytesSinceLastWindow) / duration.Seconds()
+		return
+	}
+
+	increment := float64(bytesSinceLastWindow) / duration.Seconds()
+	m.expMovingAvg = exponentialMovingAverage(betaBucket, m.expMovingAvg, increment)
+}
+
+// exponentialMovingAverage calculates the exponential moving average
+func exponentialMovingAverage(beta, previousAvg, incrementAvg float64) float64 {
+	return (1-beta)*incrementAvg + beta*previousAvg
+}
+
+// getExpMovingAvgBytesPerSecond returns the exponential moving average for the bucket in bytes
+func (m *bucketMeasurement) getExpMovingAvgBytesPerSecond() float64 {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	return m.expMovingAvg
+}

pkg/bucket/bandwidth/monitor.go

@@ -0,0 +1,180 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bandwidth
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/pkg/bandwidth"
+	"github.com/minio/minio/pkg/pubsub"
+)
+
+// throttleBandwidth gets the throttle for bucket with the configured value
+func (m *Monitor) throttleBandwidth(ctx context.Context, bucket string, bandwidthBytesPerSecond int64, clusterBandwidth int64) *throttle {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	throttle, ok := m.bucketThrottle[bucket]
+	if !ok {
+		throttle = newThrottle(ctx, bandwidthBytesPerSecond, clusterBandwidth)
+		m.bucketThrottle[bucket] = throttle
+		return throttle
+	}
+	throttle.SetBandwidth(bandwidthBytesPerSecond, clusterBandwidth)
+	return throttle
+}
+
+// SubscribeToBuckets subscribes to buckets. Empty array for monitoring all buckets.
+func (m *Monitor) SubscribeToBuckets(subCh chan interface{}, doneCh <-chan struct{}, buckets []string) {
+	m.pubsub.Subscribe(subCh, doneCh, func(f interface{}) bool {
+		if buckets != nil || len(buckets) == 0 {
+			return true
+		}
+		report, ok := f.(*bandwidth.Report)
+		if !ok {
+			return false
+		}
+		for _, b := range buckets {
+			_, ok := report.BucketStats[b]
+			if ok {
+				return true
+			}
+		}
+		return false
+	})
+}
+
+// Monitor implements the monitoring for bandwidth measurements.
+type Monitor struct {
+	lock sync.Mutex // lock for all updates
+
+	activeBuckets map[string]*bucketMeasurement // Buckets with objects in flight
+
+	bucketMovingAvgTicker *time.Ticker // Ticker for calculating moving averages
+
+	pubsub *pubsub.PubSub // PubSub for reporting bandwidths.
+
+	bucketThrottle map[string]*throttle
+
+	startProcessing sync.Once
+
+	doneCh <-chan struct{}
+}
+
+// NewMonitor returns a monitor with defaults.
+func NewMonitor(doneCh <-chan struct{}) *Monitor {
+	m := &Monitor{
+		activeBuckets:         make(map[string]*bucketMeasurement),
+		bucketMovingAvgTicker: time.NewTicker(2 * time.Second),
+		pubsub:                pubsub.New(),
+		bucketThrottle:        make(map[string]*throttle),
+		doneCh:                doneCh,
+	}
+	return m
+}
+
+// SelectionFunction for buckets
+type SelectionFunction func(bucket string) bool
+
+// SelectBuckets will select all the buckets passed in.
+func SelectBuckets(buckets ...string) SelectionFunction {
+	if len(buckets) == 0 {
+		return func(bucket string) bool {
+			return true
+		}
+	}
+	return func(bucket string) bool {
+		for _, b := range buckets {
+			if b == "" || b == bucket {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+// GetReport gets the report for all bucket bandwidth details.
+func (m *Monitor) GetReport(selectBucket SelectionFunction) *bandwidth.Report {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	return m.getReport(selectBucket)
+}
+
+func (m *Monitor) getReport(selectBucket SelectionFunction) *bandwidth.Report {
+	report := &bandwidth.Report{
+		BucketStats: make(map[string]bandwidth.Details),
+	}
+	for bucket, bucketMeasurement := range m.activeBuckets {
+		if !selectBucket(bucket) {
+			continue
+		}
+		report.BucketStats[bucket] = bandwidth.Details{
+			LimitInBytesPerSecond:            m.bucketThrottle[bucket].clusterBandwidth,
+			CurrentBandwidthInBytesPerSecond: bucketMeasurement.getExpMovingAvgBytesPerSecond(),
+		}
+	}
+	return report
+}
+
+func (m *Monitor) process(doneCh <-chan struct{}) {
+	for {
+		select {
+		case <-m.bucketMovingAvgTicker.C:
+			m.processAvg()
+		case <-doneCh:
+			return
+		}
+	}
+}
+
+func (m *Monitor) getBucketMeasurement(bucket string, initTime time.Time) *bucketMeasurement {
+	bucketTracker, ok := m.activeBuckets[bucket]
+	if !ok {
+		bucketTracker = newBucketMeasurement(initTime)
+		m.activeBuckets[bucket] = bucketTracker
+	}
+	return bucketTracker
+}
+
+func (m *Monitor) processAvg() {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	for _, bucketMeasurement := range m.activeBuckets {
+		bucketMeasurement.updateExponentialMovingAverage(time.Now())
+	}
+	m.pubsub.Publish(m.getReport(SelectBuckets()))
+}
+
+// track returns the measurement object for bucket and object
+func (m *Monitor) track(bucket string, object string, timeNow time.Time) *bucketMeasurement {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.startProcessing.Do(func() {
+		go m.process(m.doneCh)
+	})
+	b := m.getBucketMeasurement(bucket, timeNow)
+	return b
+}
+
+// DeleteBucket deletes monitoring the 'bucket'
+func (m *Monitor) DeleteBucket(bucket string) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	delete(m.activeBuckets, bucket)
+	delete(m.bucketThrottle, bucket)
+}

pkg/bucket/bandwidth/monitor_test.go

@@ -0,0 +1,158 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bandwidth
+
+import (
+	"context"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/pkg/bandwidth"
+)
+
+const (
+	oneMiB uint64 = 1024 * 1024
+)
+
+func TestMonitor_GetThrottle(t *testing.T) {
+	type fields struct {
+		bucketThrottles map[string]*throttle
+		bucket          string
+		bpi             int64
+	}
+	t1 := newThrottle(context.Background(), 100, 1024*1024)
+	t2 := newThrottle(context.Background(), 200, 1024*1024)
+	tests := []struct {
+		name   string
+		fields fields
+		want   *throttle
+	}{
+		{
+			name: "Existing",
+			fields: fields{
+				bucketThrottles: map[string]*throttle{"bucket": t1},
+				bucket:          "bucket",
+				bpi:             100,
+			},
+			want: t1,
+		},
+		{
+			name: "new",
+			fields: fields{
+				bucketThrottles: map[string]*throttle{"bucket": t1},
+				bucket:          "bucket2",
+				bpi:             200,
+			},
+			want: t2,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			m := &Monitor{
+				bucketThrottle: tt.fields.bucketThrottles,
+			}
+			if got := m.throttleBandwidth(context.Background(), tt.fields.bucket, tt.fields.bpi, 1024*1024); got.bytesPerInterval != tt.want.bytesPerInterval {
+				t.Errorf("throttleBandwidth() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestMonitor_GetReport(t *testing.T) {
+	type fields struct {
+		activeBuckets map[string]*bucketMeasurement
+		endTime       time.Time
+		update2       uint64
+		endTime2      time.Time
+	}
+	start := time.Now()
+	m0 := newBucketMeasurement(start)
+	m0.incrementBytes(0)
+	m1MiBPS := newBucketMeasurement(start)
+	m1MiBPS.incrementBytes(oneMiB)
+	tests := []struct {
+		name   string
+		fields fields
+		want   *bandwidth.Report
+		want2  *bandwidth.Report
+	}{
+		{
+			name: "ZeroToOne",
+			fields: fields{
+				activeBuckets: map[string]*bucketMeasurement{
+					"bucket": m0,
+				},
+				endTime:  start.Add(1 * time.Second),
+				update2:  oneMiB,
+				endTime2: start.Add(2 * time.Second),
+			},
+			want: &bandwidth.Report{
+				BucketStats: map[string]bandwidth.Details{"bucket": {LimitInBytesPerSecond: 1024 * 1024, CurrentBandwidthInBytesPerSecond: 0}},
+			},
+			want2: &bandwidth.Report{
+				BucketStats: map[string]bandwidth.Details{"bucket": {LimitInBytesPerSecond: 1024 * 1024, CurrentBandwidthInBytesPerSecond: (1024 * 1024) / start.Add(2*time.Second).Sub(start.Add(1*time.Second)).Seconds()}},
+			},
+		},
+		{
+			name: "OneToTwo",
+			fields: fields{
+				activeBuckets: map[string]*bucketMeasurement{
+					"bucket": m1MiBPS,
+				},
+				endTime:  start.Add(1 * time.Second),
+				update2:  2 * oneMiB,
+				endTime2: start.Add(2 * time.Second),
+			},
+			want: &bandwidth.Report{
+				BucketStats: map[string]bandwidth.Details{"bucket": {LimitInBytesPerSecond: 1024 * 1024, CurrentBandwidthInBytesPerSecond: float64(oneMiB)}},
+			},
+			want2: &bandwidth.Report{
+				BucketStats: map[string]bandwidth.Details{"bucket": {
+					LimitInBytesPerSecond:            1024 * 1024,
+					CurrentBandwidthInBytesPerSecond: exponentialMovingAverage(betaBucket, float64(oneMiB), 2*float64(oneMiB))}},
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			thr := throttle{
+				bytesPerSecond:   1024 * 1024,
+				clusterBandwidth: 1024 * 1024,
+			}
+			m := &Monitor{
+				activeBuckets:  tt.fields.activeBuckets,
+				bucketThrottle: map[string]*throttle{"bucket": &thr},
+			}
+			m.activeBuckets["bucket"].updateExponentialMovingAverage(tt.fields.endTime)
+			got := m.GetReport(SelectBuckets())
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetReport() = %v, want %v", got, tt.want)
+			}
+			m.activeBuckets["bucket"].incrementBytes(tt.fields.update2)
+			m.activeBuckets["bucket"].updateExponentialMovingAverage(tt.fields.endTime2)
+			got = m.GetReport(SelectBuckets())
+			if !reflect.DeepEqual(got, tt.want2) {
+				t.Errorf("GetReport() = %v, want %v", got, tt.want2)
+			}
+		})
+	}
+}

pkg/bucket/bandwidth/reader.go

@@ -0,0 +1,86 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package bandwidth
+
+import (
+	"context"
+	"io"
+	"time"
+)
+
+// MonitoredReader monitors the bandwidth
+type MonitoredReader struct {
+	bucket            string             // Token to track bucket
+	bucketMeasurement *bucketMeasurement // bucket measurement object
+	object            string             // Token to track object
+	reader            io.Reader          // Reader to wrap
+	lastStop          time.Time          // Last timestamp for a measurement
+	headerSize        int                // Size of the header not captured by reader
+	throttle          *throttle          // throttle the rate at which replication occur
+	monitor           *Monitor           // Monitor reference
+	closed            bool               // Reader is closed
+}
+
+// NewMonitoredReader returns a io.ReadCloser that reports bandwidth details
+func NewMonitoredReader(ctx context.Context, monitor *Monitor, bucket string, object string, reader io.Reader, headerSize int, bandwidthBytesPerSecond int64, clusterBandwidth int64) *MonitoredReader {
+	timeNow := time.Now()
+	b := monitor.track(bucket, object, timeNow)
+	return &MonitoredReader{
+		bucket:            bucket,
+		object:            object,
+		bucketMeasurement: b,
+		reader:            reader,
+		lastStop:          timeNow,
+		headerSize:        headerSize,
+		throttle:          monitor.throttleBandwidth(ctx, bucket, bandwidthBytesPerSecond, clusterBandwidth),
+		monitor:           monitor,
+	}
+}
+
+// Read wraps the read reader
+func (m *MonitoredReader) Read(p []byte) (n int, err error) {
+	if m.closed {
+		err = io.ErrClosedPipe
+		return
+	}
+	p = p[:m.throttle.GetLimitForBytes(int64(len(p)))]
+
+	n, err = m.reader.Read(p)
+	stop := time.Now()
+	update := uint64(n + m.headerSize)
+
+	m.bucketMeasurement.incrementBytes(update)
+	m.lastStop = stop
+	unused := len(p) - (n + m.headerSize)
+	m.headerSize = 0 // Set to 0 post first read
+
+	if unused > 0 {
+		m.throttle.ReleaseUnusedBandwidth(int64(unused))
+	}
+	return
+}
+
+// Close stops tracking the io
+func (m *MonitoredReader) Close() error {
+	rc, ok := m.reader.(io.ReadCloser)
+	m.closed = true
+	if ok {
+		return rc.Close()
+	}
+	return nil
+}

pkg/bucket/bandwidth/throttle.go

@@ -0,0 +1,120 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bandwidth
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	throttleInternal = 250 * time.Millisecond
+)
+
+// throttle implements the throttling for bandwidth
+type throttle struct {
+	generateTicker   *time.Ticker    // Ticker to generate available bandwidth
+	freeBytes        int64           // unused bytes in the interval
+	bytesPerSecond   int64           // max limit for bandwidth
+	bytesPerInterval int64           // bytes allocated for the interval
+	clusterBandwidth int64           // Cluster wide bandwidth needed for reporting
+	cond             *sync.Cond      // Used to notify waiting threads for bandwidth availability
+	goGenerate       int64           // Flag to track if generate routine should be running. 0 == stopped
+	ctx              context.Context // Context for generate
+}
+
+// newThrottle returns a new bandwidth throttle. Set bytesPerSecond to 0 for no limit
+func newThrottle(ctx context.Context, bytesPerSecond int64, clusterBandwidth int64) *throttle {
+	if bytesPerSecond == 0 {
+		return &throttle{}
+	}
+	t := &throttle{
+		bytesPerSecond:   bytesPerSecond,
+		generateTicker:   time.NewTicker(throttleInternal),
+		clusterBandwidth: clusterBandwidth,
+		ctx:              ctx,
+	}
+
+	t.cond = sync.NewCond(&sync.Mutex{})
+	t.SetBandwidth(bytesPerSecond, clusterBandwidth)
+	t.freeBytes = t.bytesPerInterval
+	return t
+}
+
+// GetLimitForBytes gets the bytes that are possible to send within the limit
+// if want is <= 0 or no bandwidth limit set, returns want.
+// Otherwise a value > 0 will always be returned.
+func (t *throttle) GetLimitForBytes(want int64) int64 {
+	if want <= 0 || atomic.LoadInt64(&t.bytesPerInterval) == 0 {
+		return want
+	}
+	t.cond.L.Lock()
+	defer t.cond.L.Unlock()
+	for {
+		var send int64
+		freeBytes := atomic.LoadInt64(&t.freeBytes)
+		send = want
+		if freeBytes < want {
+			send = freeBytes
+			if send <= 0 {
+				t.cond.Wait()
+				continue
+			}
+		}
+		atomic.AddInt64(&t.freeBytes, -send)
+
+		// Bandwidth was consumed, start generate routine to allocate bandwidth
+		if atomic.CompareAndSwapInt64(&t.goGenerate, 0, 1) {
+			go t.generateBandwidth(t.ctx)
+		}
+		return send
+	}
+}
+
+// SetBandwidth sets a new bandwidth limit in bytes per second.
+func (t *throttle) SetBandwidth(bandwidthBiPS int64, clusterBandwidth int64) {
+	bpi := int64(throttleInternal) * bandwidthBiPS / int64(time.Second)
+	atomic.StoreInt64(&t.bytesPerInterval, bpi)
+}
+
+// ReleaseUnusedBandwidth releases bandwidth that was allocated for a user
+func (t *throttle) ReleaseUnusedBandwidth(bytes int64) {
+	atomic.AddInt64(&t.freeBytes, bytes)
+}
+
+// generateBandwidth periodically allocates new bandwidth to use
+func (t *throttle) generateBandwidth(ctx context.Context) {
+	for {
+		select {
+		case <-t.generateTicker.C:
+			if atomic.LoadInt64(&t.freeBytes) == atomic.LoadInt64(&t.bytesPerInterval) {
+				// No bandwidth consumption stop the routine.
+				atomic.StoreInt64(&t.goGenerate, 0)
+				return
+			}
+			// A new window is available
+			t.cond.L.Lock()
+			atomic.StoreInt64(&t.freeBytes, atomic.LoadInt64(&t.bytesPerInterval))
+			t.cond.Broadcast()
+			t.cond.L.Unlock()
+		case <-ctx.Done():
+			return
+		}
+	}
+}

pkg/console/console.go

@@ -353,13 +353,16 @@ type Table struct {
 
 	// Left margin width for table
 	TableIndentWidth int
+
+	// Flag to print separator under heading. Row 0 is considered heading
+	HeaderRowSeparator bool
 }
 
 // NewTable - create a new Table instance. Takes per-row colors and
 // per-column right-align flags and table indentation width (i.e. left
 // margin width)
 func NewTable(rowColors []*color.Color, alignRight []bool, indentWidth int) *Table {
-	return &Table{rowColors, alignRight, indentWidth}
+	return &Table{rowColors, alignRight, indentWidth, false}
 }
 
 // DisplayTable - prints the table
@@ -410,6 +413,11 @@ func (t *Table) DisplayTable(rows [][]string) error {
 
 	// Print the table with colors
 	for r, row := range paddedText {
+		if t.HeaderRowSeparator && r == 1 {
+			// Draw table header-row border
+			border = fmt.Sprintf("%s├%s┤", indentText, strings.Join(segments, "┼"))
+			fmt.Println(border)
+		}
 		fmt.Print(indentText + "│ ")
 		for c, text := range row {
 			t.RowColors[r].Print(text)

pkg/iam/policy/admin-action.go

@@ -49,6 +49,8 @@ const (
 	ServerInfoAdminAction = "admin:ServerInfo"
 	// OBDInfoAdminAction - allow obtaining cluster on-board diagnostics
 	OBDInfoAdminAction = "admin:OBDInfo"
+	// BandwidthMonitorAction - allow monitoring bandwidth usage
+	BandwidthMonitorAction = "admin:BandwidthMonitor"
 
 	// ServerUpdateAdminAction - allow MinIO binary update
 	ServerUpdateAdminAction = "admin:ServerUpdate"
@@ -131,6 +133,7 @@ var supportedAdminActions = map[AdminAction]struct{}{
 	KMSKeyStatusAdminAction:        {},
 	ServerInfoAdminAction:          {},
 	OBDInfoAdminAction:             {},
+	BandwidthMonitorAction:         {},
 	ServerUpdateAdminAction:        {},
 	ServiceRestartAdminAction:      {},
 	ServiceStopAdminAction:         {},
@@ -173,6 +176,7 @@ var adminActionConditionKeyMap = map[Action]condition.KeySet{
 	ServerInfoAdminAction:          condition.NewKeySet(condition.AllSupportedAdminKeys...),
 	DataUsageInfoAdminAction:       condition.NewKeySet(condition.AllSupportedAdminKeys...),
 	OBDInfoAdminAction:             condition.NewKeySet(condition.AllSupportedAdminKeys...),
+	BandwidthMonitorAction:         condition.NewKeySet(condition.AllSupportedAdminKeys...),
 	TopLocksAdminAction:            condition.NewKeySet(condition.AllSupportedAdminKeys...),
 	ProfilingAdminAction:           condition.NewKeySet(condition.AllSupportedAdminKeys...),
 	TraceAdminAction:               condition.NewKeySet(condition.AllSupportedAdminKeys...),

pkg/iam/policy/constants.go

@@ -75,7 +75,7 @@ var AdminDiagnostics = Policy{
 			Actions: NewActionSet(ProfilingAdminAction,
 				TraceAdminAction, ConsoleLogAdminAction,
 				ServerInfoAdminAction, TopLocksAdminAction,
-				OBDInfoAdminAction),
+				OBDInfoAdminAction, BandwidthMonitorAction),
 			Resources: NewResourceSet(NewResource("*", "")),
 		},
 	},

pkg/madmin/bandwidth.go

@@ -0,0 +1,61 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/minio/minio/pkg/bandwidth"
+)
+
+// GetBucketBandwidth - Get a snapshot of the bandwidth measurements for replication buckets. If no buckets
+// generate replication traffic an empty map is returned.
+func (adm *AdminClient) GetBucketBandwidth(ctx context.Context, buckets ...string) (bandwidth.Report, error) {
+	queryValues := url.Values{}
+	if len(buckets) > 0 {
+		queryValues.Set("buckets", strings.Join(buckets, ","))
+	}
+
+	reqData := requestData{
+		relPath:     adminAPIPrefix + "/bandwidth",
+		queryValues: queryValues,
+	}
+
+	resp, err := adm.executeMethod(ctx, http.MethodGet, reqData)
+	defer closeResponse(resp)
+	if err != nil {
+		return bandwidth.Report{}, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return bandwidth.Report{}, httpRespToErrorResponse(resp)
+	}
+	dec := json.NewDecoder(resp.Body)
+	for {
+		var report bandwidth.Report
+		err = dec.Decode(&report)
+		if err != nil && err != io.EOF {
+			return bandwidth.Report{}, err
+		}
+		return report, nil
+	}
+}

pkg/madmin/examples/bucket-bandwidth.go

@@ -0,0 +1,50 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
+	// New returns an MinIO Admin client object.
+	madminClient, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	ctx := context.Background()
+	report, err := madminClient.GetBucketBandwidth(ctx)
+	if err != nil {
+		log.Fatalln(err)
+		return
+	}
+	fmt.Printf("Report: %+v\n", report)
+	report, err = madminClient.GetBucketBandwidth(ctx, "sourceBucket", "sourceBucket2")
+	if err != nil {
+		log.Fatalln(err)
+		return
+	}
+	fmt.Printf("Report: %+v\n", report)
+}

pkg/madmin/examples/bucket-target.go

@@ -42,7 +42,7 @@ func main() {
 	if err != nil {
 		log.Fatalln(err)
 	}
-	target := madmin.BucketTarget{Endpoint: "site2:9000", Credentials: creds, TargetBucket: "destbucket", IsSSL: false, Type: madmin.ReplicationArn}
+	target := madmin.BucketTarget{Endpoint: "site2:9000", Credentials: creds, TargetBucket: "destbucket", IsSSL: false, Type: madmin.ReplicationArn, BandwidthLimit: 2 * 1024 * 1024}
 	// Set bucket target
 	if err := madmClnt.SetBucketTarget(ctx, "srcbucket", &target); err != nil {
 		log.Fatalln(err)

pkg/madmin/remote-target-commands.go

@@ -84,17 +84,18 @@ func ParseARN(s string) (*ARN, error) {
 
 // BucketTarget represents the target bucket and site association.
 type BucketTarget struct {
-	SourceBucket string            `json:"sourcebucket"`
-	Endpoint     string            `json:"endpoint"`
-	Credentials  *auth.Credentials `json:"credentials"`
-	TargetBucket string            `json:"targetbucket"`
-	Secure       bool              `json:"secure"`
-	Path         string            `json:"path,omitempty"`
-	API          string            `json:"api,omitempty"`
-	Arn          string            `json:"arn,omitempty"`
-	Type         ServiceType       `json:"type"`
-	Region       string            `json:"omitempty"`
-	Label        string            `json:"label,omitempty"`
+	SourceBucket   string            `json:"sourcebucket"`
+	Endpoint       string            `json:"endpoint"`
+	Credentials    *auth.Credentials `json:"credentials"`
+	TargetBucket   string            `json:"targetbucket"`
+	Secure         bool              `json:"secure"`
+	Path           string            `json:"path,omitempty"`
+	API            string            `json:"api,omitempty"`
+	Arn            string            `json:"arn,omitempty"`
+	Type           ServiceType       `json:"type"`
+	Region         string            `json:"omitempty"`
+	Label          string            `json:"label,omitempty"`
+	BandwidthLimit int64             `json:"bandwidthlimit,omitempty"`
 }
 
 // Clone returns shallow clone of BucketTarget without secret key in credentials

pkg/net/url.go

@@ -148,21 +148,24 @@ func IsNetworkOrHostDown(err error) bool {
 	if errors.Is(err, context.Canceled) {
 		return false
 	}
+
 	// We need to figure if the error either a timeout
 	// or a non-temporary error.
 	e, ok := err.(net.Error)
 	if ok {
-		urlErr, ok := e.(*url.Error)
-		if ok {
+		if urlErr, ok := e.(*url.Error); ok {
 			switch urlErr.Err.(type) {
 			case *net.DNSError, *net.OpError, net.UnknownNetworkError:
 				return true
 			}
 		}
+
 		if e.Timeout() {
 			return true
 		}
+
 	}
+
 	ok = false
 	// Fallback to other mechanisms.
 	if strings.Contains(err.Error(), "Connection closed by foreign host") {