Avoid running lock maintenance from all nodes (#8737)

Co-Authored-By: Krishnan Parthasarathi <krisis@users.noreply.github.com>

buildscripts/gateway-tests.sh

@@ -46,9 +46,7 @@ function main()
     gw_pid="$(start_minio_gateway_s3)"
 
     SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
-                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh aws-sdk-go \
-                   aws-sdk-java aws-sdk-php aws-sdk-ruby awscli healthcheck minio-dotnet \
-                   minio-go minio-java minio-js minio-py
+                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh
     rv=$?
 
     kill "$sr_pid"

cmd/admin-handlers-config-kv.go

@@ -427,6 +427,10 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	var s strings.Builder
 	hkvs := config.HelpSubSysMap[""]
+	var count int
+	for _, hkv := range hkvs {
+		count += len(cfg[hkv.Key])
+	}
 	for _, hkv := range hkvs {
 		v := cfg[hkv.Key]
 		for target, kv := range v {
@@ -460,7 +464,10 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			}
 			s.WriteString(config.KvSpaceSeparator)
 			s.WriteString(kv.String())
-			s.WriteString(config.KvNewline)
+			count--
+			if count > 0 {
+				s.WriteString(config.KvNewline)
+			}
 		}
 	}
 

cmd/admin-server-info.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"

cmd/bootstrap-peer-server.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"

cmd/bucket-handlers.go

@@ -278,7 +278,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 	// If etcd, dns federation configured list buckets from etcd.
 	var bucketsInfo []BucketInfo
-	if globalDNSConfig != nil {
+	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/bucket-notification-handlers.go

@@ -228,6 +228,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	bucketName := vars["bucket"]
 
 	values := r.URL.Query()
+	values.Set(peerRESTListenBucket, bucketName)
 
 	var prefix string
 	if len(values[peerRESTListenPrefix]) > 1 {
@@ -295,6 +296,9 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		if !ok {
 			return false
 		}
+		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
+			return false
+		}
 		objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
 		if uerr != nil {
 			objectName = ev.S3.Object.Key
@@ -306,7 +310,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		if peer == nil {
 			continue
 		}
-		peer.Listen(listenCh, doneCh, r.URL.Query())
+		peer.Listen(listenCh, doneCh, values)
 	}
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)

cmd/config-current.go

@@ -315,6 +315,13 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
+	// Bucket federation is 'true' only when IAM assets are not namespaced
+	// per tenant and all tenants interested in globally available users
+	// if namespace was requested such as specifying etcdPathPrefix then
+	// we assume that users are interested in global bucket support
+	// but not federation.
+	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
+
 	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 			dns.DomainNames(globalDomainNames),
@@ -415,6 +422,11 @@ func lookupConfigs(s config.Config) {
 			logger.AddAuditTarget(http.New(l.Endpoint, loggerUserAgent, string(logger.All), NewCustomHTTPTransport()))
 		}
 	}
+
+	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalServiceDoneCh, NewCustomHTTPTransport())
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
+	}
 }
 
 // Help - return sub-system level help

cmd/config-encrypted.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"strings"
 	"unicode/utf8"
@@ -39,7 +40,6 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 
 	// If its server mode or nas gateway, migrate the backend.
 	doneCh := make(chan struct{})
-	defer close(doneCh)
 
 	var encrypted bool
 	var err error
@@ -48,17 +48,27 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 	// the following reasons:
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
-	for range newRetryTimerSimple(doneCh) {
-		if encrypted, err = checkBackendEncrypted(objAPI); err != nil {
-			if err == errDiskNotFound ||
-				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) {
-				logger.Info("Waiting for config backend to be encrypted..")
-				continue
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	var stop bool
+	for !stop {
+		select {
+		case <-retryTimerCh:
+			if encrypted, err = checkBackendEncrypted(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) {
+					logger.Info("Waiting for config backend to be encrypted..")
+					continue
+				}
+				close(doneCh)
+				return err
 			}
-			return err
+			stop = true
+		case <-globalOSSignalCh:
+			close(doneCh)
+			return fmt.Errorf("Config encryption process stopped gracefully")
 		}
-		break
 	}
+	close(doneCh)
 
 	if encrypted {
 		// backend is encrypted, but credentials are not specified
@@ -83,24 +93,33 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		return err
 	}
 
+	doneCh = make(chan struct{})
+	defer close(doneCh)
+
+	retryTimerCh = newRetryTimerSimple(doneCh)
+
 	// Migrating Config backend needs a retry mechanism for
 	// the following reasons:
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
-	for range newRetryTimerSimple(doneCh) {
-		// Migrate IAM configuration
-		if err = migrateConfigPrefixToEncrypted(objAPI, activeCredOld, encrypted); err != nil {
-			if err == errDiskNotFound ||
-				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-				logger.Info("Waiting for config backend to be encrypted..")
-				continue
+	for {
+		select {
+		case <-retryTimerCh:
+			// Migrate IAM configuration
+			if err = migrateConfigPrefixToEncrypted(objAPI, activeCredOld, encrypted); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for config backend to be encrypted..")
+					continue
+				}
+				return err
 			}
-			return err
+			return nil
+		case <-globalOSSignalCh:
+			return fmt.Errorf("Config encryption process stopped gracefully")
 		}
-		break
 	}
-	return nil
 }
 
 const (

cmd/config/config.go

@@ -24,7 +24,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"

cmd/config/errors-utils.go

@@ -20,7 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net"
 	"syscall"
 
 	"github.com/minio/minio/pkg/color"
@@ -36,22 +36,39 @@ type Err struct {
 	hint   string
 }
 
+// Clone returns a new Err struct with the same information
+func (u Err) Clone() Err {
+	return Err{
+		msg:    u.msg,
+		detail: u.detail,
+		action: u.action,
+		hint:   u.hint,
+	}
+}
+
 // Return the error message
 func (u Err) Error() string {
 	if u.detail == "" {
-		return u.msg
+		if u.msg != "" {
+			return u.msg
+		}
+		return "<nil>"
 	}
 	return u.detail
 }
 
 // Msg - Replace the current error's message
 func (u Err) Msg(m string, args ...interface{}) Err {
-	return Err{
-		msg:    fmt.Sprintf(m, args...),
-		detail: u.detail,
-		action: u.action,
-		hint:   u.hint,
-	}
+	e := u.Clone()
+	e.msg = fmt.Sprintf(m, args...)
+	return e
+}
+
+// Hint - Replace the current error's message
+func (u Err) Hint(m string, args ...interface{}) Err {
+	e := u.Clone()
+	e.hint = fmt.Sprintf(m, args...)
+	return e
 }
 
 // ErrFn function wrapper
@@ -77,6 +94,10 @@ func newErrFn(msg, action, hint string) ErrFn {
 // ErrorToErr inspects the passed error and transforms it
 // to the appropriate UI error.
 func ErrorToErr(err error) Err {
+	if err == nil {
+		return Err{}
+	}
+
 	// If this is already a Err, do nothing
 	if e, ok := err.(Err); ok {
 		return e
@@ -85,9 +106,11 @@ func ErrorToErr(err error) Err {
 	// Show a generic message for known golang errors
 	if errors.Is(err, syscall.EADDRINUSE) {
 		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
-	} else if errors.Is(err, syscall.EACCES) {
-		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
-	} else if os.IsPermission(err) {
+	} else if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) {
+		switch err.(type) {
+		case *net.OpError:
+			return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+		}
 		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
 	} else if errors.Is(err, io.ErrUnexpectedEOF) {
 		return ErrUnexpectedDataContent(err)
@@ -95,7 +118,6 @@ func ErrorToErr(err error) Err {
 		// Failed to identify what type of error this, return a simple UI error
 		return Err{msg: err.Error()}
 	}
-
 }
 
 // FmtError converts a fatal error message to a more clear error

cmd/config/etcd/dns/etcd_dns.go

@@ -117,9 +117,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		if err != nil {
 			return nil, err
 		}
-		if r.Count == 0 {
-			return nil, ErrNoEntriesFound
-		}
 	}
 
 	var srvRecords []SrvRecord
@@ -143,9 +140,6 @@ func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
 		srvRecords = append(srvRecords, srvRecord)
 
 	}
-	if len(srvRecords) == 0 {
-		return nil, ErrNoEntriesFound
-	}
 	sort.Slice(srvRecords, func(i int, j int) bool {
 		return srvRecords[i].Key < srvRecords[j].Key
 	})

cmd/endpoint.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net"
 	"net/url"
@@ -230,23 +229,25 @@ func (endpoints Endpoints) GetString(i int) string {
 	return endpoints[i].String()
 }
 
-func (endpoints Endpoints) doAnyHostsResolveToLocalhost() bool {
-	for _, endpoint := range endpoints {
-		hostIPs, err := getHostIP(endpoint.Hostname())
-		if err != nil {
-			continue
-		}
-		var loopback int
-		for _, hostIP := range hostIPs.ToSlice() {
-			if net.ParseIP(hostIP).IsLoopback() {
-				loopback++
-			}
-		}
-		if loopback == len(hostIPs) {
-			return true
+func hostResolveToLocalhost(endpoint Endpoint) bool {
+	hostIPs, err := getHostIP(endpoint.Hostname())
+	if err != nil {
+		// Log the message to console about the host resolving
+		reqInfo := (&logger.ReqInfo{}).AppendTags(
+			"host",
+			endpoint.Hostname(),
+		)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err, logger.Application)
+		return false
+	}
+	var loopback int
+	for _, hostIP := range hostIPs.ToSlice() {
+		if net.ParseIP(hostIP).IsLoopback() {
+			loopback++
 		}
 	}
-	return false
+	return loopback == len(hostIPs)
 }
 
 func (endpoints Endpoints) atleastOneEndpointLocal() bool {
@@ -294,8 +295,9 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 					endpoints[i].Hostname(),
 				)
 
-				if k8sReplicaSet && endpoints.doAnyHostsResolveToLocalhost() {
-					err := errors.New("host found resolves to 127.*, DNS incorrectly configured retrying")
+				if k8sReplicaSet && hostResolveToLocalhost(endpoints[i]) {
+					err := fmt.Errorf("host %s resolves to 127.*, DNS incorrectly configured retrying",
+						endpoints[i])
 					// time elapsed
 					timeElapsed := time.Since(startTime)
 					// log error only if more than 1s elapsed
@@ -352,6 +354,21 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 						// previous local we continue to wait to look for
 						// atleast one local.
 						resolvedList[i] = false
+						// time elapsed
+						err := fmt.Errorf("no endpoint is local to this host: %s", endpoints[i])
+						timeElapsed := time.Since(startTime)
+						// log error only if more than 1s elapsed
+						if timeElapsed > time.Second {
+							reqInfo.AppendTags("elapsedTime",
+								humanize.RelTime(startTime,
+									startTime.Add(timeElapsed),
+									"elapsed",
+									"",
+								))
+							ctx := logger.SetReqInfo(context.Background(),
+								reqInfo)
+							logger.LogIf(ctx, err, logger.Application)
+						}
 						continue
 					}
 					epsResolved++

cmd/format-fs.go

@@ -343,17 +343,24 @@ func formatFSFixDeploymentID(fsFormatPath string) error {
 	defer close(doneCh)
 
 	var wlk *lock.LockedFile
-	for range newRetryTimerSimple(doneCh) {
-		wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
-		if err == lock.ErrAlreadyLocked {
-			// Lock already present, sleep and attempt again
-			logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
-			continue
+	retryCh := newRetryTimerSimple(doneCh)
+	var stop bool
+	for !stop {
+		select {
+		case <-retryCh:
+			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+			if err == lock.ErrAlreadyLocked {
+				// Lock already present, sleep and attempt again
+				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
+				continue
+			}
+			if err != nil {
+				return err
+			}
+			stop = true
+		case <-globalOSSignalCh:
+			return fmt.Errorf("Initializing FS format stopped gracefully")
 		}
-		break
-	}
-	if err != nil {
-		return err
 	}
 
 	defer wlk.Close()

cmd/format-xl.go

@@ -155,24 +155,9 @@ func newFormatXLV3(numSets int, setLen int) *formatXLV3 {
 	return format
 }
 
-// Returns formatXL.XL.Version information, this code is specifically
-// used to read XL `format.json` and capture any version information
-// that it may have.
-func formatXLGetVersion(formatPath string) (string, error) {
-	format := &formatXLVersionDetect{}
-	b, err := ioutil.ReadFile(formatPath)
-	if err != nil {
-		return "", err
-	}
-	if err = json.Unmarshal(b, format); err != nil {
-		return "", err
-	}
-	return format.XL.Version, nil
-}
-
-// Returns format meta format version from `format.json`. This code
-// is specifically used to detect meta format.
-func formatMetaGetFormatBackendXL(formatPath string) (string, error) {
+// Returns format XL version after reading `format.json`, returns
+// successfully the version only if the backend is XL.
+func formatGetBackendXLVersion(formatPath string) (string, error) {
 	meta := &formatMetaV1{}
 	b, err := ioutil.ReadFile(formatPath)
 	if err != nil {
@@ -184,7 +169,15 @@ func formatMetaGetFormatBackendXL(formatPath string) (string, error) {
 	if meta.Version != formatMetaVersionV1 {
 		return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version)
 	}
-	return meta.Format, nil
+	if meta.Format != formatBackendXL {
+		return "", fmt.Errorf(`found backend %s, expected %s`, meta.Format, formatBackendXL)
+	}
+	// XL backend found, proceed to detect version.
+	format := &formatXLVersionDetect{}
+	if err = json.Unmarshal(b, format); err != nil {
+		return "", err
+	}
+	return format.XL.Version, nil
 }
 
 // Migrates all previous versions to latest version of `format.json`,
@@ -192,30 +185,27 @@ func formatMetaGetFormatBackendXL(formatPath string) (string, error) {
 // first before it V2 migrates to V3.
 func formatXLMigrate(export string) error {
 	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
-	backend, err := formatMetaGetFormatBackendXL(formatPath)
-	if err != nil {
-		return err
-	}
-	if backend != formatBackendXL {
-		return fmt.Errorf(`Disk %s: found backend %s, expected %s`, export, backend, formatBackendXL)
-	}
-	version, err := formatXLGetVersion(formatPath)
+	version, err := formatGetBackendXLVersion(formatPath)
 	if err != nil {
 		return err
 	}
 	switch version {
 	case formatXLVersionV1:
-		if err = formatXLMigrateV1ToV2(export); err != nil {
+		if err = formatXLMigrateV1ToV2(export, version); err != nil {
 			return err
 		}
+		// Migrate successful v1 => v2, proceed to v2 => v3
+		version = formatXLVersionV2
 		fallthrough
 	case formatXLVersionV2:
-		if err = formatXLMigrateV2ToV3(export); err != nil {
+		if err = formatXLMigrateV2ToV3(export, version); err != nil {
 			return err
 		}
+		// Migrate successful v2 => v3, v3 is latest
+		version = formatXLVersionV3
 		fallthrough
 	case formatXLVersionV3:
-		// format-V3 is the latest verion.
+		// v3 is the latest version, return.
 		return nil
 	}
 	return fmt.Errorf(`%s: unknown format version %s`, export, version)
@@ -223,16 +213,13 @@ func formatXLMigrate(export string) error {
 
 // Migrates version V1 of format.json to version V2 of format.json,
 // migration fails upon any error.
-func formatXLMigrateV1ToV2(export string) error {
-	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
-	version, err := formatXLGetVersion(formatPath)
-	if err != nil {
-		return err
-	}
+func formatXLMigrateV1ToV2(export, version string) error {
 	if version != formatXLVersionV1 {
 		return fmt.Errorf(`Disk %s: format version expected %s, found %s`, export, formatXLVersionV1, version)
 	}
 
+	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
+
 	formatV1 := &formatXLV1{}
 	b, err := ioutil.ReadFile(formatPath)
 	if err != nil {
@@ -260,15 +247,12 @@ func formatXLMigrateV1ToV2(export string) error {
 }
 
 // Migrates V2 for format.json to V3 (Flat hierarchy for multipart)
-func formatXLMigrateV2ToV3(export string) error {
-	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
-	version, err := formatXLGetVersion(formatPath)
-	if err != nil {
-		return err
-	}
+func formatXLMigrateV2ToV3(export, version string) error {
 	if version != formatXLVersionV2 {
 		return fmt.Errorf(`Disk %s: format version expected %s, found %s`, export, formatXLVersionV2, version)
 	}
+
+	formatPath := pathJoin(export, minioMetaBucket, formatConfigFile)
 	formatV2 := &formatXLV2{}
 	b, err := ioutil.ReadFile(formatPath)
 	if err != nil {
@@ -360,18 +344,18 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
 		return err
 	}
 
-	tmpFormatJSON := mustGetUUID() + ".json"
+	tmpFormat := mustGetUUID()
 
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.DeleteFile(minioMetaBucket, tmpFormatJSON)
+	defer disk.DeleteFile(minioMetaBucket, tmpFormat)
 
-	// Append file `format.json.tmp`.
-	if err = disk.WriteAll(minioMetaBucket, tmpFormatJSON, bytes.NewReader(formatBytes)); err != nil {
+	// write to unique file.
+	if err = disk.WriteAll(minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
 		return err
 	}
 
 	// Rename file `uuid.json` --> `format.json`.
-	return disk.RenameFile(minioMetaBucket, tmpFormatJSON, minioMetaBucket, formatConfigFile)
+	return disk.RenameFile(minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile)
 }
 
 var ignoredHiddenDirectories = []string{
@@ -635,6 +619,39 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
 	return fmt.Errorf("Disk ID %s not found in any disk sets %s", this, format.XL.Sets)
 }
 
+// Initializes meta volume on all input storage disks.
+func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error {
+	// This happens for the first time, but keep this here since this
+	// is the only place where it can be made expensive optimizing all
+	// other calls. Create minio meta volume, if it doesn't exist yet.
+
+	// Initialize errs to collect errors inside go-routine.
+	g := errgroup.WithNErrs(len(storageDisks))
+
+	// Initialize all disks in parallel.
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if formats[index] == nil || storageDisks[index] == nil {
+				// Ignore create meta volume on disks which are not found.
+				return nil
+			}
+			return makeFormatXLMetaVolumes(storageDisks[index])
+		}, index)
+	}
+
+	// Return upon first error.
+	for _, err := range g.Wait() {
+		if err == nil {
+			continue
+		}
+		return toObjectErr(err, minioMetaBucket)
+	}
+
+	// Return success here.
+	return nil
+}
+
 // saveFormatXLAll - populates `format.json` on disks in its order.
 func saveFormatXLAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatXLV3) error {
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -646,6 +663,9 @@ func saveFormatXLAll(ctx context.Context, storageDisks []StorageAPI, formats []*
 			if formats[index] == nil || storageDisks[index] == nil {
 				return errDiskNotFound
 			}
+			if err := makeFormatXLMetaVolumes(storageDisks[index]); err != nil {
+				return err
+			}
 			return saveFormatXL(storageDisks[index], formats[index])
 		}, index)
 	}
@@ -745,11 +765,6 @@ func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, driv
 		}
 	}
 
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initFormatXLMetaVolume(storageDisks, formats); err != nil {
-		return format, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %w", err)
-	}
-
 	// Save formats `format.json` across all disks.
 	if err := saveFormatXLAll(ctx, storageDisks, formats); err != nil {
 		return nil, err
@@ -760,23 +775,9 @@ func initFormatXL(ctx context.Context, storageDisks []StorageAPI, setCount, driv
 
 // Make XL backend meta volumes.
 func makeFormatXLMetaVolumes(disk StorageAPI) error {
-	// Attempt to create `.minio.sys`.
-	if err := disk.MakeVol(minioMetaBucket); err != nil {
-		if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
-			return err
-		}
-	}
-	if err := disk.MakeVol(minioMetaTmpBucket); err != nil {
-		if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
-			return err
-		}
-	}
-	if err := disk.MakeVol(minioMetaBackgroundOpsBucket); err != nil {
-		if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
-			return err
-		}
-	}
-	if err := disk.MakeVol(minioMetaMultipartBucket); err != nil {
+	// Attempt to create MinIO internal buckets.
+	err := disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
+	if err != nil {
 		if !IsErrIgnored(err, initMetaVolIgnoredErrs...) {
 			return err
 		}
@@ -786,39 +787,6 @@ func makeFormatXLMetaVolumes(disk StorageAPI) error {
 
 var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)
 
-// Initializes meta volume on all input storage disks.
-func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) error {
-	// This happens for the first time, but keep this here since this
-	// is the only place where it can be made expensive optimizing all
-	// other calls. Create minio meta volume, if it doesn't exist yet.
-
-	// Initialize errs to collect errors inside go-routine.
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Initialize all disks in parallel.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if formats[index] == nil || storageDisks[index] == nil {
-				// Ignore create meta volume on disks which are not found.
-				return nil
-			}
-			return makeFormatXLMetaVolumes(storageDisks[index])
-		}, index)
-	}
-
-	// Return upon first error.
-	for _, err := range g.Wait() {
-		if err == nil {
-			continue
-		}
-		return toObjectErr(err, minioMetaBucket)
-	}
-
-	// Return success here.
-	return nil
-}
-
 // Get all UUIDs which are present in reference format should
 // be present in the list of formats provided, those are considered
 // as online UUIDs.

cmd/format-xl_test.go

@@ -149,95 +149,6 @@ func TestFormatXLEmpty(t *testing.T) {
 	}
 }
 
-// Tests format xl get version.
-func TestFormatXLGetVersion(t *testing.T) {
-	// Get test root.
-	rootPath, err := getTestRoot()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	v := &formatXLVersionDetect{}
-	v.XL.Version = "1"
-	b, err := json.Marshal(v)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err = ioutil.WriteFile(pathJoin(rootPath, formatConfigFile), b, os.FileMode(0644)); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = formatXLGetVersion("not-found")
-	if err == nil {
-		t.Fatal("Expected to fail but found success")
-	}
-
-	vstr, err := formatXLGetVersion(pathJoin(rootPath, formatConfigFile))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if vstr != "1" {
-		t.Fatalf("Expected version '1', got '%s'", vstr)
-	}
-}
-
-// Tests format get backend format.
-func TestFormatMetaGetFormatBackendXL(t *testing.T) {
-	// Get test root.
-	rootPath, err := getTestRoot()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(rootPath)
-
-	m := &formatMetaV1{
-		Format:  "fs",
-		Version: formatMetaVersionV1,
-	}
-
-	b, err := json.Marshal(m)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err = ioutil.WriteFile(pathJoin(rootPath, formatConfigFile), b, os.FileMode(0644)); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = formatMetaGetFormatBackendXL("not-found")
-	if err == nil {
-		t.Fatal("Expected to fail but found success")
-	}
-
-	format, err := formatMetaGetFormatBackendXL(pathJoin(rootPath, formatConfigFile))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if format != m.Format {
-		t.Fatalf("Expected format value %s, got %s", m.Format, format)
-	}
-
-	m = &formatMetaV1{
-		Format:  "xl",
-		Version: "2",
-	}
-
-	b, err = json.Marshal(m)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err = ioutil.WriteFile(pathJoin(rootPath, formatConfigFile), b, os.FileMode(0644)); err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = formatMetaGetFormatBackendXL(pathJoin(rootPath, formatConfigFile))
-	if err == nil {
-		t.Fatal("Expected to fail with incompatible meta version")
-	}
-}
-
 // Tests xl format migration.
 func TestFormatXLMigrate(t *testing.T) {
 	// Get test root.
@@ -271,10 +182,11 @@ func TestFormatXLMigrate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	migratedVersion, err := formatXLGetVersion(pathJoin(rootPath, minioMetaBucket, formatConfigFile))
+	migratedVersion, err := formatGetBackendXLVersion(pathJoin(rootPath, minioMetaBucket, formatConfigFile))
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	if migratedVersion != formatXLVersionV3 {
 		t.Fatalf("expected version: %s, got: %s", formatXLVersionV3, migratedVersion)
 	}

cmd/fs-v1.go

@@ -20,10 +20,12 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/user"
 	"path"
 	"path/filepath"
 	"sort"
@@ -129,7 +131,15 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		if err == errMinDiskSize {
 			return nil, err
 		}
-		return nil, config.ErrUnableToWriteInBackend(err)
+		// Show a descriptive error with a hint about how to fix it.
+		var username string
+		if u, err := user.Current(); err == nil {
+			username = u.Username
+		} else {
+			username = "<your-username>"
+		}
+		hint := fmt.Sprintf("Use 'sudo chown %s %s && sudo chmod u+rxw %s' to provide sufficient permissions.", username, fsPath, fsPath)
+		return nil, config.ErrUnableToWriteInBackend(err).Hint(hint)
 	}
 
 	// Assign a new UUID for FS minio mode. Each server instance
@@ -1346,3 +1356,9 @@ func (fs *FSObjects) IsEncryptionSupported() bool {
 func (fs *FSObjects) IsCompressionSupported() bool {
 	return true
 }
+
+// IsReady - Check if the backend disk is ready to accept traffic.
+func (fs *FSObjects) IsReady(_ context.Context) bool {
+	_, err := os.Stat(fs.fsPath)
+	return err == nil
+}

cmd/gateway-unsupported.go

@@ -192,3 +192,8 @@ func (a GatewayUnsupported) IsEncryptionSupported() bool {
 func (a GatewayUnsupported) IsCompressionSupported() bool {
 	return false
 }
+
+// IsReady - No Op.
+func (a GatewayUnsupported) IsReady(_ context.Context) bool {
+	return false
+}

cmd/gateway/azure/gateway-azure.go

@@ -1296,3 +1296,8 @@ func (a *azureObjects) DeleteBucketPolicy(ctx context.Context, bucket string) er
 func (a *azureObjects) IsCompressionSupported() bool {
 	return false
 }
+
+// IsReady returns whether the layer is ready to take requests.
+func (a *azureObjects) IsReady(ctx context.Context) bool {
+	return minio.IsBackendOnline(ctx, a.httpClient, a.endpoint)
+}

cmd/gateway/b2/gateway-b2.go

@@ -862,3 +862,8 @@ func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 func (l *b2Objects) IsCompressionSupported() bool {
 	return false
 }
+
+// IsReady returns whether the layer is ready to take requests.
+func (l *b2Objects) IsReady(ctx context.Context) bool {
+	return minio.IsBackendOnline(ctx, l.httpClient, "https://api.backblazeb2.com/b2api/v1")
+}

cmd/gateway/gcs/gateway-gcs.go

@@ -1467,3 +1467,8 @@ func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 func (l *gcsGateway) IsCompressionSupported() bool {
 	return false
 }
+
+// IsReady returns whether the layer is ready to take requests.
+func (l *gcsGateway) IsReady(ctx context.Context) bool {
+	return minio.IsBackendOnline(ctx, l.httpClient, "https://storage.googleapis.com")
+}

cmd/gateway/hdfs/gateway-hdfs.go

@@ -715,3 +715,8 @@ func (n *hdfsObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
 	}
 	return hdfsToObjectErr(ctx, n.clnt.Remove(minio.PathJoin(hdfsSeparator, minioMetaTmpBucket, uploadID)), bucket, object, uploadID)
 }
+
+// IsReady returns whether the layer is ready to take requests.
+func (n *hdfsObjects) IsReady(_ context.Context) bool {
+	return true
+}

cmd/gateway/nas/gateway-nas.go

@@ -117,3 +117,9 @@ func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
 type nasObjects struct {
 	minio.ObjectLayer
 }
+
+// IsReady returns whether the layer is ready to take requests.
+func (n *nasObjects) IsReady(ctx context.Context) bool {
+	sinfo := n.ObjectLayer.StorageInfo(ctx)
+	return sinfo.Backend.Type == minio.BackendFS
+}

cmd/gateway/oss/gateway-oss.go

@@ -1098,3 +1098,8 @@ func (l *ossObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 func (l *ossObjects) IsCompressionSupported() bool {
 	return false
 }
+
+// IsReady returns whether the layer is ready to take requests.
+func (l *ossObjects) IsReady(ctx context.Context) bool {
+	return minio.IsBackendOnline(ctx, l.Client.HTTPClient, l.Client.Config.Endpoint)
+}

cmd/gateway/s3/gateway-s3.go

@@ -670,3 +670,8 @@ func (l *s3Objects) IsCompressionSupported() bool {
 func (l *s3Objects) IsEncryptionSupported() bool {
 	return minio.GlobalKMS != nil || len(minio.GlobalGatewaySSE) > 0
 }
+
+// IsReady returns whether the layer is ready to take requests.
+func (l *s3Objects) IsReady(ctx context.Context) bool {
+	return minio.IsBackendOnline(ctx, l.HTTPClient, l.Client.EndpointURL().String())
+}

cmd/generic-handlers.go

@@ -619,7 +619,8 @@ type bucketForwardingHandler struct {
 func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if globalDNSConfig == nil || len(globalDomainNames) == 0 ||
 		guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
-		guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {
+		guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) ||
+		!globalBucketFederation {
 		f.handler.ServeHTTP(w, r)
 		return
 	}

cmd/globals.go

@@ -36,6 +36,7 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/pubsub"
 )
 
@@ -137,9 +138,10 @@ var (
 	// globalConfigSys server config system.
 	globalConfigSys *ConfigSys
 
-	globalNotificationSys *NotificationSys
-	globalPolicySys       *PolicySys
-	globalIAMSys          *IAMSys
+	globalNotificationSys  *NotificationSys
+	globalConfigTargetList *event.TargetList
+	globalPolicySys        *PolicySys
+	globalIAMSys           *IAMSys
 
 	globalLifecycleSys *LifecycleSys
 
@@ -210,6 +212,10 @@ var (
 	// Allocated etcd endpoint for config and bucket DNS.
 	globalEtcdClient *etcd.Client
 
+	// Is set to true when Bucket federation is requested
+	// and is 'true' when etcdConfig.PathPrefix is empty
+	globalBucketFederation bool
+
 	// Allocated DNS config wrapper over etcd client.
 	globalDNSConfig *dns.CoreDNS
 

cmd/healthcheck-handler.go

@@ -17,31 +17,26 @@
 package cmd
 
 import (
-	"fmt"
 	"net/http"
 	"os"
-	"runtime"
 
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
-const (
-	minioHealthGoroutineThreshold = 10000
-)
-
-// ReadinessCheckHandler -- checks if there are more than threshold
-// number of goroutines running, returns service unavailable.
-//
-// Readiness probes are used to detect situations where application
-// is under heavy load and temporarily unable to serve. In a orchestrated
-// setup like Kubernetes, containers reporting that they are not ready do
-// not receive traffic through Kubernetes Services.
+// ReadinessCheckHandler -- Checks if the quorum number of disks are available.
+// For FS - Checks if the backend disk is available
+// For Zones - Checks if all the zones have enough read quorum
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	if err := goroutineCountCheck(minioHealthGoroutineThreshold); err != nil {
+	ctx := newContext(r, w, "ReadinessCheckHandler")
+
+	objLayer := newObjectLayerFn()
+	// Service not initialized yet
+	if objLayer == nil || !objLayer.IsReady(ctx) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
@@ -102,13 +97,3 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
-
-// checks threshold against total number of go-routines in the system and
-// throws error if more than threshold go-routines are running.
-func goroutineCountCheck(threshold int) error {
-	count := runtime.NumGoroutine()
-	if count > threshold {
-		return fmt.Errorf("too many goroutines (%d > %d)", count, threshold)
-	}
-	return nil
-}

cmd/healthcheck-handler_test.go

@@ -1,44 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"testing"
-	"time"
-)
-
-func TestGoroutineCountCheck(t *testing.T) {
-	tests := []struct {
-		threshold int
-		wantErr   bool
-	}{
-		{5000, false},
-		{5, true},
-		{6, true},
-	}
-	for _, tt := range tests {
-		// Make goroutines -- to make sure number of go-routines is higher than threshold
-		if tt.threshold == 5 || tt.threshold == 6 {
-			for i := 0; i < 6; i++ {
-				go time.Sleep(5 * time.Nanosecond)
-			}
-		}
-		if err := goroutineCountCheck(tt.threshold); (err != nil) != tt.wantErr {
-			t.Errorf("goroutineCountCheck() error = %v, wantErr %v", err, tt.wantErr)
-		}
-	}
-}

cmd/iam.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"strings"
 	"sync"
 
@@ -365,46 +366,41 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	// Migrating IAM needs a retry mechanism for
+	// Migrating IAM amd Loading IAM needs a retry mechanism for
 	// the following reasons:
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
-	for range newRetryTimerSimple(doneCh) {
-		// Migrate IAM configuration
-		if err := sys.doIAMConfigMigration(objAPI); err != nil {
-			if err == errDiskNotFound ||
-				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-				logger.Info("Waiting for IAM subsystem to be initialized..")
-				continue
+	retryCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case <-retryCh:
+			// Migrate IAM configuration
+			if err := sys.doIAMConfigMigration(objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for IAM subsystem to be initialized..")
+					continue
+				}
+				return err
 			}
-			return err
-		}
-		break
-	}
 
-	sys.store.watch(sys)
+			sys.store.watch(sys)
 
-	// Initializing IAM needs a retry mechanism for
-	// the following reasons:
-	//  - Read quorum is lost just after the initialization
-	//    of the object layer.
-	for range newRetryTimerSimple(doneCh) {
-		// Load IAMSys once during boot. Need to pass in
-		// objAPI as server has not yet initialized.
-		if err := sys.store.loadAll(sys, objAPI); err != nil {
-			if err == errDiskNotFound ||
-				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-				logger.Info("Waiting for IAM subsystem to be initialized..")
-				continue
+			if err := sys.store.loadAll(sys, objAPI); err != nil {
+				if err == errDiskNotFound ||
+					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for IAM subsystem to be initialized..")
+					continue
+				}
+				return err
 			}
-			return err
+			return nil
+		case <-globalOSSignalCh:
+			return fmt.Errorf("Initializing IAM sub-system gracefully stopped")
 		}
-		break
 	}
-
-	return nil
 }
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
@@ -433,6 +429,38 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 	}
 
 	delete(sys.iamPolicyDocsMap, policyName)
+
+	// Delete user-policy mappings that will no longer apply
+	var usersToDel []string
+	var isUserSTS []bool
+	for u, mp := range sys.iamUserPolicyMap {
+		if mp.Policy == policyName {
+			usersToDel = append(usersToDel, u)
+			cr, ok := sys.iamUsersMap[u]
+			if !ok {
+				// This case cannot happen
+				return errNoSuchUser
+			}
+			// User is from STS if the creds are temporary
+			isSTS := cr.IsTemp()
+			isUserSTS = append(isUserSTS, isSTS)
+		}
+	}
+	for i, u := range usersToDel {
+		sys.policyDBSet(u, "", isUserSTS[i], false)
+	}
+
+	// Delete group-policy mappings that will no longer apply
+	var groupsToDel []string
+	for g, mp := range sys.iamGroupPolicyMap {
+		if mp.Policy == policyName {
+			groupsToDel = append(groupsToDel, g)
+		}
+	}
+	for _, g := range groupsToDel {
+		sys.policyDBSet(g, "", false, true)
+	}
+
 	return err
 }
 
@@ -509,6 +537,19 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		return errServerNotInitialized
 	}
 
+	// First we remove the user from their groups.
+	userInfo, getErr := sys.GetUserInfo(accessKey)
+	if getErr != nil {
+		return getErr
+	}
+	for _, group := range userInfo.MemberOf {
+		removeErr := sys.RemoveUsersFromGroup(group, []string{accessKey})
+		if removeErr != nil {
+			return removeErr
+		}
+	}
+
+	// Next we can remove the user from memory and IAM store
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -1069,17 +1110,17 @@ func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
 	return sys.policyDBSet(name, policy, false, isGroup)
 }
 
-// policyDBSet - sets a policy for user in the policy db. Assumes that
-// caller has sys.Lock().
+// policyDBSet - sets a policy for user in the policy db. Assumes that caller
+// has sys.Lock(). If policy == "", then policy mapping is removed.
 func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
 	if sys.store == nil {
 		return errServerNotInitialized
 	}
 
-	if name == "" || policy == "" {
+	if name == "" {
 		return errInvalidArgument
 	}
-	if _, ok := sys.iamPolicyDocsMap[policy]; !ok {
+	if _, ok := sys.iamPolicyDocsMap[policy]; !ok && policy != "" {
 		return errNoSuchPolicy
 	}
 
@@ -1095,6 +1136,20 @@ func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
 		}
 	}
 
+	// Handle policy mapping removal
+	if policy == "" {
+		if err := sys.store.deleteMappedPolicy(name, isSTS, isGroup); err != nil {
+			return err
+		}
+		if !isGroup {
+			delete(sys.iamUserPolicyMap, name)
+		} else {
+			delete(sys.iamGroupPolicyMap, name)
+		}
+		return nil
+	}
+
+	// Handle policy mapping set/update
 	mp := newMappedPolicy(policy)
 	if err := sys.store.saveMappedPolicy(name, isSTS, isGroup, mp); err != nil {
 		return err

cmd/lock-rest-server.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"math/rand"
 	"net/http"
@@ -29,7 +30,7 @@ import (
 
 const (
 	// Lock maintenance interval.
-	lockMaintenanceInterval = 30 * time.Second
+	lockMaintenanceInterval = 1 * time.Minute
 
 	// Lock validity check interval.
 	lockValidityCheckInterval = 2 * time.Minute
@@ -179,6 +180,8 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 	return nlripMap
 }
 
+var lockMaintenanceTimeout = newDynamicTimeout(60*time.Second, time.Second)
+
 // lockMaintenance loops over locks that have been active for some time and checks back
 // with the original server whether it is still alive or not
 //
@@ -187,7 +190,14 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 // - some network error (and server is up normally)
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
-func lockMaintenance(interval time.Duration) {
+func lockMaintenance(ctx context.Context, interval time.Duration, objAPI ObjectLayer) error {
+	// Lock to avoid concurrent lock maintenance loops
+	maintenanceLock := objAPI.NewNSLock(ctx, "system", "lock-maintenance-ops")
+	if err := maintenanceLock.GetLock(lockMaintenanceTimeout); err != nil {
+		return err
+	}
+	defer maintenanceLock.Unlock()
+
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
 	for lendpoint, nlrips := range getLongLivedLocks(interval) {
@@ -203,7 +213,8 @@ func lockMaintenance(interval time.Duration) {
 						continue
 					}
 
-					// Call back to original server verify whether the lock is still active (based on name & uid)
+					// Call back to original server verify whether the lock is
+					// still active (based on name & uid)
 					expired, err := c.Expired(dsync.LockArgs{
 						UID:      nlrip.lri.UID,
 						Resource: nlrip.name,
@@ -230,15 +241,31 @@ func lockMaintenance(interval time.Duration) {
 			}
 		}
 	}
+
+	return nil
 }
 
 // Start lock maintenance from all lock servers.
 func startLockMaintenance() {
+	var objAPI ObjectLayer
+	var ctx = context.Background()
+
+	// Wait until the object API is ready
+	for {
+		objAPI = newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
 	// Initialize a new ticker with a minute between each ticks.
 	ticker := time.NewTicker(lockMaintenanceInterval)
 	// Stop the timer upon service closure and cleanup the go-routine.
 	defer ticker.Stop()
 
+	r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 	for {
 		// Verifies every minute for locks held more than 2 minutes.
 		select {
@@ -247,10 +274,13 @@ func startLockMaintenance() {
 		case <-ticker.C:
 			// Start with random sleep time, so as to avoid
 			// "synchronous checks" between servers
-			r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 			duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
 			time.Sleep(duration)
-			lockMaintenance(lockValidityCheckInterval)
+			if err := lockMaintenance(ctx, lockValidityCheckInterval, objAPI); err != nil {
+				// Sleep right after an error.
+				duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
+				time.Sleep(duration)
+			}
 		}
 	}
 }

cmd/logger/console.go

@@ -23,9 +23,9 @@ import (
 	"strings"
 	"time"
 
-	c "github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/color"
+	c "github.com/minio/minio/pkg/console"
 )
 
 // Logger interface describes the methods that need to be implemented to satisfy the interface requirements.

cmd/main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@ import (
 	"sort"
 
 	"github.com/minio/cli"
-	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/trie"
 	"github.com/minio/minio/pkg/words"
 )
@@ -74,8 +74,8 @@ FLAGS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
 VERSION:
-  ` + Version +
-	`{{ "\n"}}`
+{{.Version}}
+`
 
 func newApp(name string) *cli.App {
 	// Collection of minio commands currently supported are.
@@ -126,11 +126,10 @@ func newApp(name string) *cli.App {
 	app := cli.NewApp()
 	app.Name = name
 	app.Author = "MinIO, Inc."
-	app.Version = Version
+	app.Version = ReleaseTag
 	app.Usage = "High Performance Object Storage"
 	app.Description = `Build high performance data infrastructure for machine learning, analytics and application data workloads with MinIO`
 	app.Flags = GlobalFlags
-	app.HideVersion = true     // Hide `--version` flag, we already have `minio version`.
 	app.HideHelpCommand = true // Hide `help, h` command, we already have `minio --help`.
 	app.Commands = commands
 	app.CustomAppHelpTemplate = minioHelpTemplate

cmd/naughty-disk_test.go

@@ -91,6 +91,13 @@ func (d *naughtyDisk) DiskInfo() (info DiskInfo, err error) {
 	return d.disk.DiskInfo()
 }
 
+func (d *naughtyDisk) MakeVolBulk(volumes ...string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.MakeVolBulk(volumes...)
+}
+
 func (d *naughtyDisk) MakeVol(volume string) (err error) {
 	if err := d.calcError(); err != nil {
 		return err

cmd/notification.go

@@ -30,8 +30,6 @@ import (
 	"time"
 
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/notify"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
@@ -589,20 +587,6 @@ func (sys *NotificationSys) PutBucketNotification(ctx context.Context, bucketNam
 	}()
 }
 
-// AddNotificationTargetsFromConfig - adds notification targets from server config.
-func (sys *NotificationSys) AddNotificationTargetsFromConfig(cfg config.Config) error {
-	targetList, err := notify.GetNotificationTargets(cfg, GlobalServiceDoneCh, NewCustomHTTPTransport())
-	if err != nil {
-		return err
-	}
-	for _, target := range targetList.Targets() {
-		if err = sys.targetList.Add(target); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // AddRemoteTarget - adds event rules map, HTTP/PeerRPC client target to bucket name.
 func (sys *NotificationSys) AddRemoteTarget(bucketName string, target event.Target, rulesMap event.RulesMap) error {
 	if err := sys.targetList.Add(target); err != nil {
@@ -718,6 +702,14 @@ func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error
 		return nil
 	}
 
+	if globalConfigTargetList != nil {
+		for _, target := range globalConfigTargetList.Targets() {
+			if err := sys.targetList.Add(target); err != nil {
+				return err
+			}
+		}
+	}
+
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
@@ -726,8 +718,7 @@ func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
 	retryTimerCh := newRetryTimerSimple(doneCh)
-	stop := false
-	for !stop {
+	for {
 		select {
 		case <-retryTimerCh:
 			if err := sys.load(buckets, objAPI); err != nil {
@@ -739,17 +730,8 @@ func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error
 				}
 				return err
 			}
-			stop = true
-		case <-globalOSSignalCh:
-			return fmt.Errorf("Initializing Notification sub-system gracefully stopped")
-		}
-	}
-
-	// Initializing bucket retention config needs a retry mechanism if
-	// read quorum is lost just after the initialization of the object layer.
-	for {
-		select {
-		case <-retryTimerCh:
+			// Initializing bucket retention config needs a retry mechanism if
+			// read quorum is lost just after the initialization of the object layer.
 			if err := sys.initBucketObjectLockConfig(objAPI); err != nil {
 				if err == errDiskNotFound ||
 					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||

cmd/object-api-interface.go

@@ -122,4 +122,7 @@ type ObjectLayer interface {
 
 	// Backend related metrics
 	GetMetrics(ctx context.Context) (*Metrics, error)
+
+	// Check Readiness
+	IsReady(ctx context.Context) bool
 }

cmd/object-handlers.go

@@ -640,8 +640,11 @@ func isRemoteCallRequired(ctx context.Context, bucket string, objAPI ObjectLayer
 	if globalDNSConfig == nil {
 		return false
 	}
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	return err == toObjectErr(errVolumeNotFound, bucket)
+	if globalBucketFederation {
+		_, err := objAPI.GetBucketInfo(ctx, bucket)
+		return err == toObjectErr(errVolumeNotFound, bucket)
+	}
+	return false
 }
 
 // CopyObjectHandler - Copy Object
@@ -2421,6 +2424,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(err), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if globalDNSConfig != nil {
 		_, err := globalDNSConfig.Get(bucket)
 		if err != nil {

cmd/peer-rest-common.go

@@ -82,6 +82,7 @@ const (
 	peerRESTTraceAll      = "all"
 	peerRESTTraceErr      = "err"
 
+	peerRESTListenBucket = "bucket"
 	peerRESTListenPrefix = "prefix"
 	peerRESTListenSuffix = "suffix"
 	peerRESTListenEvents = "events"

cmd/peer-rest-server.go

@@ -978,6 +978,9 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 		if !ok {
 			return false
 		}
+		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
+			return false
+		}
 		objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
 		if uerr != nil {
 			objectName = ev.S3.Object.Key

cmd/posix-diskid-check.go

@@ -74,6 +74,13 @@ func (p *posixDiskIDCheck) DiskInfo() (info DiskInfo, err error) {
 	return p.storage.DiskInfo()
 }
 
+func (p *posixDiskIDCheck) MakeVolBulk(volumes ...string) (err error) {
+	if p.isDiskStale() {
+		return errDiskNotFound
+	}
+	return p.storage.MakeVolBulk(volumes...)
+}
+
 func (p *posixDiskIDCheck) MakeVol(volume string) (err error) {
 	if p.isDiskStale() {
 		return errDiskNotFound

cmd/posix.go

@@ -551,6 +551,15 @@ func (s *posix) SetDiskID(id string) {
 	// storage rest server for remote disks.
 }
 
+func (s *posix) MakeVolBulk(volumes ...string) (err error) {
+	for _, volume := range volumes {
+		if err = s.MakeVol(volume); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Make a volume entry.
 func (s *posix) MakeVol(volume string) (err error) {
 	defer func() {

cmd/prepare-storage.go

@@ -223,12 +223,13 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		// Assign globalDeploymentID on first run for the
 		// minio server managing the first disk
 		globalDeploymentID = format.ID
-	} else {
-		// The first will always recreate some directories inside .minio.sys
-		// such as, tmp, multipart and background-ops
-		if firstDisk {
-			initFormatXLMetaVolume(storageDisks, formatConfigs)
-		}
+		return format, nil
+	}
+
+	// The first will always recreate some directories inside .minio.sys
+	// such as, tmp, multipart and background-ops
+	if firstDisk {
+		initFormatXLMetaVolume(storageDisks, formatConfigs)
 	}
 
 	// Return error when quorum unformatted disks - indicating we are
@@ -288,6 +289,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 	if err = formatXLFixLocalDeploymentID(endpoints, storageDisks, format); err != nil {
 		return nil, err
 	}
+
 	return format, nil
 }
 

cmd/server-main.go

@@ -227,10 +227,6 @@ func initAllSubsystems(buckets []BucketInfo, newObject ObjectLayer) (err error)
 		return fmt.Errorf("Unable to initialize config system: %w", err)
 	}
 
-	if err = globalNotificationSys.AddNotificationTargetsFromConfig(globalServerConfig); err != nil {
-		return fmt.Errorf("Unable to initialize notification target(s) from config: %w", err)
-	}
-
 	if globalEtcdClient != nil {
 		// ****  WARNING ****
 		// Migrating to encrypted backend on etcd should happen before initialization of

cmd/storage-interface.go

@@ -36,6 +36,7 @@ type StorageAPI interface {
 
 	// Volume operations.
 	MakeVol(volume string) (err error)
+	MakeVolBulk(volumes ...string) (err error)
 	ListVols() (vols []VolInfo, err error)
 	StatVol(volume string) (vol VolInfo, err error)
 	DeleteVol(volume string) (err error)

cmd/storage-rest-client.go

@@ -28,6 +28,7 @@ import (
 	"net/url"
 	"path"
 	"strconv"
+	"strings"
 	"sync/atomic"
 
 	"github.com/minio/minio/cmd/http"
@@ -189,6 +190,15 @@ func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) {
 	return info, err
 }
 
+// MakeVolBulk - create multiple volumes in a bulk operation.
+func (client *storageRESTClient) MakeVolBulk(volumes ...string) (err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolumes, strings.Join(volumes, ","))
+	respBody, err := client.call(storageRESTMethodMakeVolBulk, values, nil, -1)
+	defer http.DrainBody(respBody)
+	return err
+}
+
 // MakeVol - create a volume on a remote disk.
 func (client *storageRESTClient) MakeVol(volume string) (err error) {
 	values := make(url.Values)

cmd/storage-rest-common.go

@@ -17,7 +17,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v11"
+	storageRESTVersion       = "v12"
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )
@@ -26,6 +26,7 @@ const (
 	storageRESTMethodDiskInfo             = "/diskinfo"
 	storageRESTMethodCrawlAndGetDataUsage = "/crawlandgetdatausage"
 	storageRESTMethodMakeVol              = "/makevol"
+	storageRESTMethodMakeVolBulk          = "/makevolbulk"
 	storageRESTMethodStatVol              = "/statvol"
 	storageRESTMethodDeleteVol            = "/deletevol"
 	storageRESTMethodListVols             = "/listvols"
@@ -47,6 +48,7 @@ const (
 
 const (
 	storageRESTVolume     = "volume"
+	storageRESTVolumes    = "volumes"
 	storageRESTDirPath    = "dir-path"
 	storageRESTFilePath   = "file-path"
 	storageRESTSrcVolume  = "source-volume"

cmd/storage-rest-server.go

@@ -24,8 +24,10 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os/user"
 	"path"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -150,6 +152,19 @@ func (s *storageRESTServer) MakeVolHandler(w http.ResponseWriter, r *http.Reques
 	}
 }
 
+// MakeVolBulkHandler - create multiple volumes as a bulk operation.
+func (s *storageRESTServer) MakeVolBulkHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volumes := strings.Split(vars[storageRESTVolumes], ",")
+	err := s.storage.MakeVolBulk(volumes...)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
 // ListVolsHandler - list volumes.
 func (s *storageRESTServer) ListVolsHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -594,7 +609,16 @@ func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones
 			}
 			storage, err := newPosix(endpoint.Path)
 			if err != nil {
-				logger.Fatal(config.ErrUnableToWriteInBackend(err),
+				// Show a descriptive error with a hint about how to fix it.
+				var username string
+				if u, err := user.Current(); err == nil {
+					username = u.Username
+				} else {
+					username = "<your-username>"
+				}
+				hint := fmt.Sprintf("Run the following command to add the convenient permissions: `sudo chown %s %s && sudo chmod u+rxw %s`",
+					username, endpoint.Path, endpoint.Path)
+				logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint(hint),
 					"Unable to initialize posix backend")
 			}
 
@@ -605,6 +629,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCrawlAndGetDataUsage).HandlerFunc(httpTraceHdrs(server.CrawlAndGetDataUsageHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler)).Queries(restQueries(storageRESTVolume)...)
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVolBulk).HandlerFunc(httpTraceHdrs(server.MakeVolBulkHandler)).Queries(restQueries(storageRESTVolumes)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatVol).HandlerFunc(httpTraceHdrs(server.StatVolHandler)).Queries(restQueries(storageRESTVolume)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVol).HandlerFunc(httpTraceHdrs(server.DeleteVolHandler)).Queries(restQueries(storageRESTVolume)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListVols).HandlerFunc(httpTraceHdrs(server.ListVolsHandler))

cmd/web-handlers.go

@@ -258,10 +258,6 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
 		return toJSONError(ctx, err, args.BucketName)
 	}
 
-	globalNotificationSys.RemoveNotification(args.BucketName)
-	globalPolicySys.Remove(args.BucketName)
-	globalNotificationSys.DeleteBucket(ctx, args.BucketName)
-
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(args.BucketName); err != nil {
 			// Deleting DNS entry failed, attempt to create the bucket again.
@@ -270,6 +266,8 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
 		}
 	}
 
+	globalNotificationSys.DeleteBucket(ctx, args.BucketName)
+
 	return nil
 }
 

cmd/xl-sets.go

@@ -1491,12 +1491,6 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 			}
 		}
 
-		// Initialize meta volume, if volume already exists ignores it, all disks which
-		// are not found are ignored as well.
-		if err = initFormatXLMetaVolume(storageDisks, tmpNewFormats); err != nil {
-			return madmin.HealResultItem{}, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %w", err)
-		}
-
 		// Save formats `format.json` across all disks.
 		if err = saveFormatXLAll(ctx, storageDisks, tmpNewFormats); err != nil {
 			return madmin.HealResultItem{}, err
@@ -1662,3 +1656,30 @@ func (s *xlSets) GetMetrics(ctx context.Context) (*Metrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return &Metrics{}, NotImplemented{}
 }
+
+// IsReady - Returns true if atleast n/2 disks (read quorum) are online
+func (s *xlSets) IsReady(_ context.Context) bool {
+	s.xlDisksMu.RLock()
+	defer s.xlDisksMu.RUnlock()
+
+	var activeDisks int
+	for i := 0; i < s.setCount; i++ {
+		for j := 0; j < s.drivesPerSet; j++ {
+			if s.xlDisks[i][j] == nil {
+				continue
+			}
+			if !s.xlLockers[i][j].IsOnline() {
+				continue
+			}
+			if s.xlDisks[i][j].IsOnline() {
+				activeDisks++
+			}
+			// Return true if read quorum is available.
+			if activeDisks >= len(s.endpoints)/2 {
+				return true
+			}
+		}
+	}
+	// Disks are not ready
+	return false
+}

cmd/xl-v1.go

@@ -245,3 +245,9 @@ func (xl xlObjects) crawlAndGetDataUsage(ctx context.Context, endCh <-chan struc
 
 	return dataUsageInfo
 }
+
+// IsReady - No Op.
+func (xl xlObjects) IsReady(ctx context.Context) bool {
+	logger.CriticalIf(ctx, NotImplemented{})
+	return true
+}

cmd/xl-zones.go

@@ -1359,3 +1359,16 @@ func (z *xlZones) GetMetrics(ctx context.Context) (*Metrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return &Metrics{}, NotImplemented{}
 }
+
+// IsReady - Returns True if all the zones have enough quorum to accept requests.
+func (z *xlZones) IsReady(ctx context.Context) bool {
+	if z.SingleZone() {
+		return z.zones[0].IsReady(ctx)
+	}
+	for _, xlsets := range z.zones {
+		if !xlsets.IsReady(ctx) {
+			return false
+		}
+	}
+	return true
+}

docs/bucket/notifications/README.md

@@ -38,6 +38,8 @@ notify_elasticsearch  publish bucket notifications to Elasticsearch endpoints
 notify_redis          publish bucket notifications to Redis datastores
 ```
 
+> NOTE: '*' at the end of arg means its mandatory, '*' at the end of the values, means its the default value for the arg.
+
 <a name="AMQP"></a>
 
 ## Publish MinIO events via AMQP
@@ -48,22 +50,47 @@ Install RabbitMQ from [here](https://www.rabbitmq.com/).
 
 The AMQP configuration is located under the sub-system `notify_amqp` top-level key. Create a configuration key-value pair here for your AMQP instance. The key is a name for your AMQP endpoint, and the value is a collection of key-value parameters described in the table below.
 
-| Parameter       | Description                                                                      |
-| :-------------  | :------------------------------------------------------------------------------- |
-| `state`         | (Required) Is this server endpoint configuration active/enabled?                 |
-| `url`           | (Required) AMQP server endpoint, e.g. `amqp://myuser:mypassword@localhost:5672`  |
-| `exchange`      | Name of the exchange.                                                            |
-| `routing_key`   | Routing key for publishing.                                                      |
-| `exchange_type` | Kind of exchange.                                                                |
-| `delivery_mode` | Delivery mode for publishing. 0 or 1 - transient; 2 - persistent.                |
-| `mandatory`     | Publishing related bool.                                                         |
-| `immediate`     | Publishing related bool.                                                         |
-| `durable`       | Exchange declaration related bool.                                               |
-| `internal`      | Exchange declaration related bool.                                               |
-| `no_wait`       | Exchange declaration related bool.                                               |
-| `auto_deleted`  | Exchange declaration related bool.                                               |
-| `queue_dir`     | Persistent store for events when AMQP broker is offline                          |
-| `queue_limit`   | Set the maximum event limit for the persistent store. The default limit is 10000 |
+```
+KEY:
+notify_amqp[:name]  publish bucket notifications to AMQP endpoints
+
+ARGS:
+url*           (url)       AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672`
+exchange       (string)    name of the AMQP exchange
+exchange_type  (string)    AMQP exchange type
+routing_key    (string)    routing key for publishing
+mandatory      (on|off)    quietly ignore undelivered messages when set to 'off', default is 'on'
+durable        (on|off)    persist queue across broker restarts when set to 'on', default is 'off'
+no_wait        (on|off)    non-blocking message delivery when set to 'on', default is 'off'
+internal       (on|off)    set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges
+auto_deleted   (on|off)    auto delete queue when set to 'on', when there are no consumers
+delivery_mode  (number)    set to '1' for non-persistent or '2' for persistent queue
+queue_dir      (path)      staging dir for undelivered messages e.g. '/home/events'
+queue_limit    (number)    maximum limit for undelivered messages, defaults to '10000'
+comment        (sentence)  optionally add a comment to this setting
+```
+
+Or environment variables
+
+```
+KEY:
+notify_amqp[:name]  publish bucket notifications to AMQP endpoints
+
+ARGS:
+MINIO_NOTIFY_AMQP_URL*           (url)       AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672`
+MINIO_NOTIFY_AMQP_EXCHANGE       (string)    name of the AMQP exchange
+MINIO_NOTIFY_AMQP_EXCHANGE_TYPE  (string)    AMQP exchange type
+MINIO_NOTIFY_AMQP_ROUTING_KEY    (string)    routing key for publishing
+MINIO_NOTIFY_AMQP_MANDATORY      (on|off)    quietly ignore undelivered messages when set to 'off', default is 'on'
+MINIO_NOTIFY_AMQP_DURABLE        (on|off)    persist queue across broker restarts when set to 'on', default is 'off'
+MINIO_NOTIFY_AMQP_NO_WAIT        (on|off)    non-blocking message delivery when set to 'on', default is 'off'
+MINIO_NOTIFY_AMQP_INTERNAL       (on|off)    set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges
+MINIO_NOTIFY_AMQP_AUTO_DELETED   (on|off)    auto delete queue when set to 'on', when there are no consumers
+MINIO_NOTIFY_AMQP_DELIVERY_MODE  (number)    set to '1' for non-persistent or '2' for persistent queue
+MINIO_NOTIFY_AMQP_QUEUE_DIR      (path)      staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_AMQP_QUEUE_LIMIT    (number)    maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_AMQP_COMMENT        (sentence)  optionally add a comment to this setting
+```
 
 MinIO supports persistent event store. The persistent store will backup events when the AMQP broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 10000.
 
@@ -159,17 +186,41 @@ Install an MQTT Broker from [here](https://mosquitto.org/).
 
 The MQTT configuration is located as `notify_mqtt` key. Create a configuration key-value pair here for your MQTT instance. The key is a name for your MQTT endpoint, and the value is a collection of key-value parameters described in the table below.
 
-| Parameter     | Description                                                                      |
-| :-----------  | :------------------------------------------------------------------------------- |
-| `state`       | (Required) Is this server endpoint configuration active/enabled?                 |
-| `broker`      | (Required) MQTT server endpoint, e.g. `tcp://localhost:1883`                     |
-| `topic`       | (Required) Name of the MQTT topic to publish on, e.g. `minio`                    |
-| `qos`         | Set the Quality of Service Level                                                 |
-| `username`    | Username to connect to the MQTT server (if required)                             |
-| `password`    | Password to connect to the MQTT server (if required)                             |
-| `queue_dir`   | Persistent store for events when MQTT broker is offline                          |
-| `queue_limit` | Set the maximum event limit for the persistent store. The default limit is 10000 |
+```
+KEY:
+notify_mqtt[:name]  publish bucket notifications to MQTT endpoints
 
+ARGS:
+broker*              (uri)       MQTT server endpoint e.g. `tcp://localhost:1883`
+topic*               (string)    name of the MQTT topic to publish
+username             (string)    MQTT username
+password             (string)    MQTT password
+qos                  (number)    set the quality of service priority, defaults to '0'
+keep_alive_interval  (duration)  keep-alive interval for MQTT connections in s,m,h,d
+reconnect_interval   (duration)  reconnect interval for MQTT connections in s,m,h,d
+queue_dir            (path)      staging dir for undelivered messages e.g. '/home/events'
+queue_limit          (number)    maximum limit for undelivered messages, defaults to '10000'
+comment              (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+
+```
+KEY:
+notify_mqtt[:name]  publish bucket notifications to MQTT endpoints
+
+ARGS:
+MINIO_NOTIFY_MQTT_BROKER*              (uri)       MQTT server endpoint e.g. `tcp://localhost:1883`
+MINIO_NOTIFY_MQTT_TOPIC*               (string)    name of the MQTT topic to publish
+MINIO_NOTIFY_MQTT_USERNAME             (string)    MQTT username
+MINIO_NOTIFY_MQTT_PASSWORD             (string)    MQTT password
+MINIO_NOTIFY_MQTT_QOS                  (number)    set the quality of service priority, defaults to '0'
+MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL  (duration)  keep-alive interval for MQTT connections in s,m,h,d
+MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL   (duration)  reconnect interval for MQTT connections in s,m,h,d
+MINIO_NOTIFY_MQTT_QUEUE_DIR            (path)      staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_MQTT_QUEUE_LIMIT          (number)    maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_MQTT_COMMENT              (sentence)  optionally add a comment to this setting
+```
 
 MinIO supports persistent event store. The persistent store will backup events when the MQTT broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 10000.
 
@@ -271,14 +322,33 @@ MinIO requires a 5.x series version of Elasticsearch. This is the latest major r
 
 The Elasticsearch configuration is located in the `notify_elasticsearch` key. Create a configuration key-value pair here for your Elasticsearch instance. The key is a name for your Elasticsearch endpoint, and the value is a collection of key-value parameters described in the table below.
 
-| Parameter     | Description                                                                        |
-| :-----------  | :-------------------------------------------------------------------------------   |
-| `state`       | (Required) Is this server endpoint configuration active/enabled?                   |
-| `format`      | (Required) Either `namespace` or `access`.                                         |
-| `url`         | (Required) The Elasticsearch server's address, with optional authentication info.  |
-| `index`       | (Required) The name of an Elasticsearch index in which MinIO will store documents. |
-| `queue_dir`   | Persistent store for events when Elasticsearch broker is offline                   |
-| `queue_limit` | Set the maximum event limit for the persistent store. The default limit is 10000   |
+```
+KEY:
+notify_elasticsearch[:name]  publish bucket notifications to Elasticsearch endpoints
+
+ARGS:
+url*         (url)                Elasticsearch server's address, with optional authentication info
+index*       (string)             Elasticsearch index to store/update events, index is auto-created
+format*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+queue_dir    (path)               staging dir for undelivered messages e.g. '/home/events'
+queue_limit  (number)             maximum limit for undelivered messages, defaults to '10000'
+comment      (sentence)           optionally add a comment to this setting
+```
+
+or environment variables
+
+```
+KEY:
+notify_elasticsearch[:name]  publish bucket notifications to Elasticsearch endpoints
+
+ARGS:
+MINIO_NOTIFY_ELASTICSEARCH_URL*         (url)                Elasticsearch server's address, with optional authentication info
+MINIO_NOTIFY_ELASTICSEARCH_INDEX*       (string)             Elasticsearch index to store/update events, index is auto-created
+MINIO_NOTIFY_ELASTICSEARCH_FORMAT*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR    (path)               staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT  (number)             maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_ELASTICSEARCH_COMMENT      (sentence)           optionally add a comment to this setting
+```
 
 For example: `http://localhost:9200` or with authentication info `http://elastic:MagicWord@127.0.0.1:9200`.
 
@@ -416,13 +486,35 @@ The steps below show how to use this notification target in `namespace` and `acc
 
 The MinIO server configuration file is stored on the backend in json format.The Redis configuration is located in the `redis` key under the `notify` top-level key. Create a configuration key-value pair here for your Redis instance. The key is a name for your Redis endpoint, and the value is a collection of key-value parameters described in the table below.
 
-| Parameter  | Description                                                                                                                                             |
-| :--------- | :-------------                                                                                                                                          |
-| `state`    | (Required) Is this server endpoint configuration active/enabled?                                                                                        |
-| `format`   | (Required) Either `namespace` or `access`                                                                                                               |
-| `address`  | (Required) The Redis server's address. For example: `localhost:6379`                                                                                    |
-| `password` | (Optional) The Redis server's password                                                                                                                  |
-| `key`      | (Required) The name of the redis key under which events are stored. A hash is used in case of `namespace` format and a list in case of `access` format. |
+```
+KEY:
+notify_redis[:name]  publish bucket notifications to Redis datastores
+
+ARGS:
+address*     (address)            Redis server's address. For example: `localhost:6379`
+key*         (string)             Redis key to store/update events, key is auto-created
+format*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+password     (string)             Redis server password
+queue_dir    (path)               staging dir for undelivered messages e.g. '/home/events'
+queue_limit  (number)             maximum limit for undelivered messages, defaults to '10000'
+comment      (sentence)           optionally add a comment to this setting
+```
+
+or environment variables
+
+```
+KEY:
+notify_redis[:name]  publish bucket notifications to Redis datastores
+
+ARGS:
+MINIO_NOTIFY_REDIS_ADDRESS*     (address)            Redis server's address. For example: `localhost:6379`
+MINIO_NOTIFY_REDIS_KEY*         (string)             Redis key to store/update events, key is auto-created
+MINIO_NOTIFY_REDIS_FORMAT*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+MINIO_NOTIFY_REDIS_PASSWORD     (string)             Redis server password
+MINIO_NOTIFY_REDIS_QUEUE_DIR    (path)               staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_REDIS_QUEUE_LIMIT  (number)             maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_REDIS_COMMENT      (sentence)           optionally add a comment to this setting
+```
 
 MinIO supports persistent event store. The persistent store will backup events when the Redis broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 10000.
 
@@ -495,6 +587,57 @@ Install NATS from [here](http://nats.io/).
 
 MinIO supports persistent event store. The persistent store will backup events when the NATS broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 10000.
 
+```
+KEY:
+notify_nats[:name]  publish bucket notifications to NATS endpoints
+
+ARGS:
+address*                          (address)   NATS server address e.g. '0.0.0.0:4222'
+subject*                          (string)    NATS subscription subject
+username                          (string)    NATS username
+password                          (string)    NATS password
+token                             (string)    NATS token
+tls                               (on|off)    set to 'on' to enable TLS
+tls_skip_verify                   (on|off)    trust server TLS without verification, defaults to "on" (verify)
+ping_interval                     (duration)  client ping commands interval in s,m,h,d. Disabled by default
+streaming                         (on|off)    set to 'on', to use streaming NATS server
+streaming_async                   (on|off)    set to 'on', to enable asynchronous publish
+streaming_max_pub_acks_in_flight  (number)    number of messages to publish without waiting for ACKs
+streaming_cluster_id              (string)    unique ID for NATS streaming cluster
+cert_authority                    (string)    path to certificate chain of the target NATS server
+client_cert                       (string)    client cert for NATS mTLS auth
+client_key                        (string)    client cert key for NATS mTLS auth
+queue_dir                         (path)      staging dir for undelivered messages e.g. '/home/events'
+queue_limit                       (number)    maximum limit for undelivered messages, defaults to '10000'
+comment                           (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+notify_nats[:name]  publish bucket notifications to NATS endpoints
+
+ARGS:
+MINIO_NOTIFY_NATS_ADDRESS*                          (address)   NATS server address e.g. '0.0.0.0:4222'
+MINIO_NOTIFY_NATS_SUBJECT*                          (string)    NATS subscription subject
+MINIO_NOTIFY_NATS_USERNAME                          (string)    NATS username
+MINIO_NOTIFY_NATS_PASSWORD                          (string)    NATS password
+MINIO_NOTIFY_NATS_TOKEN                             (string)    NATS token
+MINIO_NOTIFY_NATS_TLS                               (on|off)    set to 'on' to enable TLS
+MINIO_NOTIFY_NATS_TLS_SKIP_VERIFY                   (on|off)    trust server TLS without verification, defaults to "on" (verify)
+MINIO_NOTIFY_NATS_PING_INTERVAL                     (duration)  client ping commands interval in s,m,h,d. Disabled by default
+MINIO_NOTIFY_NATS_STREAMING                         (on|off)    set to 'on', to use streaming NATS server
+MINIO_NOTIFY_NATS_STREAMING_ASYNC                   (on|off)    set to 'on', to enable asynchronous publish
+MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT  (number)    number of messages to publish without waiting for ACKs
+MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID              (string)    unique ID for NATS streaming cluster
+MINIO_NOTIFY_NATS_CERT_AUTHORITY                    (string)    path to certificate chain of the target NATS server
+MINIO_NOTIFY_NATS_CLIENT_CERT                       (string)    client cert for NATS mTLS auth
+MINIO_NOTIFY_NATS_CLIENT_KEY                        (string)    client cert key for NATS mTLS auth
+MINIO_NOTIFY_NATS_QUEUE_DIR                         (path)      staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_NATS_QUEUE_LIMIT                       (number)    maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_NATS_COMMENT                           (sentence)  optionally add a comment to this setting
+```
+
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment.
 
 ```sh
@@ -675,18 +818,42 @@ MinIO requires PostgreSQL version 9.5 or above. MinIO uses the [`INSERT ON CONFL
 
 The PostgreSQL configuration is located in the `notify_postgresql` key. Create a configuration key-value pair here for your PostgreSQL instance. The key is a name for your PostgreSQL endpoint, and the value is a collection of key-value parameters described in the table below.
 
-| Parameter           | Description                                                                                                                                                                          |
-| :-----------------  | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `state`             | (Required) Is this server endpoint configuration active/enabled?                                                                                                                     |
-| `format`            | (Required) Either `namespace` or `access`.                                                                                                                                           |
-| `connection_string` | (Optional) [Connection string parameters](https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters) for the PostgreSQL server. Can be used to set `sslmode` for example. |
-| `table`             | (Required) Table name in which events will be stored/updated. If the table does not exist, the MinIO server creates it at start-up.                                                  |
-| `host`              | (Optional) Host name of the PostgreSQL server. Defaults to `localhost`. IPv6 host should be enclosed with `[` and `]`                                                                |
-| `port`              | (Optional) Port on which to connect to PostgreSQL server. Defaults to `5432`.                                                                                                        |
-| `user`              | (Optional) Database user name. Defaults to user running the server process.                                                                                                          |
-| `password`          | (Optional) Database password.                                                                                                                                                        |
-| `database`          | (Optional) Database name.                                                                                                                                                            |
-|                     |                                                                                                                                                                                     |
+```
+KEY:
+notify_postgres[:name]  publish bucket notifications to Postgres databases
+
+ARGS:
+connection_string*  (string)             Postgres server connection-string
+table*              (string)             DB table name to store/update events, table is auto-created
+format*             (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+host                (hostname)           Postgres server hostname (used only if `connection_string` is empty)
+port                (port)               Postgres server port, defaults to `5432` (used only if `connection_string` is empty)
+username            (string)             database username (used only if `connection_string` is empty)
+password            (string)             database password (used only if `connection_string` is empty)
+database            (string)             database name (used only if `connection_string` is empty)
+queue_dir           (path)               staging dir for undelivered messages e.g. '/home/events'
+queue_limit         (number)             maximum limit for undelivered messages, defaults to '10000'
+comment             (sentence)           optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+notify_postgres[:name]  publish bucket notifications to Postgres databases
+
+ARGS:
+MINIO_NOTIFY_POSTGRES_CONNECTION_STRING*  (string)             Postgres server connection-string
+MINIO_NOTIFY_POSTGRES_TABLE*              (string)             DB table name to store/update events, table is auto-created
+MINIO_NOTIFY_POSTGRES_FORMAT*             (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+MINIO_NOTIFY_POSTGRES_HOST                (hostname)           Postgres server hostname (used only if `connection_string` is empty)
+MINIO_NOTIFY_POSTGRES_PORT                (port)               Postgres server port, defaults to `5432` (used only if `connection_string` is empty)
+MINIO_NOTIFY_POSTGRES_USERNAME            (string)             database username (used only if `connection_string` is empty)
+MINIO_NOTIFY_POSTGRES_PASSWORD            (string)             database password (used only if `connection_string` is empty)
+MINIO_NOTIFY_POSTGRES_DATABASE            (string)             database name (used only if `connection_string` is empty)
+MINIO_NOTIFY_POSTGRES_QUEUE_DIR           (path)               staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT         (number)             maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_POSTGRES_COMMENT             (sentence)           optionally add a comment to this setting
+```
 
 MinIO supports persistent event store. The persistent store will backup events when the PostgreSQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 10000.
 
@@ -767,16 +934,42 @@ MinIO requires MySQL version 5.7.8 or above. MinIO uses the [JSON](https://dev.m
 
 The MySQL configuration is located in the `notify_mysql` key. Create a configuration key-value pair here for your MySQL instance. The key is a name for your MySQL endpoint, and the value is a collection of key-value parameters described in the table below.
 
-| Parameter    | Description                                                                                                                         |
-| :----------  | :-------------                                                                                                                      |
-| `format`     | (Required) Either `namespace` or `access`.                                                                                          |
-| `dsn_string` | (Optional) [Data-Source-Name connection string](https://github.com/go-sql-driver/mysql#dsn-data-source-name) for the MySQL server.  |
-| `table`      | (Required) Table name in which events will be stored/updated. If the table does not exist, the MinIO server creates it at start-up. |
-| `host`       | Host name of the MySQL server (used only if `dsnString` is empty).                                                                  |
-| `port`       | Port on which to connect to the MySQL server (used only if `dsn_string` is empty).                                                  |
-| `user`       | Database user-name (used only if `dsnString` is empty).                                                                             |
-| `password`   | Database password (used only if `dsnString` is empty).                                                                              |
-| `database`   | Database name (used only if `dsnString` is empty).                                                                                  |
+```
+KEY:
+notify_mysql[:name]  publish bucket notifications to MySQL databases
+
+ARGS:
+dsn_string*  (string)             MySQL data-source-name connection string
+table*       (string)             DB table name to store/update events, table is auto-created
+format*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+host         (hostname)           MySQL server hostname (used only if `dsn_string` is empty)
+port         (port)               MySQL server port (used only if `dsn_string` is empty)
+username     (string)             database username (used only if `dsn_string` is empty)
+password     (string)             database password (used only if `dsn_string` is empty)
+database     (string)             database name (used only if `dsn_string` is empty)
+queue_dir    (path)               staging dir for undelivered messages e.g. '/home/events'
+queue_limit  (number)             maximum limit for undelivered messages, defaults to '10000'
+comment      (sentence)           optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+notify_mysql[:name]  publish bucket notifications to MySQL databases
+
+ARGS:
+MINIO_NOTIFY_MYSQL_DSN_STRING*  (string)             MySQL data-source-name connection string
+MINIO_NOTIFY_MYSQL_TABLE*       (string)             DB table name to store/update events, table is auto-created
+MINIO_NOTIFY_MYSQL_FORMAT*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
+MINIO_NOTIFY_MYSQL_HOST         (hostname)           MySQL server hostname (used only if `dsn_string` is empty)
+MINIO_NOTIFY_MYSQL_PORT         (port)               MySQL server port (used only if `dsn_string` is empty)
+MINIO_NOTIFY_MYSQL_USERNAME     (string)             database username (used only if `dsn_string` is empty)
+MINIO_NOTIFY_MYSQL_PASSWORD     (string)             database password (used only if `dsn_string` is empty)
+MINIO_NOTIFY_MYSQL_DATABASE     (string)             database name (used only if `dsn_string` is empty)
+MINIO_NOTIFY_MYSQL_QUEUE_DIR    (path)               staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_MYSQL_QUEUE_LIMIT  (number)             maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_MYSQL_COMMENT      (sentence)           optionally add a comment to this setting
+```
 
 `dns_string` is optional, if not specified, the connection information specified by the `host`, `port`, `user`, `password` and `database` parameters are used.
 
@@ -851,6 +1044,47 @@ MinIO requires Kafka version 0.10 or 0.9. Internally MinIO uses the [Shopify/sar
 
 MinIO supports persistent event store. The persistent store will backup events when the kafka broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 10000.
 
+```
+KEY:
+notify_kafka[:name]  publish bucket notifications to Kafka endpoints
+
+ARGS:
+brokers*         (csv)       comma separated list of Kafka broker addresses
+topic            (string)    Kafka topic used for bucket notifications
+sasl_username    (string)    username for SASL/PLAIN or SASL/SCRAM authentication
+sasl_password    (string)    password for SASL/PLAIN or SASL/SCRAM authentication
+tls_client_auth  (string)    clientAuth determines the Kafka server's policy for TLS client auth
+sasl             (on|off)    set to 'on' to enable SASL authentication
+tls              (on|off)    set to 'on' to enable TLS
+tls_skip_verify  (on|off)    trust server TLS without verification, defaults to "on" (verify)
+client_tls_cert  (path)      path to client certificate for mTLS auth
+client_tls_key   (path)      path to client key for mTLS auth
+queue_dir        (path)      staging dir for undelivered messages e.g. '/home/events'
+queue_limit      (number)    maximum limit for undelivered messages, defaults to '10000'
+comment          (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+notify_kafka[:name]  publish bucket notifications to Kafka endpoints
+
+ARGS:
+MINIO_NOTIFY_KAFKA_BROKERS*         (csv)       comma separated list of Kafka broker addresses
+MINIO_NOTIFY_KAFKA_TOPIC            (string)    Kafka topic used for bucket notifications
+MINIO_NOTIFY_KAFKA_SASL_USERNAME    (string)    username for SASL/PLAIN or SASL/SCRAM authentication
+MINIO_NOTIFY_KAFKA_SASL_PASSWORD    (string)    password for SASL/PLAIN or SASL/SCRAM authentication
+MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH  (string)    clientAuth determines the Kafka server's policy for TLS client auth
+MINIO_NOTIFY_KAFKA_SASL             (on|off)    set to 'on' to enable SASL authentication
+MINIO_NOTIFY_KAFKA_TLS              (on|off)    set to 'on' to enable TLS
+MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY  (on|off)    trust server TLS without verification, defaults to "on" (verify)
+MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT  (path)      path to client certificate for mTLS auth
+MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY   (path)      path to client key for mTLS auth
+MINIO_NOTIFY_KAFKA_QUEUE_DIR        (path)      staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_KAFKA_QUEUE_LIMIT      (number)    maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_KAFKA_COMMENT          (sentence)  optionally add a comment to this setting
+```
+
 To update the configuration, use `mc admin config get` command to get the current configuration.
 
 ```sh
@@ -960,9 +1194,33 @@ MinIO supports persistent event store. The persistent store will backup events w
 
 To update the configuration, use `mc admin config get` command to get the current configuration.
 
+```
+KEY:
+notify_webhook[:name]  publish bucket notifications to webhook endpoints
+
+ARGS:
+endpoint*    (url)       webhook server endpoint e.g. http://localhost:8080/minio/events
+auth_token   (string)    opaque string or JWT authorization token
+queue_dir    (path)      staging dir for undelivered messages e.g. '/home/events'
+queue_limit  (number)    maximum limit for undelivered messages, defaults to '10000'
+comment      (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+notify_webhook[:name]  publish bucket notifications to webhook endpoints
+
+ARGS:
+MINIO_NOTIFY_WEBHOOK_ENDPOINT*    (url)       webhook server endpoint e.g. http://localhost:8080/minio/events
+MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN   (string)    opaque string or JWT authorization token
+MINIO_NOTIFY_WEBHOOK_QUEUE_DIR    (path)      staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT  (number)    maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_WEBHOOK_COMMENT      (sentence)  optionally add a comment to this setting
+```
+
 ```sh
-$ mc admin config get myminio/ notify_webhook
-notify_webhook:1 queue_limit="0"  endpoint="" queue_dir=""
+$ mc admin config get myminio/ notify_webhook notify_webhook:1 queue_limit="0"  endpoint="" queue_dir=""
 ```
 
 Use `mc admin config set` command to update the configuration for the deployment. Here the endpoint is the server listening for webhook notifications. Save the settings and restart the MinIO server for changes to take effect. Note that the endpoint needs to be live and reachable when you restart your MinIO server.
@@ -1039,6 +1297,35 @@ MinIO supports persistent event store. The persistent store will backup events w
 
 To update the configuration, use `mc admin config get` command to get the current configuration for `notify_nsq`.
 
+```
+KEY:
+notify_nsq[:name]  publish bucket notifications to NSQ endpoints
+
+ARGS:
+nsqd_address*    (address)   NSQ server address e.g. '127.0.0.1:4150'
+topic*           (string)    NSQ topic
+tls              (on|off)    set to 'on' to enable TLS
+tls_skip_verify  (on|off)    trust server TLS without verification, defaults to "on" (verify)
+queue_dir        (path)      staging dir for undelivered messages e.g. '/home/events'
+queue_limit      (number)    maximum limit for undelivered messages, defaults to '10000'
+comment          (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+notify_nsq[:name]  publish bucket notifications to NSQ endpoints
+
+ARGS:
+MINIO_NOTIFY_NSQ_NSQD_ADDRESS*    (address)   NSQ server address e.g. '127.0.0.1:4150'
+MINIO_NOTIFY_NSQ_TOPIC*           (string)    NSQ topic
+MINIO_NOTIFY_NSQ_TLS              (on|off)    set to 'on' to enable TLS
+MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY  (on|off)    trust server TLS without verification, defaults to "on" (verify)
+MINIO_NOTIFY_NSQ_QUEUE_DIR        (path)      staging dir for undelivered messages e.g. '/home/events'
+MINIO_NOTIFY_NSQ_QUEUE_LIMIT      (number)    maximum limit for undelivered messages, defaults to '10000'
+MINIO_NOTIFY_NSQ_COMMENT          (sentence)  optionally add a comment to this setting
+```
+
 ```sh
 $ mc admin config get myminio/ notify_nsq
 notify_nsq:1 nsqd_address="" queue_dir="" queue_limit="0"  tls_enable="off" tls_skip_verify="off" topic=""

docs/config/README.md

@@ -59,9 +59,24 @@ Once the migration is complete, server will automatically unset the `MINIO_ACCES
 > **NOTE: Make sure to remove `MINIO_ACCESS_KEY_OLD` and `MINIO_SECRET_KEY_OLD` in scripts or service files before next service restarts of the server to avoid double encryption of your existing contents.**
 
 #### Region
-| Field                     | Type     | Description                                                                                                                                                                             |
-|:--------------------------|:---------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| ``region name=my_region`` | _string_ | `region` describes the physical location of the server. By default it is blank. You may override this field with `MINIO_REGION_NAME` environment variable. If you are unsure leave it unset. |
+```
+KEY:
+region  label the location of the server
+
+ARGS:
+name     (string)    name of the location of the server e.g. "us-west-rack2"
+comment  (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+region  label the location of the server
+
+ARGS:
+MINIO_REGION_NAME     (string)    name of the location of the server e.g. "us-west-rack2"
+MINIO_REGION_COMMENT  (sentence)  optionally add a comment to this setting
+```
 
 Example:
 
@@ -71,37 +86,104 @@ minio server /data
 ```
 
 ### Storage Class
-
-| Field                          | Type     | Description                                                                                                                                                                                  |
-|:-------------------------------|:---------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| ``storage_class``               |          | Set storage class for configurable data and parity, as per object basis.                                                                                                                     |
-| ``storage_class standard=EC:4`` | _string_ | Value for standard storage class. It should be in the format `EC:Parity`, for example to set 4 disk parity for standard storage class objects, set this field to `EC:4`.                     |
-| ``storage_class rrs=EC:2``      | _string_ | Value for reduced redundancy storage class. It should be in the format `EC:Parity`, for example to set 3 disk parity for reduced redundancy storage class objects, set this field to `EC:3`. |
-
 By default, parity for objects with standard storage class is set to `N/2`, and parity for objects with reduced redundancy storage class objects is set to `2`. Read more about storage class support in MinIO server [here](https://github.com/minio/minio/blob/master/docs/erasure/storage-class/README.md).
 
+```
+KEY:
+storage_class  define object level redundancy
+
+ARGS:
+standard  (string)    set the parity count for default standard storage class e.g. "EC:4"
+rrs       (string)    set the parity count for reduced redundancy storage class e.g. "EC:2"
+comment   (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+storage_class  define object level redundancy
+
+ARGS:
+MINIO_STORAGE_CLASS_STANDARD  (string)    set the parity count for default standard storage class e.g. "EC:4"
+MINIO_STORAGE_CLASS_RRS       (string)    set the parity count for reduced redundancy storage class e.g. "EC:2"
+MINIO_STORAGE_CLASS_COMMENT   (sentence)  optionally add a comment to this setting
+```
+
 ### Cache
-| Field                                                        | Type       | Description                                                                                                 |
-|:-------------------------------------------------------------|:-----------|:------------------------------------------------------------------------------------------------------------|
-| ``cache drives="/mnt/drive1,/mnt/drive2,/mnt/cache{1...3}"`` | _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled |
-| ``cache exclude="*.pdf,mybucket/*"``                         | _[]string_ | List of wildcard patterns for prefixes to exclude from cache                                                |
-| ``cache expiry=90``                                          | _int_      | Days to cache expiry                                                                                        |
-| ``cache quota=70``                                           | _int_      | Percentage of disk available to cache                                                                       |
-|                                                              |            |                                                                                                             |
+MinIO provides caching storage tier for primarily gateway deployments, allowing you to cache content for faster reads, cost savings on repeated downloads from the cloud.
 
-#### Notify
+```
+KEY:
+cache  add caching storage tier
 
-| Field                    | Type | Description                                                                                                                           |
-|:-------------------------|:-----|:--------------------------------------------------------------------------------------------------------------------------------------|
-| ``notify_amqp``          |      | [Configure to publish MinIO events via AMQP target.](https://docs.min.io/docs/minio-bucket-notification-guide#AMQP)                   |
-| ``notify_nats``          |      | [Configure to publish MinIO events via NATS target.](https://docs.min.io/docs/minio-bucket-notification-guide#NATS)                   |
-| ``notify_elasticsearch`` |      | [Configure to publish MinIO events via Elasticsearch target.](https://docs.min.io/docs/minio-bucket-notification-guide#Elasticsearch) |
-| ``notify_redis``         |      | [Configure to publish MinIO events via Redis target.](https://docs.min.io/docs/minio-bucket-notification-guide#Redis)                 |
-| ``notify_postgresql``    |      | [Configure to publish MinIO events via PostgreSQL target.](https://docs.min.io/docs/minio-bucket-notification-guide#PostgreSQL)       |
-| ``notify_kafka``         |      | [Configure to publish MinIO events via Apache Kafka target.](https://docs.min.io/docs/minio-bucket-notification-guide#apache-kafka)   |
-| ``notify_webhook``       |      | [Configure to publish MinIO events via Webhooks target.](https://docs.min.io/docs/minio-bucket-notification-guide#webhooks)           |
-| ``notify_mysql``         |      | [Configure to publish MinIO events via MySql target.](https://docs.min.io/docs/minio-bucket-notification-guide#MySQL)                 |
-| ``notify_mqtt``          |      | [Configure to publish MinIO events via MQTT target.](https://docs.min.io/docs/minio-bucket-notification-guide#MQTT)                   |
+ARGS:
+drives*  (csv)       comma separated mountpoints e.g. "/optane1,/optane2"
+expiry   (number)    cache expiry duration in days e.g. "90"
+quota    (number)    limit cache drive usage in percentage e.g. "90"
+exclude  (csv)       comma separated wildcard exclusion patterns e.g. "bucket/*.tmp,*.exe"
+comment  (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+cache  add caching storage tier
+
+ARGS:
+MINIO_CACHE_DRIVES*  (csv)       comma separated mountpoints e.g. "/optane1,/optane2"
+MINIO_CACHE_EXPIRY   (number)    cache expiry duration in days e.g. "90"
+MINIO_CACHE_QUOTA    (number)    limit cache drive usage in percentage e.g. "90"
+MINIO_CACHE_EXCLUDE  (csv)       comma separated wildcard exclusion patterns e.g. "bucket/*.tmp,*.exe"
+MINIO_CACHE_COMMENT  (sentence)  optionally add a comment to this setting
+```
+
+#### Etcd
+MinIO supports storing encrypted IAM assets and bucket DNS records on etcd.
+
+> NOTE: if *path_prefix* is set then MinIO will not federate your buckets, namespaced IAM assets are assumed as isolated tenants, only buckets are considered globally unique but performing a lookup with a *bucket* which belongs to a different tenant will fail unlike federated setups where MinIO would port-forward and route the request to relevant cluster accordingly. This is a special feature, federated deployments should not need to set *path_prefix*.
+
+```
+KEY:
+etcd  federate multiple clusters for IAM and Bucket DNS
+
+ARGS:
+endpoints*       (csv)       comma separated list of etcd endpoints e.g. "http://localhost:2379"
+path_prefix      (path)      namespace prefix to isolate tenants e.g. "customer1/"
+coredns_path     (path)      shared bucket DNS records, default is "/skydns"
+client_cert      (path)      client cert for mTLS authentication
+client_cert_key  (path)      client cert key for mTLS authentication
+comment          (sentence)  optionally add a comment to this setting
+```
+
+or environment variables
+```
+KEY:
+etcd  federate multiple clusters for IAM and Bucket DNS
+
+ARGS:
+MINIO_ETCD_ENDPOINTS*       (csv)       comma separated list of etcd endpoints e.g. "http://localhost:2379"
+MINIO_ETCD_PATH_PREFIX      (path)      namespace prefix to isolate tenants e.g. "customer1/"
+MINIO_ETCD_COREDNS_PATH     (path)      shared bucket DNS records, default is "/skydns"
+MINIO_ETCD_CLIENT_CERT      (path)      client cert for mTLS authentication
+MINIO_ETCD_CLIENT_CERT_KEY  (path)      client cert key for mTLS authentication
+MINIO_ETCD_COMMENT          (sentence)  optionally add a comment to this setting
+```
+
+#### Notifications
+Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://docs.min.io/docs/minio-bucket-notification-guide.html)
+
+```
+notify_webhook        publish bucket notifications to webhook endpoints
+notify_amqp           publish bucket notifications to AMQP endpoints
+notify_kafka          publish bucket notifications to Kafka endpoints
+notify_mqtt           publish bucket notifications to MQTT endpoints
+notify_nats           publish bucket notifications to NATS endpoints
+notify_nsq            publish bucket notifications to NSQ endpoints
+notify_mysql          publish bucket notifications to MySQL databases
+notify_postgres       publish bucket notifications to Postgres databases
+notify_elasticsearch  publish bucket notifications to Elasticsearch endpoints
+notify_redis          publish bucket notifications to Redis datastores
+```
 
 ### Accessing configuration file
 All configuration changes can be made using [`mc admin config` get/set commands](https://github.com/minio/mc/blob/master/docs/minio-admin-complete-guide.md). Following sections provide brief explanation of fields and how to customize them. A complete example of `config.json` is available [here](https://raw.githubusercontent.com/minio/minio/master/docs/config/config.sample.json)

docs/distributed/DESIGN.md

@@ -1,14 +1,15 @@
 # Distributed Server Design Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
-This document explains the design approach and advanced use cases of the MinIO distributed server.
+This document explains the design, architecture and advanced use cases of the MinIO distributed server.
 
 ## Command-line
 ```
 NAME:
-  minio server - start object storage server.
+  minio server - start object storage server
 
 USAGE:
   minio server [FLAGS] DIR1 [DIR2..]
   minio server [FLAGS] DIR{1...64}
+  minio server [FLAGS] DIR{1...64} DIR{65...128}
 
 DIR:
   DIR points to a directory on a filesystem. When you want to combine
@@ -48,46 +49,30 @@ Expansion of ellipses and choice of erasure sets based on this expansion is an a
 - In this algorithm, we also make sure that we spread the disks out evenly. MinIO server expands ellipses passed as arguments. Here is a sample expansion to demonstrate the process.
 
 ```
-minio server http://host{1...4}/export{1...8}
+minio server http://host{1...2}/export{1...8}
 ```
 
 Expected expansion
 ```
 > http://host1/export1
 > http://host2/export1
-> http://host3/export1
-> http://host4/export1
 > http://host1/export2
 > http://host2/export2
-> http://host3/export2
-> http://host4/export2
 > http://host1/export3
 > http://host2/export3
-> http://host3/export3
-> http://host4/export3
 > http://host1/export4
 > http://host2/export4
-> http://host3/export4
-> http://host4/export4
 > http://host1/export5
 > http://host2/export5
-> http://host3/export5
-> http://host4/export5
 > http://host1/export6
 > http://host2/export6
-> http://host3/export6
-> http://host4/export6
 > http://host1/export7
 > http://host2/export7
-> http://host3/export7
-> http://host4/export7
 > http://host1/export8
 > http://host2/export8
-> http://host3/export8
-> http://host4/export8
 ```
 
-A noticeable trait of this expansion is that it chooses unique hosts such that the erasure code is efficient across drives and hosts.
+*A noticeable trait of this expansion is that it chooses unique hosts such the setup provides maximum protection and availability.*
 
 - Choosing an erasure set for the object is decided during `PutObject()`, object names are used to find the right erasure set using the following pseudo code.
 ```go
@@ -103,6 +88,40 @@ Input for the key is the object name specified in `PutObject()`, returns a uniqu
 
 - MinIO does erasure coding at the object level not at the volume level, unlike other object storage vendors. This allows applications to choose different storage class by setting `x-amz-storage-class=STANDARD/REDUCED_REDUNDANCY` for each object uploads so effectively utilizing the capacity of the cluster. Additionally these can also be enforced using IAM policies to make sure the client uploads with correct HTTP headers.
 
+- MinIO also supports expansion of existing clusters in zones. Each zone is a self contained entity with same SLA's (read/write quorum) for each object as original cluster. By using the existing namespace for lookup validation MinIO ensures conflicting objects are not created. When no such object exists then MinIO simply uses the least used zone.
+
+*There are no limits on how many zones can be combined*
+
+```
+minio server http://host{1...32}/export{1...32} http://host{5...6}/export{1...8}
+```
+
+In above example there are two zones
+
+- 32 * 32 = 1024 drives zone1
+- 2 * 8 = 16 drives zone2
+
+> Notice the requirement of common SLA here original cluster had 1024 drives with 16 drives per erasure set, second zone is expected to have a minimum of 16 drives to match the original cluster SLA or it should be in multiples of 16.
+
+Following pseudo code returns the correct least used zone index to upload an object.
+```go
+func getAvailableZoneIdx(ctx context.Context) int {
+        zones := z.getZonesAvailableSpace(ctx)
+        total := zones.TotalAvailable()
+        // choose when we reach this many
+        choose := rand.Uint64() % total
+        atTotal := uint64(0)
+        for _, zone := range zones {
+                atTotal += zone.Available
+                if atTotal > choose && zone.Available > 0 {
+                        return zone.Index
+                }
+        }
+        // Should not happen, but print values just in case.
+        panic(fmt.Errorf("reached end of zones (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
+}
+```
+
 ## Other usages
 
 ### Advanced use cases with multiple ellipses
@@ -114,7 +133,7 @@ minio server /mnt/controller{1...4}/data{1...16}
 
 Standalone erasure coded configuration with 16 sets, 16 disks per set, across mounts and controllers.
 ```
-minio server /mnt{1..4}/controller{1...4}/data{1...16}
+minio server /mnt{1...4}/controller{1...4}/data{1...16}
 ```
 
 Distributed erasure coded configuration with 2 sets, 16 disks per set across hosts.
@@ -126,81 +145,3 @@ Distributed erasure coded configuration with rack level redundancy 32 sets in to
 ```
 minio server http://rack{1...4}-host{1...8}.example.net/export{1...16}
 ```
-
-## Backend `format.json` changes
-
-`format.json` has new fields
-
-- `disk` is changed to `this`
-- `jbod` is changed to `sets` , along with this change sets is also a two dimensional list representing total sets and disks per set.
-
-A sample `format.json` looks like below
-
-```json
-{
-  "version": "1",
-  "format": "xl",
-  "xl": {
-    "version": "2",
-    "this": "4ec63786-3dbd-4a9e-96f5-535f6e850fb1",
-    "sets": [
-    [
-      "4ec63786-3dbd-4a9e-96f5-535f6e850fb1",
-      "1f3cf889-bc90-44ca-be2a-732b53be6c9d",
-      "4b23eede-1846-482c-b96f-bfb647f058d3",
-      "e1f17302-a850-419d-8cdb-a9f884a63c92"
-    ], [
-      "2ca4c5c1-dccb-4198-a840-309fea3b5449",
-      "6d1e666e-a22c-4db4-a038-2545c2ccb6d5",
-      "d4fa35ab-710f-4423-a7c2-e1ca33124df0",
-      "88c65e8b-00cb-4037-a801-2549119c9a33"
-       ]
-    ],
-    "distributionAlgo": "CRCMOD"
-  }
-}
-```
-
-New `format-xl.go` behavior is format structure is used as a opaque type, `Format` field signifies the format of the backend. Once the format has been identified it is now the job of the identified backend to further interpret the next structures and validate them.
-
-```go
-type formatType string
-
-const (
-     formatFS formatType = "fs"
-     formatXL            = "xl"
-)
-
-type format struct {
-     Version string
-     Format  BackendFormat
-}
-```
-
-### Current format
-
-```go
-type formatXLV1 struct{
-     format
-     XL struct{
-        Version string
-        Disk string
-        JBOD []string
-     }
-}
-```
-
-### New format
-
-```go
-type formatXLV2 struct {
-        Version string `json:"version"`
-        Format  string `json:"format"`
-        XL      struct {
-                Version          string     `json:"version"`
-                This             string     `json:"this"`
-                Sets             [][]string `json:"sets"`
-                DistributionAlgo string     `json:"distributionAlgo"`
-        } `json:"xl"`
-}
-```

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # 9001 through 9004.
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+    image: minio/minio:RELEASE.2019-12-30T05-45-39Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+        image: minio/minio:RELEASE.2019-12-30T05-45-39Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-headless-service.yaml

@@ -5,6 +5,7 @@ metadata:
   labels:
     app: minio
 spec:
+  publishNotReadyAddresses: true
   clusterIP: None
   ports:
     - port: 9000

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -5,6 +5,7 @@ metadata:
   name: minio
 spec:
   serviceName: minio
+  podManagementPolicy: Parallel
   replicas: 4
   selector:
     matchLabels:
@@ -21,7 +22,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+        image: minio/minio:RELEASE.2019-12-30T05-45-39Z
         args:
         - server
         - http://minio-{0...3}.minio.default.svc.cluster.local/data
@@ -41,6 +42,15 @@ spec:
             port: 9000
           initialDelaySeconds: 120
           periodSeconds: 20
+        # Readiness probe detects situations where MinIO server instance
+        # is not ready to accept connections. Kubernetes automatically
+        # stops all the traffic to the pods if readiness checks fail.
+        readinessProbe:
+          httpGet:
+            path: /minio/health/ready
+            port: 9000
+          initialDelaySeconds: 120
+          periodSeconds: 20
   # These are converted to volume claims by the controller
   # and mounted at the paths mentioned above.
   volumeClaimTemplates:

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -24,7 +24,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+        image: minio/minio:RELEASE.2019-12-30T05-45-39Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -32,7 +32,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-12-17T23-16-33Z
+        image: minio/minio:RELEASE.2019-12-30T05-45-39Z
         args:
         - server
         - /data

go.mod

@@ -4,55 +4,82 @@ go 1.13
 
 require (
 	cloud.google.com/go v0.39.0
+	contrib.go.opencensus.io/exporter/ocagent v0.5.0 // indirect
 	github.com/Azure/azure-pipeline-go v0.2.1
 	github.com/Azure/azure-storage-blob-go v0.8.0
+	github.com/Azure/go-autorest v11.7.1+incompatible // indirect
 	github.com/Shopify/sarama v1.24.1
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/alecthomas/participle v0.2.1
 	github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5
 	github.com/aws/aws-sdk-go v1.20.21
+	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2
 	github.com/beevik/ntp v0.2.0
 	github.com/cheggaaa/pb v1.0.28
 	github.com/coredns/coredns v1.4.0
+	github.com/coreos/bbolt v1.3.3 // indirect
 	github.com/coreos/etcd v3.3.12+incompatible
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/djherbis/atime v1.0.0
 	github.com/dustin/go-humanize v1.0.0
+	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/eclipse/paho.mqtt.golang v1.2.0
 	github.com/elazarl/go-bindata-assetfs v1.0.0
 	github.com/fatih/color v1.7.0
 	github.com/fatih/structs v1.1.0
+	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/go-sql-driver/mysql v1.4.1
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible
+	github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f // indirect
 	github.com/gorilla/handlers v1.4.0
 	github.com/gorilla/mux v1.7.0
 	github.com/gorilla/rpc v1.2.0+incompatible
+	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.9.0 // indirect
+	github.com/hashicorp/go-hclog v0.9.2 // indirect
+	github.com/hashicorp/raft v1.1.1-0.20190703171940-f639636d18e0 // indirect
 	github.com/hashicorp/vault/api v1.0.4
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
+	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/json-iterator/go v1.1.7
-	github.com/klauspost/compress v1.8.3
+	github.com/klauspost/compress v1.9.4
+	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/pgzip v1.2.1
 	github.com/klauspost/readahead v1.3.1
 	github.com/klauspost/reedsolomon v1.9.3
 	github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3
 	github.com/lib/pq v1.1.1
+	github.com/mattn/go-colorable v0.1.1
 	github.com/mattn/go-ieproxy v0.0.0-20190805055040-f9202b1cfdeb // indirect
+	github.com/mattn/go-isatty v0.0.7
+	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/miekg/dns v1.1.8
 	github.com/minio/cli v1.22.0
 	github.com/minio/gokrb5/v7 v7.2.5
 	github.com/minio/hdfs/v3 v3.0.1
 	github.com/minio/highwayhash v1.0.0
 	github.com/minio/lsync v1.0.1
-	github.com/minio/mc v0.0.0-20191012041914-735aa139b19c
-	github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f
-	github.com/minio/minio-go/v6 v6.0.39
-	github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679
+	github.com/minio/minio-go/v6 v6.0.44
+	github.com/minio/parquet-go v0.0.0-20191231003236-20b3c07bcd2c
 	github.com/minio/sha256-simd v0.1.1
 	github.com/minio/sio v0.2.0
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/nats-io/gnatsd v1.4.1 // indirect
+	github.com/nats-io/go-nats v1.7.2 // indirect
+	github.com/nats-io/go-nats-streaming v0.4.4 // indirect
+	github.com/nats-io/nats-server v1.4.1 // indirect
 	github.com/nats-io/nats-server/v2 v2.1.2
+	github.com/nats-io/nats-streaming-server v0.14.2 // indirect
 	github.com/nats-io/nats.go v1.9.1
 	github.com/nats-io/stan.go v0.4.5
 	github.com/ncw/directio v1.0.5
@@ -60,18 +87,36 @@ require (
 	github.com/pkg/errors v0.8.1
 	github.com/pkg/profile v1.3.0
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
+	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81 // indirect
 	github.com/rjeczalik/notify v0.9.2
 	github.com/rs/cors v1.6.0
+	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/secure-io/sio-go v0.3.0
 	github.com/shirou/gopsutil v2.18.12+incompatible
 	github.com/sirupsen/logrus v1.4.2
 	github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e
+	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
+	github.com/soheilhy/cmux v0.1.4 // indirect
 	github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
+	github.com/ugorji/go v1.1.5-pre // indirect
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	go.etcd.io/bbolt v1.3.3 // indirect
 	go.uber.org/atomic v1.3.2
+	go.uber.org/multierr v1.1.0 // indirect
+	go.uber.org/zap v1.10.0 // indirect
 	golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f
+	golang.org/x/net v0.0.0-20190923162816-aa69164e4478 // indirect
+	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
 	golang.org/x/sys v0.0.0-20190922100055-0a153f010e69
+	golang.org/x/text v0.3.2 // indirect
 	google.golang.org/api v0.5.0
+	google.golang.org/appengine v1.6.0 // indirect
+	google.golang.org/genproto v0.0.0-20190513181449-d00d292a067c // indirect
+	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
+	gopkg.in/ini.v1 v1.48.0 // indirect
 	gopkg.in/ldap.v3 v3.0.3
 	gopkg.in/olivere/elastic.v5 v5.0.80
 	gopkg.in/yaml.v2 v2.2.4
@@ -79,15 +124,3 @@ require (
 
 // Added for go1.13 migration https://github.com/golang/go/issues/32805
 replace github.com/gorilla/rpc v1.2.0+incompatible => github.com/gorilla/rpc v1.2.0
-
-// Allow this for offline builds
-replace github.com/eapache/go-xerial-snappy => github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21
-
-replace github.com/eapache/queue => github.com/eapache/queue v1.1.0
-
-replace github.com/mattn/go-runewidth => github.com/mattn/go-runewidth v0.0.4
-
-replace github.com/mitchellh/mapstructure => github.com/mitchellh/mapstructure v1.1.2
-
-// Version 1.2.0 adds support for go modules
-replace github.com/hashicorp/vault => github.com/hashicorp/vault v1.2.0-beta2

mint/build/awscli/install.sh

@@ -15,6 +15,6 @@
 #  limitations under the License.
 #
 
-AWS_CLI_VERSION="1.11.177"
+AWS_CLI_VERSION="1.16.309"
 
 python -m pip install awscli==$AWS_CLI_VERSION

mint/preinstall.sh

@@ -19,7 +19,7 @@ export APT="apt --quiet --yes"
 export WGET="wget --quiet --no-check-certificate"
 
 # install nodejs source list
-if ! $WGET --output-document=- https://deb.nodesource.com/setup_6.x | bash -; then
+if ! $WGET --output-document=- https://deb.nodesource.com/setup_13.x | bash -; then
     echo "unable to set nodejs repository"
     exit 1
 fi
@@ -32,7 +32,7 @@ rm -f packages-microsoft-prod.deb
 $APT update
 
 # download and install golang
-GO_VERSION="1.13"
+GO_VERSION="1.13.5"
 GO_INSTALL_PATH="/usr/local"
 download_url="https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz"
 if ! $WGET --output-document=- "$download_url" | tar -C "${GO_INSTALL_PATH}" -zxf -; then

pkg/console/console.go

@@ -0,0 +1,435 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package console implements console printing helpers
+package console
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/mattn/go-colorable"
+	"github.com/mattn/go-isatty"
+)
+
+var (
+	// DebugPrint enables/disables console debug printing.
+	DebugPrint = false
+
+	// Used by the caller to print multiple lines atomically. Exposed by Lock/Unlock methods.
+	publicMutex = &sync.Mutex{}
+	// Used internally by console.
+	privateMutex = &sync.Mutex{}
+
+	stderrColoredOutput = colorable.NewColorableStderr()
+
+	// Print prints a message.
+	Print = func(data ...interface{}) {
+		consolePrint("Print", Theme["Print"], data...)
+	}
+
+	// PrintC prints a message with color.
+	PrintC = func(data ...interface{}) {
+		consolePrint("PrintC", Theme["PrintC"], data...)
+	}
+
+	// Printf prints a formatted message.
+	Printf = func(format string, data ...interface{}) {
+		consolePrintf("Print", Theme["Print"], format, data...)
+	}
+
+	// Println prints a message with a newline.
+	Println = func(data ...interface{}) {
+		consolePrintln("Print", Theme["Print"], data...)
+	}
+
+	// Fatal print a error message and exit.
+	Fatal = func(data ...interface{}) {
+		consolePrint("Fatal", Theme["Fatal"], data...)
+		os.Exit(1)
+	}
+
+	// Fatalf print a error message with a format specified and exit.
+	Fatalf = func(format string, data ...interface{}) {
+		consolePrintf("Fatal", Theme["Fatal"], format, data...)
+		os.Exit(1)
+	}
+
+	// Fatalln print a error message with a new line and exit.
+	Fatalln = func(data ...interface{}) {
+		consolePrintln("Fatal", Theme["Fatal"], data...)
+		os.Exit(1)
+	}
+
+	// Error prints a error message.
+	Error = func(data ...interface{}) {
+		consolePrint("Error", Theme["Error"], data...)
+	}
+
+	// Errorf print a error message with a format specified.
+	Errorf = func(format string, data ...interface{}) {
+		consolePrintf("Error", Theme["Error"], format, data...)
+	}
+
+	// Errorln prints a error message with a new line.
+	Errorln = func(data ...interface{}) {
+		consolePrintln("Error", Theme["Error"], data...)
+	}
+
+	// Info prints a informational message.
+	Info = func(data ...interface{}) {
+		consolePrint("Info", Theme["Info"], data...)
+	}
+
+	// Infof prints a informational message in custom format.
+	Infof = func(format string, data ...interface{}) {
+		consolePrintf("Info", Theme["Info"], format, data...)
+	}
+
+	// Infoln prints a informational message with a new line.
+	Infoln = func(data ...interface{}) {
+		consolePrintln("Info", Theme["Info"], data...)
+	}
+
+	// Debug prints a debug message without a new line
+	// Debug prints a debug message.
+	Debug = func(data ...interface{}) {
+		if DebugPrint {
+			consolePrint("Debug", Theme["Debug"], data...)
+		}
+	}
+
+	// Debugf prints a debug message with a new line.
+	Debugf = func(format string, data ...interface{}) {
+		if DebugPrint {
+			consolePrintf("Debug", Theme["Debug"], format, data...)
+		}
+	}
+
+	// Debugln prints a debug message with a new line.
+	Debugln = func(data ...interface{}) {
+		if DebugPrint {
+			consolePrintln("Debug", Theme["Debug"], data...)
+		}
+	}
+
+	// Colorize prints message in a colorized form, dictated by the corresponding tag argument.
+	Colorize = func(tag string, data interface{}) string {
+		if isatty.IsTerminal(os.Stdout.Fd()) {
+			colorized, ok := Theme[tag]
+			if ok {
+				return colorized.SprintFunc()(data)
+			} // else: No theme found. Return as string.
+		}
+		return fmt.Sprint(data)
+	}
+
+	// Eraseline Print in new line and adjust to top so that we don't print over the ongoing progress bar.
+	Eraseline = func() {
+		consolePrintf("Print", Theme["Print"], "%c[2K\n", 27)
+		consolePrintf("Print", Theme["Print"], "%c[A", 27)
+	}
+)
+
+// wrap around standard fmt functions.
+// consolePrint prints a message prefixed with message type and program name.
+func consolePrint(tag string, c *color.Color, a ...interface{}) {
+	privateMutex.Lock()
+	defer privateMutex.Unlock()
+
+	switch tag {
+	case "Debug":
+		// if no arguments are given do not invoke debug printer.
+		if len(a) == 0 {
+			return
+		}
+		output := color.Output
+		color.Output = stderrColoredOutput
+		if isatty.IsTerminal(os.Stderr.Fd()) {
+			c.Print(ProgramName() + ": <DEBUG> ")
+			c.Print(a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": <DEBUG> ")
+			fmt.Fprint(color.Output, a...)
+		}
+		color.Output = output
+	case "Fatal":
+		fallthrough
+	case "Error":
+		// if no arguments are given do not invoke fatal and error printer.
+		if len(a) == 0 {
+			return
+		}
+		output := color.Output
+		color.Output = stderrColoredOutput
+		if isatty.IsTerminal(os.Stderr.Fd()) {
+			c.Print(ProgramName() + ": <ERROR> ")
+			c.Print(a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": <ERROR> ")
+			fmt.Fprint(color.Output, a...)
+		}
+		color.Output = output
+	case "Info":
+		// if no arguments are given do not invoke info printer.
+		if len(a) == 0 {
+			return
+		}
+		if isatty.IsTerminal(os.Stdout.Fd()) {
+			c.Print(ProgramName() + ": ")
+			c.Print(a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": ")
+			fmt.Fprint(color.Output, a...)
+		}
+	default:
+		if isatty.IsTerminal(os.Stdout.Fd()) {
+			c.Print(a...)
+		} else {
+			fmt.Fprint(color.Output, a...)
+		}
+	}
+}
+
+// consolePrintf - same as print with a new line.
+func consolePrintf(tag string, c *color.Color, format string, a ...interface{}) {
+	privateMutex.Lock()
+	defer privateMutex.Unlock()
+
+	switch tag {
+	case "Debug":
+		// if no arguments are given do not invoke debug printer.
+		if len(a) == 0 {
+			return
+		}
+		output := color.Output
+		color.Output = stderrColoredOutput
+		if isatty.IsTerminal(os.Stderr.Fd()) {
+			c.Print(ProgramName() + ": <DEBUG> ")
+			c.Printf(format, a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": <DEBUG> ")
+			fmt.Fprintf(color.Output, format, a...)
+		}
+		color.Output = output
+	case "Fatal":
+		fallthrough
+	case "Error":
+		// if no arguments are given do not invoke fatal and error printer.
+		if len(a) == 0 {
+			return
+		}
+		output := color.Output
+		color.Output = stderrColoredOutput
+		if isatty.IsTerminal(os.Stderr.Fd()) {
+			c.Print(ProgramName() + ": <ERROR> ")
+			c.Printf(format, a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": <ERROR> ")
+			fmt.Fprintf(color.Output, format, a...)
+		}
+		color.Output = output
+	case "Info":
+		// if no arguments are given do not invoke info printer.
+		if len(a) == 0 {
+			return
+		}
+		if isatty.IsTerminal(os.Stdout.Fd()) {
+			c.Print(ProgramName() + ": ")
+			c.Printf(format, a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": ")
+			fmt.Fprintf(color.Output, format, a...)
+		}
+	default:
+		if isatty.IsTerminal(os.Stdout.Fd()) {
+			c.Printf(format, a...)
+		} else {
+			fmt.Fprintf(color.Output, format, a...)
+		}
+	}
+}
+
+// consolePrintln - same as print with a new line.
+func consolePrintln(tag string, c *color.Color, a ...interface{}) {
+	privateMutex.Lock()
+	defer privateMutex.Unlock()
+
+	switch tag {
+	case "Debug":
+		// if no arguments are given do not invoke debug printer.
+		if len(a) == 0 {
+			return
+		}
+		output := color.Output
+		color.Output = stderrColoredOutput
+		if isatty.IsTerminal(os.Stderr.Fd()) {
+			c.Print(ProgramName() + ": <DEBUG> ")
+			c.Println(a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": <DEBUG> ")
+			fmt.Fprintln(color.Output, a...)
+		}
+		color.Output = output
+	case "Fatal":
+		fallthrough
+	case "Error":
+		// if no arguments are given do not invoke fatal and error printer.
+		if len(a) == 0 {
+			return
+		}
+		output := color.Output
+		color.Output = stderrColoredOutput
+		if isatty.IsTerminal(os.Stderr.Fd()) {
+			c.Print(ProgramName() + ": <ERROR> ")
+			c.Println(a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": <ERROR> ")
+			fmt.Fprintln(color.Output, a...)
+		}
+		color.Output = output
+	case "Info":
+		// if no arguments are given do not invoke info printer.
+		if len(a) == 0 {
+			return
+		}
+		if isatty.IsTerminal(os.Stdout.Fd()) {
+			c.Print(ProgramName() + ": ")
+			c.Println(a...)
+		} else {
+			fmt.Fprint(color.Output, ProgramName()+": ")
+			fmt.Fprintln(color.Output, a...)
+		}
+	default:
+		if isatty.IsTerminal(os.Stdout.Fd()) {
+			c.Println(a...)
+		} else {
+			fmt.Fprintln(color.Output, a...)
+		}
+	}
+}
+
+// Lock console.
+func Lock() {
+	publicMutex.Lock()
+}
+
+// Unlock locked console.
+func Unlock() {
+	publicMutex.Unlock()
+}
+
+// ProgramName - return the name of the executable program.
+func ProgramName() string {
+	_, progName := filepath.Split(os.Args[0])
+	return progName
+}
+
+// Table - data to print in table format with fixed row widths.
+type Table struct {
+	// per-row colors
+	RowColors []*color.Color
+
+	// per-column align-right flag (aligns left by default)
+	AlignRight []bool
+
+	// Left margin width for table
+	TableIndentWidth int
+}
+
+// NewTable - create a new Table instance. Takes per-row colors and
+// per-column right-align flags and table indentation width (i.e. left
+// margin width)
+func NewTable(rowColors []*color.Color, alignRight []bool, indentWidth int) *Table {
+	return &Table{rowColors, alignRight, indentWidth}
+}
+
+// DisplayTable - prints the table
+func (t *Table) DisplayTable(rows [][]string) error {
+	numRows := len(rows)
+	numCols := len(rows[0])
+	if numRows != len(t.RowColors) {
+		return fmt.Errorf("row count and row-colors mismatch")
+	}
+
+	// Compute max. column widths
+	maxColWidths := make([]int, numCols)
+	for _, row := range rows {
+		if len(row) != len(t.AlignRight) {
+			return fmt.Errorf("col count and align-right mismatch")
+		}
+		for i, v := range row {
+			if len([]rune(v)) > maxColWidths[i] {
+				maxColWidths[i] = len([]rune(v))
+			}
+		}
+	}
+
+	// Compute per-cell text with padding and alignment applied.
+	paddedText := make([][]string, numRows)
+	for r, row := range rows {
+		paddedText[r] = make([]string, numCols)
+		for c, cell := range row {
+			if t.AlignRight[c] {
+				fmtStr := fmt.Sprintf("%%%ds", maxColWidths[c])
+				paddedText[r][c] = fmt.Sprintf(fmtStr, cell)
+			} else {
+				extraWidth := maxColWidths[c] - len([]rune(cell))
+				fmtStr := fmt.Sprintf("%%s%%%ds", extraWidth)
+				paddedText[r][c] = fmt.Sprintf(fmtStr, cell, "")
+			}
+		}
+	}
+
+	// Draw table top border
+	segments := make([]string, numCols)
+	for i, c := range maxColWidths {
+		segments[i] = strings.Repeat("─", c+2)
+	}
+	indentText := strings.Repeat(" ", t.TableIndentWidth)
+	border := fmt.Sprintf("%s┌%s┐", indentText, strings.Join(segments, "┬"))
+	fmt.Println(border)
+
+	// Print the table with colors
+	for r, row := range paddedText {
+		fmt.Print(indentText + "│ ")
+		for c, text := range row {
+			t.RowColors[r].Print(text)
+			if c != numCols-1 {
+				fmt.Print(" │ ")
+			}
+		}
+		fmt.Println(" │")
+	}
+
+	// Draw table bottom border
+	border = fmt.Sprintf("%s└%s┘", indentText, strings.Join(segments, "┴"))
+	fmt.Println(border)
+
+	return nil
+}
+
+// RewindLines - uses terminal escape symbols to clear and rewind
+// upwards on the console for `n` lines.
+func RewindLines(n int) {
+	for i := 0; i < n; i++ {
+		fmt.Printf("\033[1A\033[K")
+	}
+}

pkg/console/console_test.go

@@ -0,0 +1,37 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package console
+
+import (
+	"testing"
+
+	"github.com/fatih/color"
+)
+
+func TestSetColor(t *testing.T) {
+	SetColor("unknown", color.New(color.FgWhite))
+	_, ok := Theme["unknown"]
+	if !ok {
+		t.Fatal("missing theme")
+	}
+}
+
+func TestColorLock(t *testing.T) {
+	Lock()
+	Print("") // Test for deadlocks.
+	Unlock()
+}

pkg/console/themes.go

@@ -0,0 +1,54 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package console
+
+import "github.com/fatih/color"
+
+var (
+	// Theme contains default color mapping.
+	Theme = map[string]*color.Color{
+		"Debug":  color.New(color.FgWhite, color.Faint, color.Italic),
+		"Fatal":  color.New(color.FgRed, color.Italic, color.Bold),
+		"Error":  color.New(color.FgYellow, color.Italic),
+		"Info":   color.New(color.FgGreen, color.Bold),
+		"Print":  color.New(),
+		"PrintB": color.New(color.FgBlue, color.Bold),
+		"PrintC": color.New(color.FgGreen, color.Bold),
+	}
+)
+
+// SetColorOff disables coloring for the entire session.
+func SetColorOff() {
+	privateMutex.Lock()
+	defer privateMutex.Unlock()
+	color.NoColor = true
+}
+
+// SetColorOn enables coloring for the entire session.
+func SetColorOn() {
+	privateMutex.Lock()
+	defer privateMutex.Unlock()
+	color.NoColor = false
+}
+
+// SetColor sets a color for a particular tag.
+func SetColor(tag string, cl *color.Color) {
+	privateMutex.Lock()
+	defer privateMutex.Unlock()
+	// add new theme
+	Theme[tag] = cl
+}

pkg/ioutil/ioutil.go

@@ -30,7 +30,7 @@ import (
 const defaultAppendBufferSize = humanize.MiByte
 
 // WriteOnCloser implements io.WriteCloser and always
-// exectues at least one write operation if it is closed.
+// executes at least one write operation if it is closed.
 //
 // This can be useful within the context of HTTP. At least
 // one write operation must happen to send the HTTP headers

pkg/policy/action.go

@@ -85,10 +85,10 @@ const (
 	PutObjectAction = "s3:PutObject"
 
 	// PutBucketLifecycleAction - PutBucketLifecycle Rest API action.
-	PutBucketLifecycleAction = "s3:PutBucketLifecycle"
+	PutBucketLifecycleAction = "s3:PutLifecycleConfiguration"
 
 	// GetBucketLifecycleAction - GetBucketLifecycle Rest API action.
-	GetBucketLifecycleAction = "s3:GetBucketLifecycle"
+	GetBucketLifecycleAction = "s3:GetLifecycleConfiguration"
 
 	// BypassGovernanceModeAction - bypass governance mode for DeleteObject Rest API action.
 	BypassGovernanceModeAction = "s3:BypassGovernanceMode"

pkg/s3select/csv/reader.go

@@ -84,7 +84,7 @@ func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
 		}
 	}
 
-	// If no index max, add that.
+	// If no index map, add that.
 	if r.nameIndexMap == nil {
 		r.nameIndexMap = make(map[string]int64)
 		for i := range r.columnNames {