simplify webhook DNS further generalize for gateway (#10448)

continuation of the changes from eaaf05a7cc
this further simplifies, enables this for gateway deployments as well

README.md

@@ -88,7 +88,7 @@ service minio start
 ```
 
 ## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.13](https://golang.org/dl/#stable)
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.14](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio

README_zh_CN.md

@@ -89,7 +89,7 @@ service minio start
 
 ## 使用源码安装
 
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.13](https://golang.org/dl/#stable)
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.14](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio

cmd/api-errors.go

@@ -29,7 +29,7 @@ import (
 
 	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"

cmd/api-router.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -66,6 +67,16 @@ type objectAPIHandlers struct {
 	AllowSSEKMS func() bool
 }
 
+// getHost tries its best to return the request host.
+// According to section 14.23 of RFC 2616 the Host header
+// can include the port number if the default value of 80 is not used.
+func getHost(r *http.Request) string {
+	if r.URL.IsAbs() {
+		return r.URL.Host
+	}
+	return r.Host
+}
+
 // registerAPIRouter - registers S3 compatible APIs.
 func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
 	// Initialize API.
@@ -82,9 +93,28 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 
 	// API Router
 	apiRouter := router.PathPrefix(SlashSeparator).Subrouter()
+
 	var routers []*mux.Router
 	for _, domainName := range globalDomainNames {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		if IsKubernetes() {
+			routers = append(routers, apiRouter.MatcherFunc(func(r *http.Request, match *mux.RouteMatch) bool {
+				host, _, _ := net.SplitHostPort(getHost(r))
+				// Make sure to skip matching minio.<domain>` this is
+				// specifically meant for operator/k8s deployment
+				// The reason we need to skip this is for a special
+				// usecase where we need to make sure that
+				// minio.<namespace>.svc.<cluster_domain> is ignored
+				// by the bucketDNS style to ensure that path style
+				// is available and honored at this domain.
+				//
+				// All other `<bucket>.<namespace>.svc.<cluster_domain>`
+				// makes sure that buckets are routed through this matcher
+				// to match for `<bucket>`
+				return host != minioReservedBucket+"."+domainName
+			}).Host("{bucket:.+}."+domainName).Subrouter())
+		} else {
+			routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		}
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
@@ -94,7 +124,10 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler))))
 		// CopyObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).
+			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
@@ -138,7 +171,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler))))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
 		// PutObjectRetention
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")

cmd/bucket-handlers.go

@@ -33,7 +33,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -72,7 +72,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
-	if err != nil && err != dns.ErrNoEntriesFound {
+	if err != nil && err != dns.ErrNoEntriesFound && err != dns.ErrNotImplemented {
 		logger.LogIf(GlobalContext, err)
 		return
 	}
@@ -80,33 +80,35 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	bucketsSet := set.NewStringSet()
 	bucketsToBeUpdated := set.NewStringSet()
 	bucketsInConflict := set.NewStringSet()
-	for _, bucket := range buckets {
-		bucketsSet.Add(bucket.Name)
-		r, ok := dnsBuckets[bucket.Name]
-		if !ok {
-			bucketsToBeUpdated.Add(bucket.Name)
-			continue
-		}
-		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-			if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				// No difference in terms of domainIPs and nothing
-				// has changed so we don't change anything on the etcd.
+	if dnsBuckets != nil {
+		for _, bucket := range buckets {
+			bucketsSet.Add(bucket.Name)
+			r, ok := dnsBuckets[bucket.Name]
+			if !ok {
+				bucketsToBeUpdated.Add(bucket.Name)
 				continue
 			}
-			// if domain IPs intersect then it won't be an empty set.
-			// such an intersection means that bucket exists on etcd.
-			// but if we do see a difference with local domain IPs with
-			// hostSlice from etcd then we should update with newer
-			// domainIPs, we proceed to do that here.
-			bucketsToBeUpdated.Add(bucket.Name)
-			continue
+			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+					// No difference in terms of domainIPs and nothing
+					// has changed so we don't change anything on the etcd.
+					continue
+				}
+				// if domain IPs intersect then it won't be an empty set.
+				// such an intersection means that bucket exists on etcd.
+				// but if we do see a difference with local domain IPs with
+				// hostSlice from etcd then we should update with newer
+				// domainIPs, we proceed to do that here.
+				bucketsToBeUpdated.Add(bucket.Name)
+				continue
+			}
+			// No IPs seem to intersect, this means that bucket exists but has
+			// different IP addresses perhaps from a different deployment.
+			// bucket names are globally unique in federation at a given
+			// path prefix, name collision is not allowed. We simply log
+			// an error and continue.
+			bucketsInConflict.Add(bucket.Name)
 		}
-		// No IPs seem to intersect, this means that bucket exists but has
-		// different IP addresses perhaps from a different deployment.
-		// bucket names are globally unique in federation at a given
-		// path prefix, name collision is not allowed. We simply log
-		// an error and continue.
-		bucketsInConflict.Add(bucket.Name)
 	}
 
 	// Add/update buckets that are not registered with the DNS
@@ -562,7 +564,9 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	if globalDNSConfig != nil {
 		sr, err := globalDNSConfig.Get(bucket)
 		if err != nil {
-			if err == dns.ErrNoEntriesFound {
+			// ErrNotImplemented indicates a DNS backend that doesn't need to check if bucket already
+			// exists elsewhere
+			if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -1000,7 +1004,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}

cmd/bucket-metadata.go

@@ -354,7 +354,7 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 
 // deleteBucketMetadata deletes bucket metadata
 // If config does not exist no error is returned.
-func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
+func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string) error {
 	metadataFiles := []string{
 		dataUsageCacheName,
 		bucketMetadataFile,

cmd/config-common.go

@@ -46,7 +46,11 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 	return buffer.Bytes(), nil
 }
 
-func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+type objectDeleter interface {
+	DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
+}
+
+func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error {
 	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound

cmd/config-current.go

@@ -25,8 +25,8 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
-	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
@@ -321,6 +321,19 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		}
 	}
 
+	if dnsURL, dnsUser, dnsPass, ok := env.LookupEnv(config.EnvDNSWebhook); ok {
+		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
+			dns.Authentication(dnsUser, dnsPass),
+			dns.RootCAs(globalRootCAs))
+		if err != nil {
+			if globalIsGateway {
+				logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+			}
+		}
+	}
+
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		if globalIsGateway {
@@ -342,19 +355,25 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 
-		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil && globalDNSConfig == nil {
-			globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
-				dns.DomainNames(globalDomainNames),
-				dns.DomainIPs(globalDomainIPs),
-				dns.DomainPort(globalMinioPort),
-				dns.CoreDNSPath(etcdCfg.CoreDNSPath),
-			)
-			if err != nil {
-				if globalIsGateway {
-					logger.FatalIf(err, "Unable to initialize DNS config")
-				} else {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-						globalDomainNames, err))
+		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+			if globalDNSConfig != nil {
+				// if global DNS is already configured, indicate with a warning, incase
+				// users are confused.
+				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
+			} else {
+				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
+					dns.DomainNames(globalDomainNames),
+					dns.DomainIPs(globalDomainIPs),
+					dns.DomainPort(globalMinioPort),
+					dns.CoreDNSPath(etcdCfg.CoreDNSPath),
+				)
+				if err != nil {
+					if globalIsGateway {
+						logger.FatalIf(err, "Unable to initialize DNS config")
+					} else {
+						logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+							globalDomainNames, err))
+					}
 				}
 			}
 		}

cmd/config/constants.go

@@ -33,6 +33,7 @@ const (
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
 	EnvArgs         = "MINIO_ARGS"
+	EnvDNSWebhook   = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
 	EnvUpdate = "MINIO_UPDATE"
 

cmd/config/dns/etcd_dns.go

@@ -26,9 +26,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-go/v7/pkg/set"
-
 	"github.com/coredns/coredns/plugin/etcd/msg"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"go.etcd.io/etcd/v3/clientv3"
 )
 
@@ -214,6 +213,11 @@ func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
 	return nil
 }
 
+// String stringer name for this implementation of dns.Store
+func (c *CoreDNS) String() string {
+	return "etcdDNS"
+}
+
 // CoreDNS - represents dns config for coredns server.
 type CoreDNS struct {
 	domainNames []string
@@ -223,13 +227,13 @@ type CoreDNS struct {
 	etcdClient  *clientv3.Client
 }
 
-// Option - functional options pattern style
-type Option func(*CoreDNS)
+// EtcdOption - functional options pattern style
+type EtcdOption func(*CoreDNS)
 
 // DomainNames set a list of domain names used by this CoreDNS
 // client setting, note this will fail if set to empty when
 // constructor initializes.
-func DomainNames(domainNames []string) Option {
+func DomainNames(domainNames []string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainNames = domainNames
 	}
@@ -237,14 +241,14 @@ func DomainNames(domainNames []string) Option {
 
 // DomainIPs set a list of custom domain IPs, note this will
 // fail if set to empty when constructor initializes.
-func DomainIPs(domainIPs set.StringSet) Option {
+func DomainIPs(domainIPs set.StringSet) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainIPs = domainIPs
 	}
 }
 
 // DomainPort - is a string version of server port
-func DomainPort(domainPort string) Option {
+func DomainPort(domainPort string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainPort = domainPort
 	}
@@ -253,14 +257,14 @@ func DomainPort(domainPort string) Option {
 // CoreDNSPath - custom prefix on etcd to populate DNS
 // service records, optional and can be empty.
 // if empty then c.prefixPath is used i.e "/skydns"
-func CoreDNSPath(prefix string) Option {
+func CoreDNSPath(prefix string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.prefixPath = prefix
 	}
 }
 
 // NewCoreDNS - initialize a new coreDNS set/unset values.
-func NewCoreDNS(cfg clientv3.Config, setters ...Option) (Store, error) {
+func NewCoreDNS(cfg clientv3.Config, setters ...EtcdOption) (Store, error) {
 	etcdClient, err := clientv3.New(cfg)
 	if err != nil {
 		return nil, err

cmd/config/dns/operator_dns.go

@@ -0,0 +1,221 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dns
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/dgrijalva/jwt-go"
+	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
+)
+
+var (
+	defaultOperatorContextTimeout = 10 * time.Second
+	// ErrNotImplemented - Indicates the functionality which is not implemented
+	ErrNotImplemented = errors.New("The method is not implemented")
+)
+
+func (c *OperatorDNS) addAuthHeader(r *http.Request) error {
+	if c.username == "" || c.password == "" {
+		return nil
+	}
+
+	claims := &jwt.StandardClaims{
+		ExpiresAt: int64(15 * time.Minute),
+		Issuer:    c.username,
+		Subject:   config.EnvDNSWebhook,
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
+	ss, err := token.SignedString([]byte(c.password))
+	if err != nil {
+		return err
+	}
+
+	r.Header.Set("Authorization", "Bearer "+ss)
+	return nil
+}
+
+func (c *OperatorDNS) endpoint(bucket string, delete bool) (string, error) {
+	u, err := url.Parse(c.Endpoint)
+	if err != nil {
+		return "", err
+	}
+	q := u.Query()
+	q.Add("bucket", bucket)
+	q.Add("delete", strconv.FormatBool(delete))
+	u.RawQuery = q.Encode()
+	return u.String(), nil
+}
+
+// Put - Adds DNS entries into operator webhook server
+func (c *OperatorDNS) Put(bucket string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
+	defer cancel()
+	e, err := c.endpoint(bucket, false)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
+	if err != nil {
+		return err
+	}
+	if err = c.addAuthHeader(req); err != nil {
+		return err
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		if derr := c.Delete(bucket); derr != nil {
+			return derr
+		}
+	}
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request to create the service for bucket %s, failed with status %s", bucket, resp.Status)
+	}
+	return nil
+}
+
+// Delete - Removes DNS entries added in Put().
+func (c *OperatorDNS) Delete(bucket string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
+	defer cancel()
+	e, err := c.endpoint(bucket, true)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
+	if err != nil {
+		return err
+	}
+	if err = c.addAuthHeader(req); err != nil {
+		return err
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request to delete the service for bucket %s, failed with status %s", bucket, resp.Status)
+	}
+	return nil
+}
+
+// DeleteRecord - Removes a specific DNS entry
+// No Op for Operator because operator deals on with bucket entries
+func (c *OperatorDNS) DeleteRecord(record SrvRecord) error {
+	return ErrNotImplemented
+}
+
+// Close closes the internal http client
+func (c *OperatorDNS) Close() error {
+	c.httpClient.CloseIdleConnections()
+	return nil
+}
+
+// List - Retrieves list of DNS entries for the domain.
+// This is a No Op for Operator because, there is no intent to enforce global
+// namespace at MinIO level with this DNS entry. The global namespace in
+// enforced by the Kubernetes Operator
+func (c *OperatorDNS) List() (srvRecords map[string][]SrvRecord, err error) {
+	return nil, ErrNotImplemented
+}
+
+// Get - Retrieves DNS records for a bucket.
+// This is a No Op for Operator because, there is no intent to enforce global
+// namespace at MinIO level with this DNS entry. The global namespace in
+// enforced by the Kubernetes Operator
+func (c *OperatorDNS) Get(bucket string) (srvRecords []SrvRecord, err error) {
+	return nil, ErrNotImplemented
+}
+
+// String stringer name for this implementation of dns.Store
+func (c *OperatorDNS) String() string {
+	return "webhookDNS"
+}
+
+// OperatorDNS - represents dns config for MinIO k8s operator.
+type OperatorDNS struct {
+	httpClient *http.Client
+	Endpoint   string
+	rootCAs    *x509.CertPool
+	username   string
+	password   string
+}
+
+// OperatorOption - functional options pattern style for OperatorDNS
+type OperatorOption func(*OperatorDNS)
+
+// Authentication - custom username and password for authenticating at the endpoint
+func Authentication(username, password string) OperatorOption {
+	return func(args *OperatorDNS) {
+		args.username = username
+		args.password = password
+	}
+}
+
+// RootCAs - add custom trust certs pool
+func RootCAs(CAs *x509.CertPool) OperatorOption {
+	return func(args *OperatorDNS) {
+		args.rootCAs = CAs
+	}
+}
+
+// NewOperatorDNS - initialize a new K8S Operator DNS set/unset values.
+func NewOperatorDNS(endpoint string, setters ...OperatorOption) (Store, error) {
+	if endpoint == "" {
+		return nil, errors.New("invalid argument")
+	}
+
+	args := &OperatorDNS{
+		Endpoint: endpoint,
+	}
+	for _, setter := range setters {
+		setter(args)
+	}
+	args.httpClient = &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   3 * time.Second,
+				KeepAlive: 5 * time.Second,
+			}).DialContext,
+			ResponseHeaderTimeout: 3 * time.Second,
+			TLSHandshakeTimeout:   3 * time.Second,
+			ExpectContinueTimeout: 3 * time.Second,
+			TLSClientConfig: &tls.Config{
+				RootCAs: args.rootCAs,
+			},
+			// Go net/http automatically unzip if content-type is
+			// gzip disable this feature, as we are always interested
+			// in raw stream.
+			DisableCompression: true,
+		},
+	}
+	return args, nil
+}

cmd/config/dns/store.go

@@ -24,4 +24,5 @@ type Store interface {
 	List() (map[string][]SrvRecord, error)
 	DeleteRecord(record SrvRecord) error
 	Close() error
+	String() string
 }

cmd/crypto/metadata.go

@@ -129,7 +129,7 @@ func (ssec) IsEncrypted(metadata map[string]string) bool {
 // metadata is nil.
 func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 	if metadata == nil {
-		metadata = map[string]string{}
+		return map[string]string{SSEMultipart: ""}
 	}
 	metadata[SSEMultipart] = ""
 	return metadata
@@ -156,7 +156,7 @@ func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte
 	}
 
 	if metadata == nil {
-		metadata = map[string]string{}
+		metadata = make(map[string]string, 5)
 	}
 
 	metadata[SSESealAlgorithm] = sealedKey.Algorithm
@@ -236,7 +236,7 @@ func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[
 	}
 
 	if metadata == nil {
-		metadata = map[string]string{}
+		metadata = make(map[string]string, 3)
 	}
 	metadata[SSESealAlgorithm] = SealAlgorithm
 	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])

cmd/data-crawler.go

@@ -602,8 +602,9 @@ func (i *crawlItem) transformMetaDir() {
 
 // actionMeta contains information used to apply actions.
 type actionMeta struct {
-	oi          ObjectInfo
-	numVersions int // The number of versions of this object
+	oi               ObjectInfo
+	successorModTime time.Time // The modtime of the successor version
+	numVersions      int       // The number of versions of this object
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -636,13 +637,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	versionID := meta.oi.VersionID
 	action := i.lifeCycle.ComputeAction(
 		lifecycle.ObjectOpts{
-			Name:         i.objectPath(),
-			UserTags:     meta.oi.UserTags,
-			ModTime:      meta.oi.ModTime,
-			VersionID:    meta.oi.VersionID,
-			DeleteMarker: meta.oi.DeleteMarker,
-			IsLatest:     meta.oi.IsLatest,
-			NumVersions:  meta.numVersions,
+			Name:             i.objectPath(),
+			UserTags:         meta.oi.UserTags,
+			ModTime:          meta.oi.ModTime,
+			VersionID:        meta.oi.VersionID,
+			DeleteMarker:     meta.oi.DeleteMarker,
+			IsLatest:         meta.oi.IsLatest,
+			NumVersions:      meta.numVersions,
+			SuccessorModTime: meta.successorModTime,
 		})
 	if i.debug {
 		logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
@@ -679,13 +681,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	// Recalculate action.
 	action = i.lifeCycle.ComputeAction(
 		lifecycle.ObjectOpts{
-			Name:         i.objectPath(),
-			UserTags:     obj.UserTags,
-			ModTime:      obj.ModTime,
-			VersionID:    obj.VersionID,
-			DeleteMarker: obj.DeleteMarker,
-			IsLatest:     obj.IsLatest,
-			NumVersions:  meta.numVersions,
+			Name:             i.objectPath(),
+			UserTags:         obj.UserTags,
+			ModTime:          obj.ModTime,
+			VersionID:        obj.VersionID,
+			DeleteMarker:     obj.DeleteMarker,
+			IsLatest:         obj.IsLatest,
+			NumVersions:      meta.numVersions,
+			SuccessorModTime: meta.successorModTime,
 		})
 	if i.debug {
 		logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)

cmd/data-usage-cache.go

@@ -428,10 +428,15 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
 	}
 }
 
+type objectIO interface {
+	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+}
+
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
 // If the object is not found or unable to deserialize d is cleared and nil error is returned.
-func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name string) error {
+func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
 	var buf bytes.Buffer
 	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
 	if err != nil {
@@ -450,7 +455,7 @@ func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name strin
 }
 
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
-func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name string) error {
+func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
 	b := d.serialize()
 	size := int64(len(b))
 	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)

cmd/data-usage_test.go

@@ -240,7 +240,6 @@ func TestDataUsageUpdate(t *testing.T) {
 				t.Fatal("got nil result")
 			}
 			if w.flatten {
-				t.Log(e.Children)
 				*e = got.flatten(*e)
 			}
 			if e.Size != int64(w.size) {

cmd/disk-cache-backend.go

@@ -696,10 +696,7 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return err
 	}
-	var metadata = make(map[string]string)
-	for k, v := range opts.UserDefined {
-		metadata[k] = v
-	}
+	var metadata = cloneMSS(opts.UserDefined)
 	var reader = data
 	var actualSize = uint64(size)
 	if globalCacheKMS != nil {
@@ -739,10 +736,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return err
 	}
-	var metadata = make(map[string]string)
-	for k, v := range opts.UserDefined {
-		metadata[k] = v
-	}
+	var metadata = cloneMSS(opts.UserDefined)
 	var reader = data
 	var actualSize = uint64(rlen)
 	// objSize is the actual size of object (with encryption overhead if any)

cmd/disk-cache.go

@@ -85,16 +85,15 @@ type cacheObjects struct {
 }
 
 func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string, rs *HTTPRangeSpec) error {
-	metadata := make(map[string]string)
-	metadata["etag"] = eTag
+	metadata := map[string]string{"etag": eTag}
 	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true)
 }
 
 // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
 func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *diskCache, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo, rs *HTTPRangeSpec) error {
 
-	bkMeta := make(map[string]string)
-	cacheMeta := make(map[string]string)
+	bkMeta := make(map[string]string, len(bkObjectInfo.UserDefined))
+	cacheMeta := make(map[string]string, len(cacheObjInfo.UserDefined))
 	for k, v := range bkObjectInfo.UserDefined {
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
@@ -166,7 +165,7 @@ func (c *cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects
 
 // construct a metadata k-v map
 func getMetadata(objInfo ObjectInfo) map[string]string {
-	metadata := make(map[string]string)
+	metadata := make(map[string]string, len(objInfo.UserDefined)+4)
 	metadata["etag"] = objInfo.ETag
 	metadata["content-type"] = objInfo.ContentType
 	if objInfo.ContentEncoding != "" {
@@ -334,11 +333,12 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
+	userDefined := getMetadata(bkReader.ObjInfo)
 	go func() {
 		putErr := dcache.Put(ctx, bucket, object,
 			io.LimitReader(pipeReader, bkReader.ObjInfo.Size),
 			bkReader.ObjInfo.Size, nil, ObjectOptions{
-				UserDefined: getMetadata(bkReader.ObjInfo),
+				UserDefined: userDefined,
 			}, false)
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader

cmd/encryption-v1.go

@@ -388,12 +388,7 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 		object:            object,
 		customerKeyHeader: h.Get(crypto.SSECKey),
 		copySource:        copySource,
-	}
-
-	w.metadata = map[string]string{}
-	// Copy encryption metadata for internal use.
-	for k, v := range oi.UserDefined {
-		w.metadata[k] = v
+		metadata:          cloneMSS(oi.UserDefined),
 	}
 
 	if w.copySource {
@@ -432,10 +427,7 @@ type DecryptBlocksReader struct {
 }
 
 func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
-	m := make(map[string]string)
-	for k, v := range d.metadata {
-		m[k] = v
-	}
+	m := cloneMSS(d.metadata)
 	// Initialize the first decrypter; new decrypters will be
 	// initialized in Read() operation as needed.
 	var key []byte

cmd/erasure-common.go

@@ -34,7 +34,9 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
-// getLoadBalancedNDisks - fetches load balanced (sufficiently randomized) disk slice with N disks online
+// getLoadBalancedNDisks - fetches load balanced (sufficiently randomized) disk slice
+// with N disks online. If ndisks is zero or negative, then it will returns all disks,
+// same if ndisks is greater than the number of all disks.
 func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAPI) {
 	disks := er.getLoadBalancedDisks()
 	for _, disk := range disks {

cmd/erasure-common_test.go

@@ -32,6 +32,7 @@ func TestErasureParentDirIsObject(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize 'Erasure' object layer.")
 	}
+	defer obj.Shutdown(context.Background())
 
 	// Remove all disks.
 	for _, disk := range fsDisks {

cmd/erasure-healing-common_test.go

@@ -99,6 +99,7 @@ func TestListOnlineDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)
 
 	type tamperKind int
@@ -265,6 +266,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)
 
 	bucket := "bucket"

cmd/erasure-healing.go

@@ -29,16 +29,6 @@ import (
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
-func (er erasureObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
-func (er erasureObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return madmin.HealResultItem{}, NotImplemented{}
-}
-
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.

cmd/erasure-healing_test.go

@@ -39,6 +39,7 @@ func TestHealing(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)

cmd/erasure-list-objects.go

@@ -1,58 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-
-	"github.com/minio/minio/pkg/madmin"
-)
-
-// ListObjectVersions - This is not implemented, look for erasure-zones.ListObjectVersions()
-func (er erasureObjects) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (loi ListObjectVersionsInfo, e error) {
-	return loi, NotImplemented{}
-}
-
-// ListObjectsV2 - This is not implemented/needed anymore, look for erasure-zones.ListObjectsV2()
-func (er erasureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi ListObjectsV2Info, e error) {
-	return loi, NotImplemented{}
-}
-
-// ListObjects - This is not implemented/needed anymore, look for erasure-zones.ListObjects()
-func (er erasureObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, NotImplemented{}
-}
-
-// ListBucketsHeal - This is not implemented/needed anymore, look for erasure-zones.ListBucketHeal()
-func (er erasureObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
-	return nil, NotImplemented{}
-}
-
-// ListObjectsHeal - This is not implemented, look for erasure-zones.ListObjectsHeal()
-func (er erasureObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
-	return ListObjectsInfo{}, NotImplemented{}
-}
-
-// HealObjects - This is not implemented/needed anymore, look for erasure-zones.HealObjects()
-func (er erasureObjects) HealObjects(ctx context.Context, bucket, prefix string, _ madmin.HealOpts, _ HealObjectFn) (e error) {
-	return NotImplemented{}
-}
-
-// Walk - This is not implemented/needed anymore, look for erasure-zones.Walk()
-func (er erasureObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, _ ObjectOptions) error {
-	return NotImplemented{}
-}

cmd/erasure-multipart.go

@@ -280,10 +280,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 
 	fi.DataDir = mustGetUUID()
 	fi.ModTime = UTCNow()
-	fi.Metadata = map[string]string{}
-	for k, v := range opts.UserDefined {
-		fi.Metadata[k] = v
-	}
+	fi.Metadata = cloneMSS(opts.UserDefined)
 
 	uploadID := mustGetUUID()
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
@@ -555,10 +552,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		return result, err
 	}
 
-	result.UserDefined = map[string]string{}
-	for k, v := range fi.Metadata {
-		result.UserDefined[k] = v
-	}
+	result.UserDefined = cloneMSS(fi.Metadata)
 	return result, nil
 }
 
@@ -606,10 +600,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	result.UploadID = uploadID
 	result.MaxParts = maxParts
 	result.PartNumberMarker = partNumberMarker
-	result.UserDefined = map[string]string{}
-	for k, v := range fi.Metadata {
-		result.UserDefined[k] = v
-	}
+	result.UserDefined = cloneMSS(fi.Metadata)
 
 	// For empty number of parts or maxParts as zero, return right here.
 	if len(fi.Parts) == 0 || maxParts == 0 {

cmd/erasure-object_test.go

@@ -42,6 +42,7 @@ func TestRepeatPutObjectPart(t *testing.T) {
 	}
 
 	// cleaning up of temporary test directories
+	defer objLayer.Shutdown(context.Background())
 	defer removeRoots(disks)
 
 	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", BucketOptions{})
@@ -90,6 +91,7 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer xl.Shutdown(context.Background())
 
 	err = xl.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
 	if err != nil {
@@ -200,6 +202,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Cleanup backend directories
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)
@@ -269,6 +272,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Cleanup backend directories.
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)
@@ -331,6 +335,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	}
 
 	// Cleanup backend directories.
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)

cmd/erasure-sets.go

@@ -488,11 +488,6 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 	return storageInfo, errs
 }
 
-func (s *erasureSets) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
-	// Use the zone-level implementation instead.
-	return NotImplemented{API: "CrawlAndGetDataUsage"}
-}
-
 // Shutdown shutsdown all erasure coded sets in parallel
 // returns error upon first error.
 func (s *erasureSets) Shutdown(ctx context.Context) error {
@@ -510,7 +505,14 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 			return err
 		}
 	}
-
+	select {
+	case _, ok := <-s.disksConnectEvent:
+		if ok {
+			close(s.disksConnectEvent)
+		}
+	default:
+		close(s.disksConnectEvent)
+	}
 	return nil
 }
 
@@ -589,11 +591,6 @@ func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string) (bucketI
 	return s.getHashedSet("").GetBucketInfo(ctx, bucket)
 }
 
-// ListObjectsV2 lists all objects in bucket filtered by prefix
-func (s *erasureSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	return result, NotImplemented{}
-}
-
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (s *erasureSets) IsNotificationSupported() bool {
 	return s.getHashedSet("").IsNotificationSupported()
@@ -1038,22 +1035,6 @@ func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, mark
 	return entryChs
 }
 
-// ListObjectVersions - implements listing of objects across disks, each disk is indepenently
-// walked and merged at this layer. Resulting value through the merge process sends
-// the data in lexically sorted order.
-func (s *erasureSets) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionIDMarker, delimiter string, maxKeys int) (loi ListObjectVersionsInfo, err error) {
-	// Shouldn't be called directly, caller Zones already has an implementation
-	return loi, NotImplemented{}
-}
-
-// ListObjects - implements listing of objects across disks, each disk is indepenently
-// walked and merged at this layer. Resulting value through the merge process sends
-// the data in lexically sorted order.
-func (s *erasureSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
-	// Shouldn't be called directly, caller Zones already has an implementation
-	return loi, NotImplemented{}
-}
-
 func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	// In list multipart uploads we are going to treat input prefix as the object,
 	// this means that we are not supporting directory navigation.
@@ -1621,18 +1602,6 @@ func (s *erasureSets) GetObjectTags(ctx context.Context, bucket, object string,
 	return s.getHashedSet(object).GetObjectTags(ctx, bucket, object, opts)
 }
 
-// GetMetrics - no op
-func (s *erasureSets) GetMetrics(ctx context.Context) (*Metrics, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return &Metrics{}, NotImplemented{}
-}
-
-// Health shouldn't be called directly - will panic
-func (s *erasureSets) Health(ctx context.Context, _ HealthOptions) HealthResult {
-	logger.CriticalIf(ctx, NotImplemented{})
-	return HealthResult{}
-}
-
 // maintainMRFList gathers the list of successful partial uploads
 // from all underlying er.sets and puts them in a global map which
 // should not have more than 10000 entries.

cmd/erasure-sets_test.go

@@ -206,6 +206,7 @@ func TestHashedLayer(t *testing.T) {
 		if err != nil {
 			t.Fatal("Unable to initialize 'Erasure' object layer.", err)
 		}
+		defer obj.Shutdown(ctx)
 
 		// Remove all dirs.
 		for _, dir := range fsDirs {

cmd/erasure-zones.go

@@ -40,6 +40,9 @@ type erasureZones struct {
 	GatewayUnsupported
 
 	zones []*erasureSets
+
+	// Shut down async operations
+	shutdown context.CancelFunc
 }
 
 func (z *erasureZones) SingleZone() bool {
@@ -79,7 +82,7 @@ func newErasureZones(ctx context.Context, endpointZones EndpointZones) (ObjectLa
 			return nil, err
 		}
 	}
-
+	ctx, z.shutdown = context.WithCancel(ctx)
 	go intDataUpdateTracker.start(ctx, localDrives...)
 	return z, nil
 }
@@ -218,6 +221,7 @@ func (z *erasureZones) getZoneIdx(ctx context.Context, bucket, object string, op
 }
 
 func (z *erasureZones) Shutdown(ctx context.Context) error {
+	defer z.shutdown()
 	if z.SingleZone() {
 		return z.zones[0].Shutdown(ctx)
 	}
@@ -237,7 +241,6 @@ func (z *erasureZones) Shutdown(ctx context.Context) error {
 		}
 		// let's the rest shutdown
 	}
-
 	return nil
 }
 
@@ -908,7 +911,7 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 		entryChs, endWalkCh := zone.pool.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet, false)
+			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet+1, false)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
@@ -1277,7 +1280,7 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m
 		entryChs, endWalkCh := zone.poolVersions.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet)
+			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet+1)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)

cmd/erasure.go

@@ -81,6 +81,14 @@ func (er erasureObjects) SetDriveCount() int {
 func (er erasureObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
 	closeStorageDisks(er.getDisks())
+	select {
+	case _, ok := <-er.mrfOpCh:
+		if ok {
+			close(er.mrfOpCh)
+		}
+	default:
+		close(er.mrfOpCh)
+	}
 	return nil
 }
 
@@ -245,19 +253,6 @@ func (er erasureObjects) StorageInfo(ctx context.Context, local bool) (StorageIn
 	return getStorageInfo(disks, endpoints)
 }
 
-// GetMetrics - is not implemented and shouldn't be called.
-func (er erasureObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return &Metrics{}, NotImplemented{}
-}
-
-// CrawlAndGetDataUsage collects usage from all buckets.
-// updates are sent as different parts of the underlying
-// structure has been traversed.
-func (er erasureObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
-	return NotImplemented{API: "CrawlAndGetDataUsage"}
-}
-
 // CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
@@ -439,9 +434,3 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 
 	return nil
 }
-
-// Health shouldn't be called directly - will panic
-func (er erasureObjects) Health(ctx context.Context, _ HealthOptions) HealthResult {
-	logger.CriticalIf(ctx, NotImplemented{})
-	return HealthResult{}
-}

cmd/fs-v1-multipart.go

@@ -731,7 +731,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, toObjectErr(err, bucket, object)
 	}
 	// Save additional metadata.
-	if len(fsMeta.Meta) == 0 {
+	if fsMeta.Meta == nil {
 		fsMeta.Meta = make(map[string]string)
 	}
 	fsMeta.Meta["etag"] = s3MD5

cmd/fs-v1.go

@@ -635,10 +635,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 			fsMeta = fs.defaultFsJSON(srcObject)
 		}
 
-		fsMeta.Meta = map[string]string{}
-		for k, v := range srcInfo.UserDefined {
-			fsMeta.Meta[k] = v
-		}
+		fsMeta.Meta = cloneMSS(srcInfo.UserDefined)
 		fsMeta.Meta["etag"] = srcInfo.ETag
 		if _, err = fsMeta.WriteTo(wlk); err != nil {
 			return oi, toObjectErr(err, srcBucket, srcObject)
@@ -1124,10 +1121,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	data := r.Reader
 
 	// No metadata is set, allocate a new one.
-	meta := make(map[string]string)
-	for k, v := range opts.UserDefined {
-		meta[k] = v
-	}
+	meta := cloneMSS(opts.UserDefined)
 	var err error
 
 	// Validate if bucket name is valid and exists.

cmd/gateway-common.go

@@ -57,7 +57,7 @@ var (
 
 // FromMinioClientMetadata converts minio metadata to map[string]string
 func FromMinioClientMetadata(metadata map[string][]string) map[string]string {
-	mm := map[string]string{}
+	mm := make(map[string]string, len(metadata))
 	for k, v := range metadata {
 		mm[http.CanonicalHeaderKey(k)] = v[0]
 	}
@@ -227,7 +227,7 @@ func ToMinioClientObjectInfoMetadata(metadata map[string]string) map[string][]st
 
 // ToMinioClientMetadata converts metadata to map[string]string
 func ToMinioClientMetadata(metadata map[string]string) map[string]string {
-	mm := make(map[string]string)
+	mm := make(map[string]string, len(metadata))
 	for k, v := range metadata {
 		mm[http.CanonicalHeaderKey(k)] = v
 	}

cmd/gateway/azure/gateway-azure.go

@@ -882,7 +882,7 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	data := r.Reader
 
 	if data.Size() > azureBlockSize/2 {
-		if len(opts.UserDefined) == 0 {
+		if opts.UserDefined == nil {
 			opts.UserDefined = map[string]string{}
 		}
 

cmd/generic-handlers.go

@@ -24,7 +24,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/http/stats"

cmd/global-heal.go

@@ -68,6 +68,9 @@ func newBgHealSequence() *healSequence {
 }
 
 func getLocalBackgroundHealStatus() (madmin.BgHealState, bool) {
+	if globalBackgroundHealState == nil {
+		return madmin.BgHealState{}, false
+	}
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if !ok {
 		return madmin.BgHealState{}, false

cmd/globals.go

@@ -26,7 +26,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/policy/opa"

cmd/handler-utils.go

@@ -404,6 +404,9 @@ func getResource(path string, host string, domains []string) (string, error) {
 		}
 	}
 	for _, domain := range domains {
+		if host == minioReservedBucket+"."+domain {
+			continue
+		}
 		if !strings.HasSuffix(host, "."+domain) {
 			continue
 		}

cmd/logger/logger.go

@@ -331,8 +331,9 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
 		API = req.API
 	}
 
-	tags := make(map[string]string)
-	for _, entry := range req.GetTags() {
+	kv := req.GetTags()
+	tags := make(map[string]string, len(kv))
+	for _, entry := range kv {
 		tags[entry.Key] = entry.Val
 	}
 

cmd/logger/message/audit/entry.go

@@ -53,16 +53,18 @@ type Entry struct {
 
 // ToEntry - constructs an audit entry object.
 func ToEntry(w http.ResponseWriter, r *http.Request, reqClaims map[string]interface{}, deploymentID string) Entry {
-	reqQuery := make(map[string]string)
-	for k, v := range r.URL.Query() {
+	q := r.URL.Query()
+	reqQuery := make(map[string]string, len(q))
+	for k, v := range q {
 		reqQuery[k] = strings.Join(v, ",")
 	}
-	reqHeader := make(map[string]string)
+	reqHeader := make(map[string]string, len(r.Header))
 	for k, v := range r.Header {
 		reqHeader[k] = strings.Join(v, ",")
 	}
-	respHeader := make(map[string]string)
-	for k, v := range w.Header() {
+	wh := w.Header()
+	respHeader := make(map[string]string, len(wh))
+	for k, v := range wh {
 		respHeader[k] = strings.Join(v, ",")
 	}
 	respHeader[xhttp.ETag] = strings.Trim(respHeader[xhttp.ETag], `"`)
@@ -71,7 +73,7 @@ func ToEntry(w http.ResponseWriter, r *http.Request, reqClaims map[string]interf
 		Version:      Version,
 		DeploymentID: deploymentID,
 		RemoteHost:   handlers.GetSourceIP(r),
-		RequestID:    w.Header().Get(xhttp.AmzRequestID),
+		RequestID:    wh.Get(xhttp.AmzRequestID),
 		UserAgent:    r.UserAgent(),
 		Time:         time.Now().UTC().Format(time.RFC3339Nano),
 		ReqQuery:     reqQuery,

cmd/object-api-input-checks.go

@@ -60,7 +60,7 @@ func checkBucketAndObjectNames(ctx context.Context, bucket, object string) error
 }
 
 // Checks for all ListObjects arguments validity.
-func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj ObjectLayer) error {
+func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj getBucketInfoI) error {
 	// Verify if bucket exists before validating object name.
 	// This is done on purpose since the order of errors is
 	// important here bucket does not exist error should
@@ -173,7 +173,7 @@ func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer
 }
 
 // Checks for PutObject arguments validity, also validates if bucket exists.
-func checkPutObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer, size int64) error {
+func checkPutObjectArgs(ctx context.Context, bucket, object string, obj getBucketInfoI, size int64) error {
 	// Verify if bucket exists before validating object name.
 	// This is done on purpose since the order of errors is
 	// important here bucket does not exist error should
@@ -197,8 +197,12 @@ func checkPutObjectArgs(ctx context.Context, bucket, object string, obj ObjectLa
 	return nil
 }
 
+type getBucketInfoI interface {
+	GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
+}
+
 // Checks whether bucket exists and returns appropriate error if not.
-func checkBucketExist(ctx context.Context, bucket string, obj ObjectLayer) error {
+func checkBucketExist(ctx context.Context, bucket string, obj getBucketInfoI) error {
 	_, err := obj.GetBucketInfo(ctx, bucket)
 	if err != nil {
 		return err

cmd/object-api-options.go

@@ -168,7 +168,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 	etag := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceETag))
 	if etag != "" {
 		if metadata == nil {
-			metadata = make(map[string]string)
+			metadata = make(map[string]string, 1)
 		}
 		metadata["etag"] = etag
 	}

cmd/object-api-utils.go

@@ -38,7 +38,7 @@ import (
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio/cmd/config/compress"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -270,7 +270,7 @@ func removeStandardStorageClass(metadata map[string]string) map[string]string {
 // cleanMetadataKeys takes keyNames to be filtered
 // and returns a new map with all the entries with keyNames removed.
 func cleanMetadataKeys(metadata map[string]string, keyNames ...string) map[string]string {
-	var newMeta = make(map[string]string)
+	var newMeta = make(map[string]string, len(metadata))
 	for k, v := range metadata {
 		if contains(keyNames, k) {
 			continue

cmd/object-handlers.go

@@ -37,7 +37,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -764,7 +764,7 @@ func getRemoteInstanceClientLongTimeout(r *http.Request, host string) (*miniogo.
 // when federation is enabled, ie when globalDNSConfig is non 'nil'.
 //
 // This function is similar to isRemoteCallRequired but specifically for COPY object API
-// if destination and source are same we do not need to check for destnation bucket
+// if destination and source are same we do not need to check for destination bucket
 // to exist locally.
 func isRemoteCopyRequired(ctx context.Context, srcBucket, dstBucket string, objAPI ObjectLayer) bool {
 	if srcBucket == dstBucket {
@@ -2713,7 +2713,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 
 	if globalDNSConfig != nil {
 		_, err := globalDNSConfig.Get(bucket)
-		if err != nil {
+		if err != nil && err != dns.ErrNotImplemented {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -2739,22 +2739,35 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	if apiErr == ErrNone {
-		// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
-		objInfo, err := deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r, opts)
-		if err != nil {
-			switch err.(type) {
-			case BucketNotFound:
-				// When bucket doesn't exist specially handle it.
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Ignore delete object errors while replying to client, since we are suppposed to reply only 204.
-		}
-		setPutObjHeaders(w, objInfo, true)
+	if apiErr == ErrNoSuchKey {
+		writeSuccessNoContent(w)
+		return
 	}
 
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
+	objInfo, err := deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, r, opts)
+	if err != nil {
+		switch err.(type) {
+		case BucketNotFound:
+			// When bucket doesn't exist specially handle it.
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		// Ignore delete object errors while replying to client, since we are suppposed to reply only 204.
+	}
+	setPutObjHeaders(w, objInfo, true)
+
 	writeSuccessNoContent(w)
+
+	sendEvent(eventArgs{
+		EventName:    event.ObjectRemovedDelete,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
 }
 
 // PutObjectLegalHoldHandler - set legal hold configuration to object,

cmd/signature-v2.go

@@ -321,7 +321,7 @@ func compareSignatureV2(sig1, sig2 string) bool {
 // Return canonical headers.
 func canonicalizedAmzHeadersV2(headers http.Header) string {
 	var keys []string
-	keyval := make(map[string]string)
+	keyval := make(map[string]string, len(headers))
 	for key := range headers {
 		lkey := strings.ToLower(key)
 		if !strings.HasPrefix(lkey, "x-amz-") {

cmd/test-utils_test.go

@@ -481,6 +481,7 @@ func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
 func (testServer TestServer) Stop() {
 	testServer.cancel()
 	testServer.Server.Close()
+	testServer.Obj.Shutdown(context.Background())
 	os.RemoveAll(testServer.Root)
 	for _, ep := range testServer.Disks {
 		for _, disk := range ep.Endpoints {
@@ -1838,6 +1839,7 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
 	}
+	defer objLayer.Shutdown(ctx)
 
 	bucketErasure, erAPIRouter, err := initAPIHandlerTest(objLayer, endpoints)
 	if err != nil {
@@ -1893,6 +1895,7 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
 	}
+	defer objLayer.Shutdown(context.Background())
 
 	initAllSubsystems(ctx, objLayer)
 
@@ -1911,6 +1914,7 @@ func ExecObjectLayerTestWithDirs(t TestErrHandler, objTest objTestTypeWithDirs)
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
 	}
+	defer objLayer.Shutdown(ctx)
 
 	// initialize the server and obtain the credentials and root.
 	// credentials are necessary to sign the HTTP request.
@@ -1933,6 +1937,7 @@ func ExecObjectLayerDiskAlteredTest(t *testing.T, objTest objTestDiskNotFoundTyp
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
 	}
+	defer objLayer.Shutdown(ctx)
 
 	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatal("Failed to create config directory", err)

cmd/update.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2015, 2016, 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2015-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -163,8 +163,8 @@ func IsKubernetes() bool {
 	if env.Get("MINIO_CI_CD", "") == "" {
 		// Kubernetes env used to validate if we are
 		// indeed running inside a kubernetes pod
-		// is KUBERNETES_SERVICE_HOST but in future
-		// we might need to enhance this.
+		// is KUBERNETES_SERVICE_HOST
+		// https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet_pods.go#L541
 		return env.Get("KUBERNETES_SERVICE_HOST", "") != ""
 	}
 	return false
@@ -274,6 +274,15 @@ func getUserAgent(mode string) string {
 		if helmChartVersion != "" {
 			uaAppend(" MinIO/helm-", helmChartVersion)
 		}
+		// In Kubernetes environment, try to fetch the Operator, VSPHERE plugin version
+		opVersion := env.Get("MINIO_OPERATOR_VERSION", "")
+		if opVersion != "" {
+			uaAppend(" MinIO/operator-", opVersion)
+		}
+		vsphereVersion := env.Get("MINIO_VSPHERE_PLUGIN_VERSION", "")
+		if vsphereVersion != "" {
+			uaAppend(" MinIO/vsphere-plugin-", vsphereVersion)
+		}
 	}
 
 	pcfTileVersion := env.Get("MINIO_PCF_TILE_VERSION", "")

cmd/utils.go

@@ -112,6 +112,16 @@ func getWriteQuorum(drive int) int {
 	return quorum
 }
 
+// cloneMSS will clone a map[string]string.
+// If input is nil an empty map is returned, not nil.
+func cloneMSS(v map[string]string) map[string]string {
+	r := make(map[string]string, len(v))
+	for k, v := range v {
+		r[k] = v
+	}
+	return r
+}
+
 // URI scheme constants.
 const (
 	httpScheme  = "http"

cmd/web-handlers.go

@@ -40,7 +40,7 @@ import (
 	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio/browser"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -179,7 +179,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 
 	if globalDNSConfig != nil {
 		if _, err := globalDNSConfig.Get(args.BucketName); err != nil {
-			if err == dns.ErrNoEntriesFound {
+			if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, args.BucketName, opts); err != nil {
 					return toJSONError(ctx, err)
@@ -281,7 +281,7 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
 
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(args.BucketName); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
 			return toJSONError(ctx, err)
 		}
 	}

cmd/xl-storage-format-v2.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
@@ -267,7 +268,7 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
 			PartSizes:          make([]int64, len(fi.Parts)),
 			PartActualSizes:    make([]int64, len(fi.Parts)),
 			MetaSys:            make(map[string][]byte),
-			MetaUser:           make(map[string]string),
+			MetaUser:           make(map[string]string, len(fi.Metadata)),
 		},
 	}
 
@@ -524,6 +525,20 @@ func (z xlMetaV2) TotalSize() int64 {
 	return total
 }
 
+type versionsSorter []FileInfo
+
+func (v versionsSorter) Len() int      { return len(v) }
+func (v versionsSorter) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
+func (v versionsSorter) Less(i, j int) bool {
+	if v[i].IsLatest {
+		return true
+	}
+	if v[j].IsLatest {
+		return false
+	}
+	return v[i].ModTime.After(v[j].ModTime)
+}
+
 // ListVersions lists current versions, and current deleted
 // versions returns error for unexpected entries.
 func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTime time.Time, err error) {
@@ -562,6 +577,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
 		break
 	}
 
+	sort.Sort(versionsSorter(versions))
 	return versions, latestModTime, nil
 }
 

cmd/xl-storage.go

@@ -380,11 +380,18 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
 		}
 
 		var totalSize int64
-		for _, version := range fivs.Versions {
+		var numVersions = len(fivs.Versions)
+
+		for i, version := range fivs.Versions {
+			var successorModTime time.Time
+			if i > 0 {
+				successorModTime = fivs.Versions[i-1].ModTime
+			}
 			oi := version.ToObjectInfo(item.bucket, item.objectPath())
 			size := item.applyActions(ctx, objAPI, actionMeta{
-				numVersions: len(fivs.Versions),
-				oi:          oi,
+				numVersions:      numVersions,
+				successorModTime: successorModTime,
+				oi:               oi,
 			})
 			if !version.Deleted {
 				totalSize += size

docs/deployment/kernel-tuning/sysctl.sh

@@ -33,8 +33,8 @@ net.core.wmem_default = 4194304
 net.core.optmem_max = 4194304
 
 # increase memory thresholds to prevent packet dropping:
-net.ipv4.tcp_rmem = "4096 87380 4194304"
-net.ipv4.tcp_wmem = "4096 65536 4194304"
+net.ipv4.tcp_rmem = 4096 87380 4194304
+net.ipv4.tcp_wmem = 4096 65536 4194304
 
 # enable low latency mode for TCP:
 net.ipv4.tcp_low_latency = 1

docs/orchestration/docker-compose/README.md

@@ -11,7 +11,7 @@ With Compose, you use a Compose file to configure MinIO services. Then, using a
 
 ## 2. Run Distributed MinIO on Docker Compose
 
-To deploy Distributed MinIO on Docker Compose, please download [docker-compose.yaml](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/docker-compose.yaml?raw=true) to your current working directory. Note that Docker Compose pulls the MinIO Docker image, so there is no need to explicitly download MinIO binary. Then run one of the below commands
+To deploy Distributed MinIO on Docker Compose, please download [docker-compose.yaml](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/docker-compose.yaml?raw=true) and [nginx.conf](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/nginx.conf?raw=true) to your current working directory. Note that Docker Compose pulls the MinIO Docker image, so there is no need to explicitly download MinIO binary. Then run one of the below commands
 
 ### GNU/Linux and macOS
 
@@ -27,7 +27,7 @@ docker-compose.exe pull
 docker-compose.exe up
 ```
 
-Each instance is now accessible on the host at ports 9001 through 9004, proceed to access the Web browser at http://127.0.0.1:9001/
+Distributed instances are now accessible on the host at ports 9000, proceed to access the Web browser at http://127.0.0.1:9000/. Here 4 MinIO server instances are reverse proxied through Nginx load balancing.
 
 ### Notes
 
@@ -36,12 +36,10 @@ Each instance is now accessible on the host at ports 9001 through 9004, proceed
 * There are 4 minio distributed instances created by default. You can add more MinIO services (up to total 16) to your MinIO Compose deployment. To add a service
   * Replicate a service definition and change the name of the new service appropriately.
   * Update the command section in each service.
-  * Update the port number to exposed for the new service. Also, make sure the port assigned for the new service is not already being used on the host.
+  * Add a new MinIO server instance to the upstream directive in the Nginx configuration file.
 
   Read more about distributed MinIO [here](https://docs.min.io/docs/distributed-minio-quickstart-guide).
 
-* MinIO services in the Docker compose file expose ports 9001 to 9004. This allows multiple services to run on a host.
-
 ### Explore Further
 - [Overview of Docker Compose](https://docs.docker.com/compose/overview/)
 - [MinIO Docker Quickstart Guide](https://docs.min.io/docs/minio-docker-quickstart-guide)

docs/orchestration/docker-compose/docker-compose.yaml

@@ -1,16 +1,16 @@
 version: '3.7'
 
-# starts 4 docker containers running minio server instances. Each
-# minio server's web interface will be accessible on the host at port
-# 9001 through 9004.
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
-    ports:
-      - "9001:9000"
+    expose:
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -22,12 +22,12 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
-    ports:
-      - "9002:9000"
+    expose:
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -39,12 +39,12 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
-    ports:
-      - "9003:9000"
+    expose:
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -56,12 +56,12 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2
-    ports:
-      - "9004:9000"
+    expose:
+      - "9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123
@@ -72,6 +72,18 @@ services:
       timeout: 20s
       retries: 3
 
+  nginx:
+    image: nginx:1.19.2-alpine
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.
 volumes:

docs/orchestration/docker-compose/nginx.conf

@@ -0,0 +1,68 @@
+
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+         # To allow special characters in headers
+         ignore_invalid_headers off;
+         # Allow any size file to be uploaded.
+         # Set to a value such as 1000m; to restrict file size to a specific value
+         client_max_body_size 0;
+         # To disable buffering
+         proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-09-05T07-14-49Z
+    image: minio/minio:RELEASE.2020-09-08T23-05-18Z
     hostname: minio4
     volumes:
       - minio4-data:/export

go.mod

@@ -1,6 +1,6 @@
 module github.com/minio/minio
 
-go 1.13
+go 1.14
 
 require (
 	cloud.google.com/go v0.39.0

main.go

@@ -1,7 +1,7 @@
-// +build go1.13
+// +build go1.14
 
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

pkg/bucket/lifecycle/lifecycle.go

@@ -176,13 +176,14 @@ func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
 // ObjectOpts provides information to deduce the lifecycle actions
 // which can be triggered on the resultant object.
 type ObjectOpts struct {
-	Name         string
-	UserTags     string
-	ModTime      time.Time
-	VersionID    string
-	IsLatest     bool
-	DeleteMarker bool
-	NumVersions  int
+	Name             string
+	UserTags         string
+	ModTime          time.Time
+	VersionID        string
+	IsLatest         bool
+	DeleteMarker     bool
+	NumVersions      int
+	SuccessorModTime time.Time
 }
 
 // ComputeAction returns the action to perform by evaluating all lifecycle rules
@@ -203,9 +204,10 @@ func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
 		}
 
 		if !rule.NoncurrentVersionExpiration.IsDaysNull() {
-			if obj.VersionID != "" && !obj.IsLatest {
-				// Non current versions should be deleted.
-				if time.Now().After(expectedExpiryTime(obj.ModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
+			if obj.VersionID != "" && !obj.IsLatest && !obj.SuccessorModTime.IsZero() {
+				// Non current versions should be deleted if their age exceeds non current days configuration
+				// https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions
+				if time.Now().After(expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
 					return DeleteVersionAction
 				}
 			}

pkg/env/env.go

@@ -45,7 +45,7 @@ func SetEnvOn() {
 
 // IsSet returns if the given env key is set.
 func IsSet(key string) bool {
-	_, ok := LookupEnv(key)
+	_, _, _, ok := LookupEnv(key)
 	return ok
 }
 
@@ -60,7 +60,7 @@ func Get(key, defaultValue string) string {
 	if ok {
 		return defaultValue
 	}
-	if v, ok := LookupEnv(key); ok {
+	if v, _, _, ok := LookupEnv(key); ok {
 		return v
 	}
 	return defaultValue

pkg/env/web_env.go

@@ -72,10 +72,10 @@ func fetchHTTPConstituentParts(u *url.URL) (username string, password string, en
 	return username, password, envURL, nil
 }
 
-func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
+func getEnvValueFromHTTP(urlStr, envKey string) (string, string, string, error) {
 	u, err := url.Parse(urlStr)
 	if err != nil {
-		return "", err
+		return "", "", "", err
 	}
 
 	switch u.Scheme {
@@ -84,12 +84,12 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
 	case webEnvSchemeSecure:
 		u.Scheme = "https"
 	default:
-		return "", errors.New("invalid arguments")
+		return "", "", "", errors.New("invalid arguments")
 	}
 
 	username, password, envURL, err := fetchHTTPConstituentParts(u)
 	if err != nil {
-		return "", err
+		return "", "", "", err
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
@@ -97,7 +97,7 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, envURL+"?key="+envKey, nil)
 	if err != nil {
-		return "", err
+		return "", "", "", err
 	}
 
 	claims := &jwt.StandardClaims{
@@ -109,7 +109,7 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
 	token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
 	ss, err := token.SignedString([]byte(password))
 	if err != nil {
-		return "", err
+		return "", "", "", err
 	}
 
 	req.Header.Set("Authorization", "Bearer "+ss)
@@ -136,15 +136,15 @@ func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
 
 	resp, err := clnt.Do(req)
 	if err != nil {
-		return "", err
+		return "", "", "", err
 	}
 
 	envValueBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return "", err
+		return "", "", "", err
 	}
 
-	return string(envValueBytes), nil
+	return string(envValueBytes), username, password, nil
 }
 
 // Environ returns a copy of strings representing the
@@ -161,23 +161,27 @@ func Environ() []string {
 //
 // Additionally if the input is env://username:password@remote:port/
 // to fetch ENV values for the env value from a remote server.
-func LookupEnv(key string) (string, bool) {
+// In this case, it also returns the credentials username and password
+func LookupEnv(key string) (string, string, string, bool) {
 	v, ok := os.LookupEnv(key)
 	if ok && strings.HasPrefix(v, webEnvScheme) {
 		// If env value starts with `env*://`
 		// continue to parse and fetch from remote
 		var err error
-		v, err = getEnvValueFromHTTP(strings.TrimSpace(v), key)
+		v, user, pwd, err := getEnvValueFromHTTP(strings.TrimSpace(v), key)
 		if err != nil {
-			// fallback to cached value if-any.
-			return os.LookupEnv("_" + key)
+			env, eok := os.LookupEnv("_" + key)
+			if eok {
+				// fallback to cached value if-any.
+				return env, user, pwd, eok
+			}
 		}
 		// Set the ENV value to _env value,
 		// this value is a fallback in-case of
 		// server restarts when webhook server
 		// is down.
 		os.Setenv("_"+key, v)
-		return v, true
+		return v, user, pwd, true
 	}
-	return v, ok
+	return v, "", "", ok
 }

pkg/env/web_env_test.go

@@ -67,7 +67,7 @@ func TestWebEnv(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	v, err := getEnvValueFromHTTP(
+	v, user, pwd, err := getEnvValueFromHTTP(
 		fmt.Sprintf("env://minio:minio123@%s/webhook/v1/getenv/default/minio",
 			u.Host),
 		"MINIO_ARGS")
@@ -78,4 +78,12 @@ func TestWebEnv(t *testing.T) {
 	if v != "http://127.0.0.{1..4}:9000/data{1...4}" {
 		t.Fatalf("Unexpected value %s", v)
 	}
+
+	if user != "minio" {
+		t.Fatalf("Unexpected value %s", v)
+	}
+
+	if pwd != "minio123" {
+		t.Fatalf("Unexpected value %s", v)
+	}
 }