fix: data usage crawler env handling, usage-cache.bin location (#9163)

canonicalize the ENVs such that we can bring these ENVs 
as part of the config values, as a subsequent change.

- fix location of per bucket usage to `.minio.sys/buckets/<bucket_name>/usage-cache.bin`
- fix location of the overall usage in `json` at `.minio.sys/buckets/.usage.json`
  (avoid conflicts with a bucket named `usage.json` )
- fix location of the overall usage in `msgp` at `.minio.sys/buckets/.usage.bin`
  (avoid conflicts with a bucket named `usage.bin`

Dockerfile.arm.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm32v7/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.arm64.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm64v8/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Makefile

@@ -22,7 +22,7 @@ getdeps:
 ifeq ($(GOARCH),s390x)
 	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
 else
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2019.2.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2020.1.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
 endif
 	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
 

browser/staticcheck.conf

@@ -0,0 +1 @@
+checks = ["all", "-ST1005", "-ST1000", "-SA4000", "-SA9004", "-SA1019", "-SA1008", "-U1000", "-ST1003", "-ST1018"]

cmd/admin-handlers-users.go

@@ -378,6 +378,120 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// AddServiceAccount - PUT /minio/admin/v2/add-service-account?parentUser=<parent_user_accesskey>
+func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddServiceAccount")
+
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	var createReq madmin.AddServiceAccountReq
+	if err = json.Unmarshal(configBytes, &createReq); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	if createReq.Parent == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+
+	// Disallow creating service accounts for the root user
+	if createReq.Parent == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddServiceAccountInvalidArgument), r.URL)
+		return
+	}
+
+	creds, err := globalIAMSys.NewServiceAccount(ctx, createReq.Parent, createReq.Policy)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(creds.AccessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	var createResp = madmin.AddServiceAccountResp{
+		Credentials: creds,
+	}
+
+	data, err := json.Marshal(createResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// GetServiceAccount - GET /minio/admin/v2/get-service-account?accessKey=<access_key>
+func (a adminAPIHandlers) GetServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetServiceAccount")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	creds, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(creds)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
 // InfoCannedPolicy - GET /minio/admin/v2/info-canned-policy?name={policyName}
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")

cmd/admin-handlers.go

@@ -141,9 +141,9 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		if runtime.GOOS == "windows" {
-			u.Path = path.Dir(u.Path) + "minio.exe"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio.exe"
 		} else {
-			u.Path = path.Dir(u.Path) + "minio"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio"
 		}
 
 		updateURL = u.String()
@@ -982,13 +982,24 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{}
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
 	for _, state := range bgHealStates {
 		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
 			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
 		}
-
 	}
 
 	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
@@ -1398,10 +1409,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		} else if ldapConn == nil {
 			ldap.Status = "Not Configured"
 		} else {
+			// Close ldap connection to avoid leaks.
+			ldapConn.Close()
 			ldap.Status = "online"
 		}
-		// Close ldap connection to avoid leaks.
-		defer ldapConn.Close()
 	}
 
 	log, audit := fetchLoggerInfo(cfg)

cmd/admin-heal-ops.go

@@ -106,12 +106,12 @@ func initHealState() *allHealState {
 		healSeqMap: make(map[string]*healSequence),
 	}
 
-	go healState.periodicHealSeqsClean()
+	go healState.periodicHealSeqsClean(GlobalContext)
 
 	return healState
 }
 
-func (ahs *allHealState) periodicHealSeqsClean() {
+func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
 	ticker := time.NewTicker(time.Minute * 5)
@@ -127,7 +127,7 @@ func (ahs *allHealState) periodicHealSeqsClean() {
 				}
 			}
 			ahs.Unlock()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			// server could be restarting - need
 			// to exit immediately
 			return
@@ -369,7 +369,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	return &healSequence{
 		bucket:         bucket,
@@ -575,8 +575,6 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	h.lastHealActivity = UTCNow()
-
 	bucketsOnly := true // heal buckets only, not objects.
 	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)
@@ -605,7 +603,7 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			h.lastHealActivity = UTCNow()
 		case <-h.traverseAndHealDoneCh:
 			return nil
-		case <-GlobalServiceDoneCh:
+		case <-h.ctx.Done():
 			return nil
 		}
 	}
@@ -632,11 +630,6 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 		return err
 	}
 
-	// Start healing the background ops prefix.
-	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
-
 	// Heal buckets and objects
 	return h.healBuckets(bucketsOnly)
 }
@@ -673,7 +666,7 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, func(bucket string, object string) error {
+		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket string, object string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -767,7 +760,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.healObject); err != nil {
+	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.settings, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil

cmd/admin-router.go

@@ -110,6 +110,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
+		// Service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/get-service-account").HandlerFunc(httpTraceHdrs(adminAPI.GetServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+
 		// Info policy IAM
 		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 

cmd/api-errors.go

@@ -122,7 +122,8 @@ const (
 	ErrMissingCredTag
 	ErrCredMalformed
 	ErrInvalidRegion
-	ErrInvalidService
+	ErrInvalidServiceS3
+	ErrInvalidServiceSTS
 	ErrInvalidRequestVersion
 	ErrMissingSignTag
 	ErrMissingSignHeadersTag
@@ -332,6 +333,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrAddServiceAccountInvalidArgument
 	ErrPostPolicyConditionInvalidFormat
 )
 
@@ -652,9 +654,14 @@ var errorCodes = errorCodeMap{
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
 	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
 	// Need changes to make sure variable messages can be constructed.
-	ErrInvalidService: {
-		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+	ErrInvalidServiceS3: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidServiceSTS: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential parameter; incorrect service. This endpoint belongs to \"sts\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
@@ -1573,6 +1580,12 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrAddServiceAccountInvalidArgument: {
+		Code:           "XMinioInvalidArgument",
+		Description:    "New service accounts for admin access key is not allowed",
+		HTTPStatusCode: http.StatusConflict,
+	},
+
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
 		Description:    "Invalid according to Policy: Policy Condition failed",
@@ -1641,7 +1654,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case crypto.ErrKMSAuthLogin:
 		apiErr = ErrKMSAuthFailure
-	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
+	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
 		apiErr = ErrSlowDown
@@ -1769,6 +1782,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrOverlappingFilterNotification
 	case *event.ErrUnsupportedConfiguration:
 		apiErr = ErrUnsupportedNotification
+	case OperationTimedOut:
+		apiErr = ErrOperationTimedOut
 	case BackendDown:
 		apiErr = ErrBackendDown
 	case ObjectNameTooLong:

cmd/api-headers.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -81,6 +82,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
+	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
+		w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts)))
+	}
+
 	if objInfo.ContentType != "" {
 		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}

cmd/background-heal-ops.go

@@ -186,5 +186,5 @@ func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOp
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}
-	return objectAPI.HealObject(ctx, bucket, object, opts.DryRun, opts.Remove, opts.ScanMode)
+	return objectAPI.HealObject(ctx, bucket, object, opts)
 }

cmd/common-main.go

@@ -22,6 +22,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"net"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -226,10 +227,22 @@ func handleCommonEnvVars() {
 		globalConfigEncrypted = true
 	}
 
+	if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
+		oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate the old credentials inherited from the shell environment")
+		}
+		globalOldCred = oldCred
+		os.Unsetenv(config.EnvAccessKeyOld)
+		os.Unsetenv(config.EnvSecretKeyOld)
+	}
+
 	globalWORMEnabled, err = config.LookupWorm()
 	if err != nil {
 		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
 	}
+
 }
 
 func logStartupMessage(msg string) {

cmd/config-current.go

@@ -26,7 +26,6 @@ import (
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
 	"github.com/minio/minio/cmd/config/etcd"
-	xetcd "github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
@@ -304,13 +303,13 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	etcdCfg, err := xetcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
+	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 	}
 
 	if etcdCfg.Enabled {
-		globalEtcdClient, err = xetcd.New(etcdCfg)
+		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 		}
@@ -428,6 +427,10 @@ func lookupConfigs(s config.Config) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
+	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalServiceDoneCh, NewCustomHTTPTransport())
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
+	}
 }
 
 // Help - return sub-system level help

cmd/config-encrypted.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 	"unicode/utf8"
 
@@ -29,7 +28,6 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -88,11 +86,6 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	doneCh = make(chan struct{})
 	defer close(doneCh)
 
@@ -106,7 +99,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		select {
 		case <-retryTimerCh:
 			// Migrate IAM configuration
-			if err = migrateConfigPrefixToEncrypted(objAPI, activeCredOld, encrypted); err != nil {
+			if err = migrateConfigPrefixToEncrypted(objAPI, globalOldCred, encrypted); err != nil {
 				if err == errDiskNotFound ||
 					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
 					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
@@ -164,21 +157,6 @@ func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
 	return data, err
 }
 
-func getOldCreds() (activeCredOld auth.Credentials, err error) {
-	accessKeyOld := env.Get(config.EnvAccessKeyOld, "")
-	secretKeyOld := env.Get(config.EnvSecretKeyOld, "")
-	if accessKeyOld != "" && secretKeyOld != "" {
-		activeCredOld, err = auth.CreateCredentials(accessKeyOld, secretKeyOld)
-		if err != nil {
-			return activeCredOld, err
-		}
-		// Once we have obtained the rotating creds
-		os.Unsetenv(config.EnvAccessKeyOld)
-		os.Unsetenv(config.EnvSecretKeyOld)
-	}
-	return activeCredOld, nil
-}
-
 func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	defer cancel()
@@ -206,20 +184,15 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	if encrypted {
 		// No key rotation requested, and backend is
 		// already encrypted. We proceed without migration.
-		if !activeCredOld.IsValid() {
+		if !globalOldCred.IsValid() {
 			return nil
 		}
 
 		// No real reason to rotate if old and new creds are same.
-		if activeCredOld.Equal(globalActiveCred) {
+		if globalOldCred.Equal(globalActiveCred) {
 			return nil
 		}
 
@@ -254,8 +227,8 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 
 		var data []byte
 		// Is rotating of creds requested?
-		if activeCredOld.IsValid() {
-			data, err = decryptData(cdata, activeCredOld, globalActiveCred)
+		if globalOldCred.IsValid() {
+			data, err = decryptData(cdata, globalOldCred, globalActiveCred)
 			if err != nil {
 				if err == madmin.ErrMaliciousData {
 					return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
@@ -285,7 +258,7 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 	}
 
-	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
+	if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
 

cmd/config/errors-utils.go

@@ -46,7 +46,7 @@ func (u Err) Clone() Err {
 	}
 }
 
-// Return the error message
+// Error returns the error message
 func (u Err) Error() string {
 	if u.detail == "" {
 		if u.msg != "" {

cmd/config/etcd/dns/etcd_dns.go

@@ -60,11 +60,7 @@ func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
 			if record.Key == "" {
 				continue
 			}
-			if _, ok := srvRecords[record.Key]; ok {
-				srvRecords[record.Key] = append(srvRecords[record.Key], record)
-			} else {
-				srvRecords[record.Key] = []SrvRecord{record}
-			}
+			srvRecords[record.Key] = append(srvRecords[record.Key], record)
 		}
 	}
 	return srvRecords, nil

cmd/data-usage-cache.go

@@ -0,0 +1,517 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/tinylib/msgp/msgp"
+)
+
+const dataUsageHashLen = 8
+
+//go:generate msgp -file $GOFILE -unexported
+
+// dataUsageHash is the hash type used.
+type dataUsageHash uint64
+
+// sizeHistogram is a size histogram.
+type sizeHistogram [dataUsageBucketLen]uint64
+
+//msgp:tuple dataUsageEntry
+type dataUsageEntry struct {
+	// These fields do no include any children.
+	Size     int64
+	Objects  uint64
+	ObjSizes sizeHistogram
+
+	Children dataUsageHashMap
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageEntryInfo struct {
+	Name   string
+	Parent string
+	Entry  dataUsageEntry
+}
+
+type dataUsageCacheInfo struct {
+	// Name of the bucket. Also root element.
+	Name       string
+	LastUpdate time.Time
+	NextCycle  uint8
+}
+
+// merge other data usage entry into this, excluding children.
+func (e *dataUsageEntry) merge(other dataUsageEntry) {
+	e.Objects += other.Objects
+	e.Size += other.Size
+	for i, v := range other.ObjSizes[:] {
+		e.ObjSizes[i] += v
+	}
+}
+
+// mod returns true if the hash mod cycles == cycle.
+func (h dataUsageHash) mod(cycle uint8, cycles uint8) bool {
+	return uint8(h)%cycles == cycle%cycles
+}
+
+// addChildString will add a child based on its name.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChildString(name string) {
+	e.addChild(hashPath(name))
+}
+
+// addChild will add a child based on its hash.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChild(hash dataUsageHash) {
+	if _, ok := e.Children[hash]; ok {
+		return
+	}
+	if e.Children == nil {
+		e.Children = make(dataUsageHashMap, 1)
+	}
+	e.Children[hash] = struct{}{}
+}
+
+// find a path in the cache.
+// Returns nil if not found.
+func (d *dataUsageCache) find(path string) *dataUsageEntry {
+	due, ok := d.Cache[hashPath(path)]
+	if !ok {
+		return nil
+	}
+	return &due
+}
+
+// dui converts the flattened version of the path to DataUsageInfo.
+func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
+	e := d.find(path)
+	if e == nil {
+		return DataUsageInfo{LastUpdate: UTCNow()}
+	}
+	flat := d.flatten(*e)
+	return DataUsageInfo{
+		LastUpdate:            d.Info.LastUpdate,
+		ObjectsCount:          flat.Objects,
+		ObjectsTotalSize:      uint64(flat.Size),
+		ObjectsSizesHistogram: flat.ObjSizes.asMap(),
+		BucketsCount:          uint64(len(e.Children)),
+		BucketsSizes:          d.pathSizes(buckets),
+	}
+}
+
+// replace will add or replace an entry in the cache.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
+	hash := hashPath(path)
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != "" {
+		phash := hashPath(parent)
+		p := d.Cache[phash]
+		p.addChild(hash)
+		d.Cache[phash] = p
+	}
+}
+
+// replaceHashed add or replaces an entry to the cache based on its hash.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) {
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != nil {
+		p := d.Cache[*parent]
+		p.addChild(hash)
+		d.Cache[*parent] = p
+	}
+}
+
+// StringAll returns a detailed string representation of all entries in the cache.
+func (d *dataUsageCache) StringAll() string {
+	s := fmt.Sprintf("info:%+v\n", d.Info)
+	for k, v := range d.Cache {
+		s += fmt.Sprintf("\t%v: %+v\n", k, v)
+	}
+	return strings.TrimSpace(s)
+}
+
+// String returns a human readable representation of the string.
+func (h dataUsageHash) String() string {
+	return fmt.Sprintf("%x", uint64(h))
+}
+
+// flatten all children of the root into the root element and return it.
+func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
+	for id := range root.Children {
+		e := d.Cache[id]
+		if len(e.Children) > 0 {
+			e = d.flatten(e)
+		}
+		root.merge(e)
+	}
+	root.Children = nil
+	return root
+}
+
+// add a size to the histogram.
+func (h *sizeHistogram) add(size int64) {
+	// Fetch the histogram interval corresponding
+	// to the passed object size.
+	for i, interval := range ObjectsHistogramIntervals {
+		if size >= interval.start && size <= interval.end {
+			h[i]++
+			break
+		}
+	}
+}
+
+// asMap returns the map as a map[string]uint64.
+func (h *sizeHistogram) asMap() map[string]uint64 {
+	res := make(map[string]uint64, 7)
+	for i, count := range h {
+		res[ObjectsHistogramIntervals[i].name] = count
+	}
+	return res
+}
+
+// pathSizes returns the path sizes as a map.
+func (d *dataUsageCache) pathSizes(buckets []BucketInfo) map[string]uint64 {
+	var dst = make(map[string]uint64, len(buckets))
+	for _, bucket := range buckets {
+		e := d.find(bucket.Name)
+		if e == nil {
+			continue
+		}
+		flat := d.flatten(*e)
+		dst[bucket.Name] = uint64(flat.Size)
+	}
+	return dst
+}
+
+// sizeRecursive returns the path as a flattened entry.
+func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
+	root := d.find(path)
+	if root == nil || len(root.Children) == 0 {
+		return root
+	}
+	flat := d.flatten(*root)
+	return &flat
+}
+
+// dataUsageCache contains a cache of data usage entries.
+//msgp:ignore dataUsageCache
+type dataUsageCache struct {
+	Info  dataUsageCacheInfo
+	Cache map[dataUsageHash]dataUsageEntry
+}
+
+// root returns the root of the cache.
+func (d *dataUsageCache) root() *dataUsageEntry {
+	return d.find(d.Info.Name)
+}
+
+// rootHash returns the root of the cache.
+func (d *dataUsageCache) rootHash() dataUsageHash {
+	return hashPath(d.Info.Name)
+}
+
+// clone returns a copy of the cache with no references to the existing.
+func (d *dataUsageCache) clone() dataUsageCache {
+	clone := dataUsageCache{
+		Info:  d.Info,
+		Cache: make(map[dataUsageHash]dataUsageEntry, len(d.Cache)),
+	}
+	for k, v := range d.Cache {
+		clone.Cache[k] = v
+	}
+	return clone
+}
+
+// merge root of other into d.
+// children of root will be flattened before being merged.
+// Last update time will be set to the last updated.
+func (d *dataUsageCache) merge(other dataUsageCache) {
+	existingRoot := d.root()
+	otherRoot := other.root()
+	if existingRoot == nil && otherRoot == nil {
+		return
+	}
+	if otherRoot == nil {
+		return
+	}
+	if existingRoot == nil {
+		*d = other.clone()
+		return
+	}
+	if other.Info.LastUpdate.After(d.Info.LastUpdate) {
+		d.Info.LastUpdate = other.Info.LastUpdate
+	}
+	existingRoot.merge(*otherRoot)
+	eHash := d.rootHash()
+	for key := range otherRoot.Children {
+		entry := other.Cache[key]
+		flat := other.flatten(entry)
+		existing := d.Cache[key]
+		// If not found, merging simply adds.
+		existing.merge(flat)
+		d.replaceHashed(key, &eHash, existing)
+	}
+}
+
+// load the cache content with name from minioMetaBackgroundOpsBucket.
+// Only backend errors are returned as errors.
+// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name string) error {
+	var buf bytes.Buffer
+	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
+	if err != nil {
+		if !isErrObjectNotFound(err) {
+			return toObjectErr(err, dataUsageBucket, name)
+		}
+		*d = dataUsageCache{}
+		return nil
+	}
+	err = d.deserialize(buf.Bytes())
+	if err != nil {
+		*d = dataUsageCache{}
+		logger.LogIf(ctx, err)
+	}
+	return nil
+}
+
+// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
+func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name string) error {
+	b := d.serialize()
+	size := int64(len(b))
+	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)
+	if err != nil {
+		return err
+	}
+
+	_, err = store.PutObject(ctx,
+		dataUsageBucket,
+		name,
+		NewPutObjReader(r, nil, nil),
+		ObjectOptions{})
+	return err
+}
+
+// dataUsageCacheVer indicates the cache version.
+// Bumping the cache version will drop data from previous versions
+// and write new data with the new version.
+const dataUsageCacheVer = 1
+
+// serialize the contents of the cache.
+func (d *dataUsageCache) serialize() []byte {
+	// Alloc pessimistically
+	// dataUsageCacheVer
+	due := dataUsageEntry{}
+	msgLen := 1
+	msgLen += d.Info.Msgsize()
+	// len(d.Cache)
+	msgLen += binary.MaxVarintLen64
+	// Hashes (one for key, assume 1 child/node)
+	msgLen += len(d.Cache) * dataUsageHashLen * 2
+	msgLen += len(d.Cache) * due.Msgsize()
+
+	// Create destination buffer...
+	dst := make([]byte, 0, msgLen)
+
+	var n int
+	tmp := make([]byte, 1024)
+	// byte: version.
+	dst = append(dst, dataUsageCacheVer)
+	// Info...
+	dst, err := d.Info.MarshalMsg(dst)
+	if err != nil {
+		panic(err)
+	}
+	n = binary.PutUvarint(tmp, uint64(len(d.Cache)))
+	dst = append(dst, tmp[:n]...)
+
+	for k, v := range d.Cache {
+		// Put key
+		binary.LittleEndian.PutUint64(tmp[:dataUsageHashLen], uint64(k))
+		dst = append(dst, tmp[:8]...)
+		tmp, err = v.MarshalMsg(tmp[:0])
+		if err != nil {
+			panic(err)
+		}
+		// key, value pairs.
+		dst = append(dst, tmp...)
+
+	}
+	return dst
+}
+
+// deserialize the supplied byte slice into the cache.
+func (d *dataUsageCache) deserialize(b []byte) error {
+	if len(b) < 1 {
+		return io.ErrUnexpectedEOF
+	}
+	switch b[0] {
+	case 1:
+	default:
+		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
+	}
+	b = b[1:]
+
+	// Info...
+	b, err := d.Info.UnmarshalMsg(b)
+	if err != nil {
+		return err
+	}
+	cacheLen, n := binary.Uvarint(b)
+	if n <= 0 {
+		return fmt.Errorf("dataUsageCache: reading cachelen, n <= 0 ")
+	}
+	b = b[n:]
+	d.Cache = make(map[dataUsageHash]dataUsageEntry, cacheLen)
+
+	for i := 0; i < int(cacheLen); i++ {
+		if len(b) <= dataUsageHashLen {
+			return io.ErrUnexpectedEOF
+		}
+		k := binary.LittleEndian.Uint64(b[:dataUsageHashLen])
+		b = b[dataUsageHashLen:]
+		var v dataUsageEntry
+		b, err = v.UnmarshalMsg(b)
+		if err != nil {
+			return err
+		}
+		d.Cache[dataUsageHash(k)] = v
+	}
+	return nil
+}
+
+// Trim this from start+end of hashes.
+var hashPathCutSet = dataUsageRoot
+
+func init() {
+	if dataUsageRoot != string(filepath.Separator) {
+		hashPathCutSet = dataUsageRoot + string(filepath.Separator)
+	}
+}
+
+// hashPath calculates a hash of the provided string.
+func hashPath(data string) dataUsageHash {
+	if data != dataUsageRoot {
+		data = strings.Trim(data, hashPathCutSet)
+	}
+	data = path.Clean(data)
+	return dataUsageHash(xxhash.Sum64String(data))
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageHashMap map[dataUsageHash]struct{}
+
+// MarshalMsg implements msgp.Marshaler
+func (d dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, d.Msgsize())
+
+	// Write bin header manually
+	const mbin32 uint8 = 0xc6
+	sz := uint32(len(d)) * dataUsageHashLen
+	o = append(o, mbin32, byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz))
+
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		o = append(o, tmp[:]...)
+	}
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (d dataUsageHashMap) Msgsize() (s int) {
+	s = 5 + len(d)*dataUsageHashLen
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (d *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var hashes []byte
+	hashes, bts, err = msgp.ReadBytesZC(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "dataUsageHashMap")
+		return
+	}
+
+	var dst = make(dataUsageHashMap, len(hashes)/dataUsageHashLen)
+	for len(hashes) >= dataUsageHashLen {
+		dst[dataUsageHash(binary.LittleEndian.Uint64(hashes[:dataUsageHashLen]))] = struct{}{}
+		hashes = hashes[dataUsageHashLen:]
+	}
+	*d = dst
+	o = bts
+	return
+}
+
+func (d *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadBytesHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var dst = make(dataUsageHashMap, zb0001)
+	var tmp [8]byte
+	for i := uint32(0); i < zb0001; i++ {
+		_, err = io.ReadFull(dc, tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err, "dataUsageHashMap")
+			return
+		}
+		dst[dataUsageHash(binary.LittleEndian.Uint64(tmp[:]))] = struct{}{}
+	}
+	return nil
+}
+func (d dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteBytesHeader(uint32(len(d)) * dataUsageHashLen)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		_, err = en.Write(tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+	}
+	return nil
+}

cmd/data-usage-cache_gen.go

@@ -0,0 +1,447 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, err = dc.ReadUint8()
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "Name"
+	err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "LastUpdate"
+	err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.LastUpdate)
+	if err != nil {
+		err = msgp.WrapError(err, "LastUpdate")
+		return
+	}
+	// write "NextCycle"
+	err = en.Append(0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(z.NextCycle)
+	if err != nil {
+		err = msgp.WrapError(err, "NextCycle")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "Name"
+	o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "LastUpdate"
+	o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	o = msgp.AppendTime(o, z.LastUpdate)
+	// string "NextCycle"
+	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	o = msgp.AppendUint8(o, z.NextCycle)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, bts, err = msgp.ReadUint8Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageCacheInfo) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint8Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, err = dc.ReadInt64()
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.DecodeMsg(dc)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// array header, size 4
+	err = en.Append(0x94)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	err = en.WriteUint64(z.Objects)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		err = en.WriteUint64(z.ObjSizes[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// array header, size 4
+	o = append(o, 0x94)
+	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.Objects)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z.ObjSizes {
+		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
+	}
+	o, err = z.Children.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	bts, err = z.Children.UnmarshalMsg(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntry) Msgsize() (s int) {
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 uint64
+		zb0001, err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteUint64(uint64(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint64(o, uint64(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 uint64
+		zb0001, bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageHash) Msgsize() (s int) {
+	s = msgp.Uint64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *sizeHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *sizeHistogram) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *sizeHistogram) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *sizeHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *sizeHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size))
+	return
+}

cmd/data-usage-cache_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageCacheInfo Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageCacheInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshaldataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageEntry Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageEntry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodesizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodesizeHistogram Msgsize() is inaccurate")
+	}
+
+	vn := sizeHistogram{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage.go

@@ -20,31 +20,42 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"os"
-	"path/filepath"
+	"path"
+	"strconv"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
 )
 
 const (
-	dataUsageObjName       = "data-usage"
-	dataUsageCrawlInterval = 12 * time.Hour
-	dataUsageCrawlConf     = "MINIO_DISK_USAGE_CRAWL"
+	dataUsageObjName         = ".usage.json"
+	dataUsageCacheName       = ".usage-cache.bin"
+	envDataUsageCrawlConf    = "MINIO_DISK_USAGE_CRAWL_ENABLE"
+	envDataUsageCrawlDelay   = "MINIO_DISK_USAGE_CRAWL_DELAY"
+	envDataUsageCrawlDebug   = "MINIO_DISK_USAGE_CRAWL_DEBUG"
+	dataUsageSleepPerFolder  = 1 * time.Millisecond
+	dataUsageSleepDefMult    = 10.0
+	dataUsageUpdateDirCycles = 16
+	dataUsageRoot            = SlashSeparator
+	dataUsageBucket          = minioMetaBucket + SlashSeparator + bucketMetaPrefix
+	dataUsageStartDelay      = 5 * time.Minute // Time to wait on startup and between cycles.
 )
 
+// initDataUsageStats will start the crawler unless disabled.
 func initDataUsageStats() {
-	dataUsageEnabled, err := config.ParseBool(env.Get(dataUsageCrawlConf, config.EnableOn))
-	if err == nil && !dataUsageEnabled {
-		return
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+		go runDataUsageInfoUpdateRoutine()
 	}
-	go runDataUsageInfoUpdateRoutine()
 }
 
+// runDataUsageInfoUpdateRoutine will contain the main crawler.
 func runDataUsageInfoUpdateRoutine() {
 	// Wait until the object layer is ready
 	var objAPI ObjectLayer
@@ -57,37 +68,13 @@ func runDataUsageInfoUpdateRoutine() {
 		break
 	}
 
-	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
-}
-
-// timeToNextCrawl returns the duration until next crawl should occur
-// this is validated by verifying the LastUpdate time.
-func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
-	if err != nil {
-		// Upon an error wait for like 10
-		// seconds to start the crawler.
-		return 10 * time.Second
-	}
-	// File indeed doesn't exist when LastUpdate is zero
-	// so we have never crawled, start crawl right away.
-	if dataUsageInfo.LastUpdate.IsZero() {
-		return 1 * time.Second
-	}
-	timeSinceLastUpdate := UTCNow().Sub(dataUsageInfo.LastUpdate)
-	if timeSinceLastUpdate > dataUsageCrawlInterval {
-		// Waited long enough start crawl in a 1 second
-		return 1 * time.Second
-	}
-	// No crawling needed, ask the routine to wait until
-	// the daily interval 12hrs - delta between last update
-	// with current time.
-	return dataUsageCrawlInterval - timeSinceLastUpdate
+	runDataUsageInfo(GlobalContext, objAPI)
 }
 
 var dataUsageLockTimeout = lifecycleLockTimeout
 
-func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
+func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only 1 crawler is running on the cluster.
 	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
 	for {
 		err := locker.GetLock(dataUsageLockTimeout)
@@ -99,47 +86,51 @@ func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan stru
 		// data usage calculator role for its lifetime.
 		break
 	}
-
 	for {
-		wait := timeToCrawl(ctx, objAPI)
 		select {
-		case <-endCh:
+		case <-ctx.Done():
 			locker.Unlock()
 			return
-		case <-time.NewTimer(wait).C:
-			// Crawl only when no previous crawl has occurred,
-			// or its been too long since last crawl.
-			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
+			// Wait before starting next cycle and wait on startup.
+		case <-time.NewTimer(dataUsageStartDelay).C:
+			results := make(chan DataUsageInfo, 1)
+			go storeDataUsageInBackend(ctx, objAPI, results)
+			err := objAPI.CrawlAndGetDataUsage(ctx, results)
+			close(results)
 			logger.LogIf(ctx, err)
 		}
 	}
 }
 
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
-	dataUsageJSON, err := json.Marshal(dataUsageInfo)
-	if err != nil {
-		return err
-	}
+// storeDataUsageInBackend will store all objects sent on the gui channel until closed.
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
+	for dataUsageInfo := range gui {
+		dataUsageJSON, err := json.Marshal(dataUsageInfo)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		size := int64(len(dataUsageJSON))
+		r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
 
-	size := int64(len(dataUsageJSON))
-	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
-	if err != nil {
-		return err
+		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+		logger.LogIf(ctx, err)
 	}
-
-	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
-	return err
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	var dataUsageInfoJSON bytes.Buffer
 
-	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
+	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
 	if err != nil {
 		if isErrObjectNotFound(err) {
 			return DataUsageInfo{}, nil
 		}
-		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
+		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
 	}
 
 	var dataUsageInfo DataUsageInfo
@@ -160,52 +151,315 @@ type Item struct {
 
 type getSizeFn func(item Item) (int64, error)
 
-func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO func(), getSize getSizeFn) DataUsageInfo {
-	var dataUsageInfo = DataUsageInfo{
-		BucketsSizes:          make(map[string]uint64),
-		ObjectsSizesHistogram: make(map[string]uint64),
+type cachedFolder struct {
+	name   string
+	parent *dataUsageHash
+}
+
+type folderScanner struct {
+	root               string
+	getSize            getSizeFn
+	oldCache           dataUsageCache
+	newCache           dataUsageCache
+	waitForLowActiveIO func()
+
+	dataUsageCrawlMult  float64
+	dataUsageCrawlDebug bool
+
+	newFolders      []cachedFolder
+	existingFolders []cachedFolder
+}
+
+// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
+// sleep is limited to max 1 second.
+func sleepDuration(d time.Duration, x float64) {
+	// Don't sleep for really small amount of time
+	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
+		if d > time.Second {
+			d = time.Second
+		}
+		time.Sleep(d)
+	}
+}
+
+// scanQueuedLevels will scan the provided folders.
+// Files found in the folders will be added to f.newCache.
+// If final is provided folders will be put into f.newFolders or f.existingFolders.
+// If final is not provided the folders found are returned from the function.
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+	var nextFolders []cachedFolder
+	done := ctx.Done()
+	for _, folder := range folders {
+		select {
+		case <-done:
+			return nil, ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+
+		cache := dataUsageEntry{}
+		thisHash := hashPath(folder.name)
+
+		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+			// Parse
+			entName = path.Clean(path.Join(folder.name, entName))
+			bucket, _ := path2BucketObjectWithBasePath(f.root, entName)
+			if bucket == "" {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
+				}
+				return nil
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
+				}
+				return nil
+			}
+
+			select {
+			case <-done:
+				return ctx.Err()
+			default:
+			}
+
+			if typ&os.ModeDir != 0 {
+				h := hashPath(entName)
+				_, exists := f.oldCache.Cache[h]
+				cache.addChildString(entName)
+
+				this := cachedFolder{name: entName, parent: &thisHash}
+				cache.addChild(h)
+				if final {
+					if exists {
+						f.existingFolders = append(f.existingFolders, this)
+					} else {
+						f.newFolders = append(f.newFolders, this)
+					}
+				} else {
+					nextFolders = append(nextFolders, this)
+				}
+				return nil
+			}
+			f.waitForLowActiveIO()
+			// Dynamic time delay.
+			t := UTCNow()
+
+			// Get file size, ignore errors.
+			size, err := f.getSize(Item{Path: path.Join(f.root, entName), Typ: typ})
+
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			if err == errSkipFile {
+				return nil
+			}
+			logger.LogIf(ctx, err)
+			cache.Size += size
+			cache.Objects++
+			cache.ObjSizes.add(size)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		f.newCache.replaceHashed(thisHash, folder.parent, cache)
+	}
+	return nextFolders, nil
+}
+
+// deepScanFolder will deep scan a folder and return the size if no error occurs.
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
+	var cache dataUsageEntry
+
+	done := ctx.Done()
+
+	var addDir func(entName string, typ os.FileMode) error
+	var dirStack = []string{f.root, folder}
+
+	addDir = func(entName string, typ os.FileMode) error {
+		select {
+		case <-done:
+			return ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		if typ&os.ModeDir != 0 {
+			dirStack = append(dirStack, entName)
+			err := readDirFn(path.Join(dirStack...), addDir)
+			dirStack = dirStack[:len(dirStack)-1]
+			sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+			return err
+		}
+
+		// Dynamic time delay.
+		t := UTCNow()
+
+		// Get file size, ignore errors.
+		dirStack = append(dirStack, entName)
+		fileName := path.Join(dirStack...)
+		dirStack = dirStack[:len(dirStack)-1]
+
+		size, err := f.getSize(Item{Path: fileName, Typ: typ})
+
+		// Don't sleep for really small amount of time
+		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
+		if err == errSkipFile {
+			return nil
+		}
+		logger.LogIf(ctx, err)
+		cache.Size += size
+		cache.Objects++
+		cache.ObjSizes.add(size)
+		return nil
+	}
+	err := readDirFn(path.Join(dirStack...), addDir)
+	if err != nil {
+		return nil, err
+	}
+	return &cache, nil
+}
+
+// updateUsage will crawl the basepath+cache.Info.Name and return an updated cache.
+// The returned cache will always be valid, but may not be updated from the existing.
+// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// If the supplied context is canceled the function will return at the first chance.
+func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+	t := UTCNow()
+
+	dataUsageDebug := env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn
+	defer func() {
+		if dataUsageDebug {
+			logger.Info(color.Green("updateUsage")+" Crawl time at %s: %v", basePath, time.Since(t))
+		}
+	}()
+
+	if cache.Info.Name == "" {
+		cache.Info.Name = dataUsageRoot
 	}
 
-	fastWalk(basePath, 1, doneCh, func(path string, typ os.FileMode) error {
-		// Wait for I/O to go down.
-		waitForLowActiveIO()
+	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		delayMult = dataUsageSleepDefMult
+	}
 
-		bucket, entry := path2BucketObjectWithBasePath(basePath, path)
-		if bucket == "" {
-			return nil
-		}
+	s := folderScanner{
+		root:                basePath,
+		getSize:             getSize,
+		oldCache:            cache,
+		newCache:            dataUsageCache{Info: cache.Info},
+		waitForLowActiveIO:  waitForLowActiveIO,
+		newFolders:          nil,
+		existingFolders:     nil,
+		dataUsageCrawlMult:  delayMult,
+		dataUsageCrawlDebug: dataUsageDebug,
+	}
 
-		if isReservedOrInvalidBucket(bucket, false) {
-			return filepath.SkipDir
-		}
+	if s.dataUsageCrawlDebug {
+		logger.Info(color.Green("runDataUsageInfo:") + " Starting crawler master")
+	}
 
-		if entry == "" && typ&os.ModeDir != 0 {
-			dataUsageInfo.BucketsCount++
-			dataUsageInfo.BucketsSizes[bucket] = 0
-			return nil
-		}
+	done := ctx.Done()
+	var flattenLevels = 3
 
-		if typ&os.ModeDir != 0 {
-			return nil
-		}
+	// If we are scanning inside a bucket reduce depth by 1.
+	if cache.Info.Name != dataUsageRoot {
+		flattenLevels--
+	}
 
-		t := time.Now()
-		size, err := getSize(Item{path, typ})
-		// Use the response time of the getSize call to guess system load.
-		// Sleep equivalent time.
-		if d := time.Since(t); d > 100*time.Microsecond {
-			time.Sleep(d)
+	var logPrefix, logSuffix string
+	if s.dataUsageCrawlDebug {
+		logPrefix = color.Green("data-usage: ")
+		logSuffix = color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Cycle: %v"+logSuffix, cache.Info.NextCycle)
+	}
+
+	// Always scan flattenLevels deep. Cache root is level 0.
+	todo := []cachedFolder{{name: cache.Info.Name}}
+	for i := 0; i < flattenLevels; i++ {
+		if s.dataUsageCrawlDebug {
+			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
 		}
+		select {
+		case <-done:
+			return cache, ctx.Err()
+		default:
+		}
+		var err error
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
 		if err != nil {
-			return errSkipFile
+			// No useful information...
+			return cache, err
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+	}
+
+	// Add new folders first
+	for _, folder := range s.newFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replace(folder.name, "", *du)
+		// Add to parent manually
+		if folder.parent != nil {
+			parent := s.newCache.Cache[*folder.parent]
+			parent.addChildString(folder.name)
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+	}
+
+	// Do selective scanning of existing folders.
+	for _, folder := range s.existingFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		h := hashPath(folder.name)
+		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
+			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
+			continue
 		}
 
-		dataUsageInfo.ObjectsCount++
-		dataUsageInfo.ObjectsTotalSize += uint64(size)
-		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
-		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
-		return nil
-	})
+		// Update on this cycle...
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replaceHashed(h, folder.parent, *du)
+	}
 
-	return dataUsageInfo
+	s.newCache.Info.LastUpdate = UTCNow()
+	s.newCache.Info.NextCycle++
+	return s.newCache, nil
 }

cmd/data-usage_test.go

@@ -0,0 +1,651 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type usageTestFile struct {
+	name string
+	size int
+}
+
+func TestDataUsageUpdate(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageUpdate")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+
+	got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1322310,
+			flatten: true,
+			objs:    8,
+			oSizes:  sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:    "/dir1/dira",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:    "/dir1/dira/",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "/dir1/dira",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:   "/dir1/dira/",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1363315,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    1300010,
+			flatten: true,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    363515,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    300210,
+			objs:    4,
+			flatten: true,
+			oSizes:  sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func TestDataUsageUpdatePrefix(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUpdateUsagePrefix")
+	if err != nil {
+		t.Skip(err)
+	}
+	base = filepath.Join(base, "bucket")
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "bucket/rootfile", size: 10000},
+		{name: "bucket/rootfile2", size: 10000},
+		{name: "bucket/dir1/d1file", size: 2000},
+		{name: "bucket/dir2/d2file", size: 300},
+		{name: "bucket/dir1/dira/dafile", size: 100000},
+		{name: "bucket/dir1/dira/dbfile", size: 200000},
+		{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1322310,
+			objs:   8,
+			oSizes: sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "bucket/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "bucket/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "bucket/dir1/dira/",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "bucket/newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "bucket/newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "bucket/dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "bucket/dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "bucket/rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1363315,
+			objs:   14,
+			oSizes: sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "bucket/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "bucket/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   363515,
+			objs:   14,
+			oSizes: sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   300210,
+			objs:   4,
+			oSizes: sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) {
+	for _, f := range files {
+		err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestDataUsageCacheSerialize(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageCacheSerialize")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := want.serialize()
+	var got dataUsageCache
+	err = got.deserialize(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got.Info.LastUpdate.IsZero() {
+		t.Error("lastupdate not set")
+	}
+
+	if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) {
+		t.Fatalf("deserialize mismatch\nwant: %+v\ngot:  %+v", want, got)
+	}
+}

cmd/disk-cache-backend.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,7 +37,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/sio"
-	"github.com/ncw/directio"
 	"go.uber.org/atomic"
 )
 
@@ -158,7 +157,7 @@ func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int)
 		onlineMutex:   &sync.RWMutex{},
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := directio.AlignedBlock(int(cacheBlkSize))
+				b := disk.AlignedBlock(int(cacheBlkSize))
 				return &b
 			},
 		},

cmd/disk-cache.go

@@ -693,7 +693,7 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 	return c, nil
 }
 
-func (c *cacheObjects) gc(ctx context.Context, doneCh chan struct{}) {
+func (c *cacheObjects) gc(ctx context.Context, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cacheGCInterval)
 
 	defer ticker.Stop()

cmd/fastwalk.go

@@ -11,173 +11,12 @@ package cmd
 import (
 	"errors"
 	"os"
-	"path/filepath"
 	"strings"
-	"sync"
 )
 
 var errSkipFile = errors.New("fastwalk: skip this file")
 
-// Walk is a faster implementation of filepath.Walk.
-//
-// filepath.Walk's design necessarily calls os.Lstat on each file,
-// even if the caller needs less info.
-// Many tools need only the type of each file.
-// On some platforms, this information is provided directly by the readdir
-// system call, avoiding the need to stat each file individually.
-// fastwalk_unix.go contains a fork of the syscall routines.
-//
-// See golang.org/issue/16399
-//
-// Walk walks the file tree rooted at root, calling walkFn for
-// each file or directory in the tree, including root.
-//
-// If fastWalk returns filepath.SkipDir, the directory is skipped.
-//
-// Unlike filepath.Walk:
-//   * file stat calls must be done by the user.
-//     The only provided metadata is the file type, which does not include
-//     any permission bits.
-//   * multiple goroutines stat the filesystem concurrently. The provided
-//     walkFn must be safe for concurrent use.
-//   * fastWalk can follow symlinks if walkFn returns the TraverseLink
-//     sentinel error. It is the walkFn's responsibility to prevent
-//     fastWalk from going into symlink cycles.
-func fastWalk(root string, nworkers int, doneCh <-chan struct{}, walkFn func(path string, typ os.FileMode) error) error {
-
-	// Make sure to wait for all workers to finish, otherwise
-	// walkFn could still be called after returning. This Wait call
-	// runs after close(e.donec) below.
-	var wg sync.WaitGroup
-	defer wg.Wait()
-
-	w := &walker{
-		fn:       walkFn,
-		enqueuec: make(chan walkItem, nworkers), // buffered for performance
-		workc:    make(chan walkItem, nworkers), // buffered for performance
-		donec:    make(chan struct{}),
-
-		// buffered for correctness & not leaking goroutines:
-		resc: make(chan error, nworkers),
-	}
-	defer close(w.donec)
-
-	for i := 0; i < nworkers; i++ {
-		wg.Add(1)
-		go w.doWork(&wg)
-	}
-
-	todo := []walkItem{{dir: root}}
-	out := 0
-	for {
-		workc := w.workc
-		var workItem walkItem
-		if len(todo) == 0 {
-			workc = nil
-		} else {
-			workItem = todo[len(todo)-1]
-		}
-		select {
-		case <-doneCh:
-			return nil
-		case workc <- workItem:
-			todo = todo[:len(todo)-1]
-			out++
-		case it := <-w.enqueuec:
-			todo = append(todo, it)
-		case err := <-w.resc:
-			out--
-			if err != nil {
-				return err
-			}
-			if out == 0 && len(todo) == 0 {
-				// It's safe to quit here, as long as the buffered
-				// enqueue channel isn't also readable, which might
-				// happen if the worker sends both another unit of
-				// work and its result before the other select was
-				// scheduled and both w.resc and w.enqueuec were
-				// readable.
-				select {
-				case it := <-w.enqueuec:
-					todo = append(todo, it)
-				default:
-					return nil
-				}
-			}
-		}
-	}
-}
-
-// doWork reads directories as instructed (via workc) and runs the
-// user's callback function.
-func (w *walker) doWork(wg *sync.WaitGroup) {
-	defer wg.Done()
-	for {
-		select {
-		case <-w.donec:
-			return
-		case it := <-w.workc:
-			select {
-			case <-w.donec:
-				return
-			case w.resc <- w.walk(it.dir, !it.callbackDone):
-			}
-		}
-	}
-}
-
-type walker struct {
-	fn func(path string, typ os.FileMode) error
-
-	donec    chan struct{} // closed on fastWalk's return
-	workc    chan walkItem // to workers
-	enqueuec chan walkItem // from workers
-	resc     chan error    // from workers
-}
-
-type walkItem struct {
-	dir          string
-	callbackDone bool // callback already called; don't do it again
-}
-
-func (w *walker) enqueue(it walkItem) {
-	select {
-	case w.enqueuec <- it:
-	case <-w.donec:
-	}
-}
-
-var stringsBuilderPool = sync.Pool{
-	New: func() interface{} {
-		return &strings.Builder{}
-	},
-}
-
-func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
-	builder := stringsBuilderPool.Get().(*strings.Builder)
-	defer func() {
-		builder.Reset()
-		stringsBuilderPool.Put(builder)
-	}()
-
-	builder.WriteString(dirName)
-	if !strings.HasSuffix(dirName, SlashSeparator) {
-		builder.WriteString(SlashSeparator)
-	}
-	builder.WriteString(baseName)
-	if typ == os.ModeDir {
-		w.enqueue(walkItem{dir: builder.String()})
-		return nil
-	}
-
-	err := w.fn(builder.String(), typ)
-	if err == filepath.SkipDir || err == errSkipFile {
-		return nil
-	}
-	return err
-}
-
-func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error {
+func readDirFn(dirName string, fn func(entName string, typ os.FileMode) error) error {
 	fis, err := readDir(dirName)
 	if err != nil {
 		return err
@@ -188,23 +27,9 @@ func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode)
 			mode |= os.ModeDir
 		}
 
-		if err = fn(dirName, fi, mode); err != nil {
+		if err = fn(fi, mode); err != nil {
 			return err
 		}
 	}
 	return nil
 }
-
-func (w *walker) walk(root string, runUserCallback bool) error {
-	if runUserCallback {
-		err := w.fn(root, os.ModeDir)
-		if err == filepath.SkipDir || err == errSkipFile {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-	}
-
-	return readDirFn(root, w.onDirEnt)
-}

cmd/format-xl.go

@@ -828,7 +828,7 @@ func ecDrivesNoConfig(drivesPerSet int) int {
 // Make XL backend meta volumes.
 func makeFormatXLMetaVolumes(disk StorageAPI) error {
 	// Attempt to create MinIO internal buckets.
-	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
+	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
 }
 
 var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

cmd/fs-v1-helpers.go

@@ -429,7 +429,7 @@ func fsDeleteFile(ctx context.Context, basePath, deletePath string) error {
 		return err
 	}
 
-	if err := deleteFile(basePath, deletePath); err != nil {
+	if err := deleteFile(basePath, deletePath, false); err != nil {
 		if err != errFileNotFound {
 			logger.LogIf(ctx, err)
 		}

cmd/fs-v1-multipart.go

@@ -757,7 +757,7 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
-func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/fs-v1-multipart_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 )
@@ -34,33 +35,38 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	obj := initFSObjects(disk, t)
 	fs := obj.(*FSObjects)
 
-	// Close the go-routine, we are going to
-	// manually start it and test in this test case.
-	GlobalServiceDoneCh <- struct{}{}
-
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{})
+	// Create a context we can cancel.
+	ctx, cancel := context.WithCancel(context.Background())
+	obj.MakeBucketWithLocation(ctx, bucketName, "")
+
+	uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
-	go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, GlobalServiceDoneCh)
+	var cleanupWg sync.WaitGroup
+	cleanupWg.Add(1)
+	go func() {
+		defer cleanupWg.Done()
+		fs.cleanupStaleMultipartUploads(context.Background(), time.Millisecond, 0, ctx.Done())
+	}()
 
-	// Wait for 40ms such that - we have given enough time for
-	// cleanup routine to kick in.
-	time.Sleep(40 * time.Millisecond)
-
-	// Close the routine we do not need it anymore.
-	GlobalServiceDoneCh <- struct{}{}
+	// Wait for 100ms such that - we have given enough time for
+	// cleanup routine to kick in. Flaky on slow systems...
+	time.Sleep(100 * time.Millisecond)
+	cancel()
+	cleanupWg.Wait()
 
 	// Check if upload id was already purged.
 	if err = obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
+	} else {
+		t.Error("Item was not cleaned up.")
 	}
 }
 

cmd/fs-v1.go

@@ -37,12 +37,10 @@ import (
 	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/object/tagging"
 	"github.com/minio/minio/pkg/bucket/policy"
-
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
@@ -112,7 +110,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 		return err
 	}
 
-	if err := os.MkdirAll(pathJoin(fsPath, minioMetaBackgroundOpsBucket), 0777); err != nil {
+	if err := os.MkdirAll(pathJoin(fsPath, dataUsageBucket), 0777); err != nil {
 		return err
 	}
 
@@ -235,9 +233,22 @@ func (fs *FSObjects) waitForLowActiveIO() {
 }
 
 // CrawlAndGetDataUsage returns data usage stats of the current FS deployment
-func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-	dataUsageInfo := updateUsage(fs.fsPath, endCh, fs.waitForLowActiveIO, func(item Item) (int64, error) {
-		// Get file size, symlinks which cannot bex
+func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+	// Load bucket totals
+	var oldCache dataUsageCache
+	err := oldCache.load(ctx, fs, dataUsageCacheName)
+	if err != nil {
+		return err
+	}
+	if oldCache.Info.Name == "" {
+		oldCache.Info.Name = dataUsageRoot
+	}
+	buckets, err := fs.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+	cache, err := updateUsage(ctx, fs.fsPath, oldCache, fs.waitForLowActiveIO, func(item Item) (int64, error) {
+		// Get file size, symlinks which cannot be
 		// followed are automatically filtered by fastwalk.
 		fi, err := os.Stat(item.Path)
 		if err != nil {
@@ -246,10 +257,13 @@ func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan stru
 		return fi.Size(), nil
 	})
 
-	dataUsageInfo.LastUpdate = UTCNow()
-	atomic.StoreUint64(&fs.totalUsed, dataUsageInfo.ObjectsTotalSize)
+	// Even if there was an error, the new cache may have better info.
+	if cache.Info.LastUpdate.After(oldCache.Info.LastUpdate) {
+		logger.LogIf(ctx, cache.save(ctx, fs, dataUsageCacheName))
+		updates <- cache.dui(dataUsageRoot, buckets)
+	}
 
-	return dataUsageInfo
+	return err
 }
 
 /// Bucket operations
@@ -1058,15 +1072,18 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // is a leaf or non-leaf entry.
 func (fs *FSObjects) listDirFactory() ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
-			return
+			return false, nil
+		}
+		if len(entries) == 0 {
+			return true, nil
 		}
 		sort.Strings(entries)
-		return filterMatchingPrefix(entries, prefixEntry)
+		return false, filterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -1233,7 +1250,7 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
@@ -1256,7 +1273,7 @@ func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results ch
 }
 
 // HealObjects - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) (e error) {
+func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) (e error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }

cmd/fs-v1_test.go

@@ -356,8 +356,6 @@ func TestFSListBuckets(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	GlobalServiceDoneCh <- struct{}{}
-
 	// Create a bucket with invalid name
 	if err := os.MkdirAll(pathJoin(fs.fsPath, "vo^"), 0777); err != nil {
 		t.Fatal("Unexpected error: ", err)
@@ -392,7 +390,7 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false, madmin.HealDeepScan)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", madmin.HealOpts{})
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
@@ -404,7 +402,7 @@ func TestFSHealObjects(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	err := obj.HealObjects(context.Background(), "bucket", "prefix", nil)
+	err := obj.HealObjects(context.Background(), "bucket", "prefix", madmin.HealOpts{}, nil)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -309,8 +309,5 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
 	}
 
-	// Set uptime time after object layer has initialized.
-	globalBootTime = UTCNow()
-
 	handleSignals()
 }

cmd/gateway-unsupported.go

@@ -50,9 +50,9 @@ func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
 type GatewayUnsupported struct{}
 
 // CrawlAndGetDataUsage - crawl is not implemented for gateway
-func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
-	return DataUsageInfo{}
+	return NotImplemented{}
 }
 
 // NewNSLock is a dummy stub for gateway.
@@ -167,7 +167,7 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 }
 
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 
@@ -182,7 +182,7 @@ func (a GatewayUnsupported) Walk(ctx context.Context, bucket, prefix string, res
 }
 
 // HealObjects - Not implemented stub
-func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) (e error) {
+func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) (e error) {
 	return NotImplemented{}
 }
 

cmd/gateway/azure/gateway-azure.go

@@ -39,7 +39,6 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
-	"github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bucket/policy"
@@ -1196,7 +1195,7 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	objMetadata["md5sum"] = cmd.ComputeCompleteMultipartMD5(uploadedParts)
+	objMetadata["md5sum"] = minio.ComputeCompleteMultipartMD5(uploadedParts)
 
 	_, err = objBlob.CommitBlockList(ctx, allBlocks, objProperties, objMetadata, azblob.BlobAccessConditions{})
 	if err != nil {

cmd/gateway/hdfs/gateway-hdfs.go

@@ -326,7 +326,7 @@ func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketIn
 
 func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
 		f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, prefixDir))
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -341,6 +341,9 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 			logger.LogIf(context.Background(), err)
 			return
 		}
+		if len(fis) == 0 {
+			return true, nil
+		}
 		for _, fi := range fis {
 			if fi.IsDir() {
 				entries = append(entries, fi.Name()+hdfsSeparator)
@@ -348,7 +351,7 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 				entries = append(entries, fi.Name())
 			}
 		}
-		return minio.FilterMatchingPrefix(entries, prefixEntry)
+		return false, minio.FilterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.

cmd/gateway/s3/gateway-s3-sse.go

@@ -679,7 +679,7 @@ func getGWContentPath(object string) string {
 }
 
 // Clean-up the stale incomplete encrypted multipart uploads. Should be run in a Go routine.
-func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/generic-handlers.go

@@ -353,7 +353,7 @@ func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
 // supported amz date formats.
 func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
 	for _, amzDateHeader := range amzDateHeaders {
-		amzDateStr := req.Header.Get(http.CanonicalHeaderKey(amzDateHeader))
+		amzDateStr := req.Header.Get(amzDateHeader)
 		if amzDateStr != "" {
 			return parseAmzDate(amzDateStr)
 		}

cmd/global-heal.go

@@ -29,7 +29,6 @@ import (
 const (
 	bgHealingUUID = "0000-0000-0000-0000"
 	leaderTick    = time.Hour
-	healTick      = time.Hour
 	healInterval  = 30 * 24 * time.Hour
 )
 
@@ -40,7 +39,7 @@ var leaderLockTimeout = newDynamicTimeout(time.Minute, time.Minute)
 func newBgHealSequence(numDisks int) *healSequence {
 
 	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
@@ -75,6 +74,7 @@ func getLocalBackgroundHealStatus() madmin.BgHealState {
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.scannedItemsCount,
 		LastHealActivity:  bgSeq.lastHealActivity,
+		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
 	}
 }
 
@@ -112,6 +112,19 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
 	return nil
 }
 
+// Returns the duration to the next background healing round
+func durationToNextHealRound(lastHeal time.Time) time.Duration {
+	if lastHeal.IsZero() {
+		lastHeal = globalBootTime
+	}
+
+	d := lastHeal.Add(healInterval).Sub(UTCNow())
+	if d < 0 {
+		return time.Second
+	}
+	return d
+}
+
 // Healing leader will take the charge of healing all erasure sets
 func execLeaderTasks(z *xlZones) {
 	ctx := context.Background()
@@ -132,10 +145,7 @@ func execLeaderTasks(z *xlZones) {
 
 	lastScanTime := time.Now() // So that we don't heal immediately, but after one month.
 	for {
-		if time.Since(lastScanTime) < healInterval {
-			time.Sleep(healTick)
-			continue
-		}
+		time.Sleep(durationToNextHealRound(lastScanTime))
 		for _, zone := range z.zones {
 			// Heal set by set
 			for i, set := range zone.sets {

cmd/globals.go

@@ -144,8 +144,11 @@ var (
 
 	globalNotificationSys  *NotificationSys
 	globalConfigTargetList *event.TargetList
-	globalPolicySys        *PolicySys
-	globalIAMSys           *IAMSys
+	// globalEnvTargetList has list of targets configured via env.
+	globalEnvTargetList *event.TargetList
+
+	globalPolicySys *PolicySys
+	globalIAMSys    *IAMSys
 
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
@@ -185,11 +188,14 @@ var (
 	// Global HTTP request statisitics
 	globalHTTPStats = newHTTPStats()
 
-	// Time when object layer was initialized on start up.
-	globalBootTime time.Time
+	// Time when the server is started
+	globalBootTime = UTCNow()
 
 	globalActiveCred auth.Credentials
 
+	// Hold the old server credentials passed by the environment
+	globalOldCred auth.Credentials
+
 	// Indicates if config is to be encrypted
 	globalConfigEncrypted bool
 

cmd/http-stats.go

@@ -101,28 +101,24 @@ type HTTPAPIStats struct {
 
 // Inc increments the api stats counter.
 func (stats *HTTPAPIStats) Inc(api string) {
-	stats.Lock()
-	defer stats.Unlock()
 	if stats == nil {
 		return
 	}
+	stats.Lock()
+	defer stats.Unlock()
 	if stats.apiStats == nil {
 		stats.apiStats = make(map[string]int)
 	}
-	if _, ok := stats.apiStats[api]; ok {
-		stats.apiStats[api]++
-		return
-	}
-	stats.apiStats[api] = 1
+	stats.apiStats[api]++
 }
 
 // Dec increments the api stats counter.
 func (stats *HTTPAPIStats) Dec(api string) {
-	stats.Lock()
-	defer stats.Unlock()
 	if stats == nil {
 		return
 	}
+	stats.Lock()
+	defer stats.Unlock()
 	if val, ok := stats.apiStats[api]; ok && val > 0 {
 		stats.apiStats[api]--
 	}

cmd/http/headers.go

@@ -77,6 +77,9 @@ const (
 	AmzObjectLockLegalHold        = "X-Amz-Object-Lock-Legal-Hold"
 	AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention"
 
+	// Multipart parts count
+	AmzMpPartsCount = "x-amz-mp-parts-count"
+
 	// Dummy putBucketACL
 	AmzACL = "x-amz-acl"
 

cmd/iam-etcd-store.go

@@ -28,6 +28,7 @@ import (
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/mvcc/mvccpb"
+	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -174,7 +175,11 @@ func (ies *IAMEtcdStore) migrateUsersConfigToV1(isSTS bool) error {
 
 			// 2. copy policy to new loc.
 			mp := newMappedPolicy(policyName)
-			path := getMappedPolicyPath(user, isSTS, false)
+			userType := regularUser
+			if isSTS {
+				userType = stsUser
+			}
+			path := getMappedPolicyPath(user, userType, false)
 			if err := ies.saveIAMConfig(mp, path); err != nil {
 				return err
 			}
@@ -300,9 +305,9 @@ func (ies *IAMEtcdStore) loadPolicyDocs(m map[string]iampolicy.Policy) error {
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadUser(user string, isSTS bool, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error {
 	var u UserIdentity
-	err := ies.loadIAMConfig(&u, getUserIdentityPath(user, isSTS))
+	err := ies.loadIAMConfig(&u, getUserIdentityPath(user, userType))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchUser
@@ -313,11 +318,31 @@ func (ies *IAMEtcdStore) loadUser(user string, isSTS bool, m map[string]auth.Cre
 	if u.Credentials.IsExpired() {
 		// Delete expired identity.
 		ctx := ies.getContext()
-		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, isSTS))
-		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, isSTS, false))
+		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
+		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
 		return nil
 	}
 
+	// If this is a service account, rotate the session key if we are changing the server creds
+	if globalOldCred.IsValid() && u.Credentials.IsServiceAccount() {
+		if !globalOldCred.Equal(globalActiveCred) {
+			m := jwtgo.MapClaims{}
+			stsTokenCallback := func(t *jwtgo.Token) (interface{}, error) {
+				return []byte(globalOldCred.SecretKey), nil
+			}
+			if _, err := jwtgo.ParseWithClaims(u.Credentials.SessionToken, m, stsTokenCallback); err == nil {
+				jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.MapClaims(m))
+				if token, err := jwt.SignedString([]byte(globalActiveCred.SecretKey)); err == nil {
+					u.Credentials.SessionToken = token
+					err := ies.saveIAMConfig(&u, getUserIdentityPath(user, userType))
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
@@ -326,10 +351,15 @@ func (ies *IAMEtcdStore) loadUser(user string, isSTS bool, m map[string]auth.Cre
 
 }
 
-func (ies *IAMEtcdStore) loadUsers(isSTS bool, m map[string]auth.Credentials) error {
-	basePrefix := iamConfigUsersPrefix
-	if isSTS {
+func (ies *IAMEtcdStore) loadUsers(userType IAMUserType, m map[string]auth.Credentials) error {
+	var basePrefix string
+	switch userType {
+	case srvAccUser:
+		basePrefix = iamConfigServiceAccountsPrefix
+	case stsUser:
 		basePrefix = iamConfigSTSPrefix
+	default:
+		basePrefix = iamConfigUsersPrefix
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
@@ -345,7 +375,7 @@ func (ies *IAMEtcdStore) loadUsers(isSTS bool, m map[string]auth.Credentials) er
 
 	// Reload config for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadUser(user, isSTS, m); err != nil {
+		if err = ies.loadUser(user, userType, m); err != nil {
 			return err
 		}
 	}
@@ -388,9 +418,9 @@ func (ies *IAMEtcdStore) loadGroups(m map[string]GroupInfo) error {
 
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicy(name string, isSTS, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicy(name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
 	var p MappedPolicy
-	err := ies.loadIAMConfig(&p, getMappedPolicyPath(name, isSTS, isGroup))
+	err := ies.loadIAMConfig(&p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
@@ -402,19 +432,23 @@ func (ies *IAMEtcdStore) loadMappedPolicy(name string, isSTS, isGroup bool, m ma
 
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicies(isSTS, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	defer cancel()
 	ies.setContext(ctx)
 	defer ies.clearContext()
 	var basePrefix string
-	switch {
-	case isSTS:
-		basePrefix = iamConfigPolicyDBSTSUsersPrefix
-	case isGroup:
+	if isGroup {
 		basePrefix = iamConfigPolicyDBGroupsPrefix
-	default:
-		basePrefix = iamConfigPolicyDBUsersPrefix
+	} else {
+		switch userType {
+		case srvAccUser:
+			basePrefix = iamConfigPolicyDBServiceAccountsPrefix
+		case stsUser:
+			basePrefix = iamConfigPolicyDBSTSUsersPrefix
+		default:
+			basePrefix = iamConfigPolicyDBUsersPrefix
+		}
 	}
 	r, err := ies.client.Get(ctx, basePrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
@@ -425,7 +459,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(isSTS, isGroup bool, m map[string]Ma
 
 	// Reload config and policies for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadMappedPolicy(user, isSTS, isGroup, m); err != nil {
+		if err = ies.loadMappedPolicy(user, userType, isGroup, m); err != nil {
 			return err
 		}
 	}
@@ -452,29 +486,32 @@ func (ies *IAMEtcdStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 	}
 
 	// load STS temp users
-	if err := ies.loadUsers(true, iamUsersMap); err != nil {
+	if err := ies.loadUsers(stsUser, iamUsersMap); err != nil {
 		return err
 	}
 
 	if isMinIOUsersSys {
 		// load long term users
-		if err := ies.loadUsers(false, iamUsersMap); err != nil {
+		if err := ies.loadUsers(regularUser, iamUsersMap); err != nil {
+			return err
+		}
+		if err := ies.loadUsers(srvAccUser, iamUsersMap); err != nil {
 			return err
 		}
 		if err := ies.loadGroups(iamGroupsMap); err != nil {
 			return err
 		}
-		if err := ies.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
+		if err := ies.loadMappedPolicies(regularUser, false, iamUserPolicyMap); err != nil {
 			return err
 		}
 	}
 
 	// load STS policy mappings into the same map
-	if err := ies.loadMappedPolicies(true, false, iamUserPolicyMap); err != nil {
+	if err := ies.loadMappedPolicies(stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 	// load policies mapped to groups
-	if err := ies.loadMappedPolicies(false, true, iamGroupPolicyMap); err != nil {
+	if err := ies.loadMappedPolicies(regularUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
@@ -498,12 +535,12 @@ func (ies *IAMEtcdStore) savePolicyDoc(policyName string, p iampolicy.Policy) er
 	return ies.saveIAMConfig(&p, getPolicyDocPath(policyName))
 }
 
-func (ies *IAMEtcdStore) saveMappedPolicy(name string, isSTS, isGroup bool, mp MappedPolicy) error {
-	return ies.saveIAMConfig(mp, getMappedPolicyPath(name, isSTS, isGroup))
+func (ies *IAMEtcdStore) saveMappedPolicy(name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error {
+	return ies.saveIAMConfig(mp, getMappedPolicyPath(name, userType, isGroup))
 }
 
-func (ies *IAMEtcdStore) saveUserIdentity(name string, isSTS bool, u UserIdentity) error {
-	return ies.saveIAMConfig(u, getUserIdentityPath(name, isSTS))
+func (ies *IAMEtcdStore) saveUserIdentity(name string, userType IAMUserType, u UserIdentity) error {
+	return ies.saveIAMConfig(u, getUserIdentityPath(name, userType))
 }
 
 func (ies *IAMEtcdStore) saveGroupInfo(name string, gi GroupInfo) error {
@@ -518,16 +555,16 @@ func (ies *IAMEtcdStore) deletePolicyDoc(name string) error {
 	return err
 }
 
-func (ies *IAMEtcdStore) deleteMappedPolicy(name string, isSTS, isGroup bool) error {
-	err := ies.deleteIAMConfig(getMappedPolicyPath(name, isSTS, isGroup))
+func (ies *IAMEtcdStore) deleteMappedPolicy(name string, userType IAMUserType, isGroup bool) error {
+	err := ies.deleteIAMConfig(getMappedPolicyPath(name, userType, isGroup))
 	if err == errConfigNotFound {
 		err = errNoSuchPolicy
 	}
 	return err
 }
 
-func (ies *IAMEtcdStore) deleteUserIdentity(name string, isSTS bool) error {
-	err := ies.deleteIAMConfig(getUserIdentityPath(name, isSTS))
+func (ies *IAMEtcdStore) deleteUserIdentity(name string, userType IAMUserType) error {
+	err := ies.deleteIAMConfig(getUserIdentityPath(name, userType))
 	if err == errConfigNotFound {
 		err = errNoSuchUser
 	}
@@ -599,11 +636,11 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 		case usersPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigUsersPrefix))
-			ies.loadUser(accessKey, false, sys.iamUsersMap)
+			ies.loadUser(accessKey, regularUser, sys.iamUsersMap)
 		case stsPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigSTSPrefix))
-			ies.loadUser(accessKey, true, sys.iamUsersMap)
+			ies.loadUser(accessKey, stsUser, sys.iamUsersMap)
 		case groupsPrefix:
 			group := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigGroupsPrefix))
@@ -619,17 +656,17 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBUsersPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(user, false, false, sys.iamUserPolicyMap)
+			ies.loadMappedPolicy(user, regularUser, false, sys.iamUserPolicyMap)
 		case policyDBSTSUsersPrefix:
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBSTSUsersPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(user, true, false, sys.iamUserPolicyMap)
+			ies.loadMappedPolicy(user, stsUser, false, sys.iamUserPolicyMap)
 		case policyDBGroupsPrefix:
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBGroupsPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(user, false, true, sys.iamGroupPolicyMap)
+			ies.loadMappedPolicy(user, regularUser, true, sys.iamGroupPolicyMap)
 		}
 	case eventDelete:
 		switch {

cmd/iam-object-store.go

@@ -25,6 +25,8 @@ import (
 	"sync"
 	"time"
 
+	jwtgo "github.com/dgrijalva/jwt-go"
+
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -116,7 +118,11 @@ func (iamOS *IAMObjectStore) migrateUsersConfigToV1(isSTS bool) error {
 
 			// 2. copy policy file to new location.
 			mp := newMappedPolicy(policyName)
-			if err := iamOS.saveMappedPolicy(user, isSTS, false, mp); err != nil {
+			userType := regularUser
+			if isSTS {
+				userType = stsUser
+			}
+			if err := iamOS.saveMappedPolicy(user, userType, false, mp); err != nil {
 				return err
 			}
 
@@ -289,14 +295,14 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(m map[string]iampolicy.Policy) error
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUser(user string, isSTS bool, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error {
 	objectAPI := iamOS.getObjectAPI()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
 	var u UserIdentity
-	err := iamOS.loadIAMConfig(&u, getUserIdentityPath(user, isSTS))
+	err := iamOS.loadIAMConfig(&u, getUserIdentityPath(user, userType))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchUser
@@ -306,11 +312,31 @@ func (iamOS *IAMObjectStore) loadUser(user string, isSTS bool, m map[string]auth
 
 	if u.Credentials.IsExpired() {
 		// Delete expired identity - ignoring errors here.
-		iamOS.deleteIAMConfig(getUserIdentityPath(user, isSTS))
-		iamOS.deleteIAMConfig(getMappedPolicyPath(user, isSTS, false))
+		iamOS.deleteIAMConfig(getUserIdentityPath(user, userType))
+		iamOS.deleteIAMConfig(getMappedPolicyPath(user, userType, false))
 		return nil
 	}
 
+	// If this is a service account, rotate the session key if needed
+	if globalOldCred.IsValid() && u.Credentials.IsServiceAccount() {
+		if !globalOldCred.Equal(globalActiveCred) {
+			m := jwtgo.MapClaims{}
+			stsTokenCallback := func(t *jwtgo.Token) (interface{}, error) {
+				return []byte(globalOldCred.SecretKey), nil
+			}
+			if _, err := jwtgo.ParseWithClaims(u.Credentials.SessionToken, m, stsTokenCallback); err == nil {
+				jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.MapClaims(m))
+				if token, err := jwt.SignedString([]byte(globalActiveCred.SecretKey)); err == nil {
+					u.Credentials.SessionToken = token
+					err := iamOS.saveIAMConfig(&u, getUserIdentityPath(user, userType))
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
@@ -318,7 +344,7 @@ func (iamOS *IAMObjectStore) loadUser(user string, isSTS bool, m map[string]auth
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUsers(isSTS bool, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUsers(userType IAMUserType, m map[string]auth.Credentials) error {
 	objectAPI := iamOS.getObjectAPI()
 	if objectAPI == nil {
 		return errServerNotInitialized
@@ -326,17 +352,24 @@ func (iamOS *IAMObjectStore) loadUsers(isSTS bool, m map[string]auth.Credentials
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	basePrefix := iamConfigUsersPrefix
-	if isSTS {
+
+	var basePrefix string
+	switch userType {
+	case srvAccUser:
+		basePrefix = iamConfigServiceAccountsPrefix
+	case stsUser:
 		basePrefix = iamConfigSTSPrefix
+	default:
+		basePrefix = iamConfigUsersPrefix
 	}
+
 	for item := range listIAMConfigItems(objectAPI, basePrefix, true, doneCh) {
 		if item.Err != nil {
 			return item.Err
 		}
 
 		userName := item.Item
-		err := iamOS.loadUser(userName, isSTS, m)
+		err := iamOS.loadUser(userName, userType, m)
 		if err != nil {
 			return err
 		}
@@ -384,7 +417,7 @@ func (iamOS *IAMObjectStore) loadGroups(m map[string]GroupInfo) error {
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicy(name string, isSTS, isGroup bool,
+func (iamOS *IAMObjectStore) loadMappedPolicy(name string, userType IAMUserType, isGroup bool,
 	m map[string]MappedPolicy) error {
 
 	objectAPI := iamOS.getObjectAPI()
@@ -393,7 +426,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicy(name string, isSTS, isGroup bool,
 	}
 
 	var p MappedPolicy
-	err := iamOS.loadIAMConfig(&p, getMappedPolicyPath(name, isSTS, isGroup))
+	err := iamOS.loadIAMConfig(&p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
@@ -404,7 +437,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicy(name string, isSTS, isGroup bool,
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicies(isSTS, isGroup bool, m map[string]MappedPolicy) error {
+func (iamOS *IAMObjectStore) loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
 	objectAPI := iamOS.getObjectAPI()
 	if objectAPI == nil {
 		return errServerNotInitialized
@@ -413,13 +446,17 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(isSTS, isGroup bool, m map[strin
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 	var basePath string
-	switch {
-	case isSTS:
-		basePath = iamConfigPolicyDBSTSUsersPrefix
-	case isGroup:
+	if isGroup {
 		basePath = iamConfigPolicyDBGroupsPrefix
-	default:
-		basePath = iamConfigPolicyDBUsersPrefix
+	} else {
+		switch userType {
+		case srvAccUser:
+			basePath = iamConfigPolicyDBServiceAccountsPrefix
+		case stsUser:
+			basePath = iamConfigPolicyDBSTSUsersPrefix
+		default:
+			basePath = iamConfigPolicyDBUsersPrefix
+		}
 	}
 	for item := range listIAMConfigItems(objectAPI, basePath, false, doneCh) {
 		if item.Err != nil {
@@ -428,7 +465,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(isSTS, isGroup bool, m map[strin
 
 		policyFile := item.Item
 		userOrGroupName := strings.TrimSuffix(policyFile, ".json")
-		err := iamOS.loadMappedPolicy(userOrGroupName, isSTS, isGroup, m)
+		err := iamOS.loadMappedPolicy(userOrGroupName, userType, isGroup, m)
 		if err != nil {
 			return err
 		}
@@ -466,27 +503,30 @@ func (iamOS *IAMObjectStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 		return err
 	}
 	// load STS temp users
-	if err := iamOS.loadUsers(true, iamUsersMap); err != nil {
+	if err := iamOS.loadUsers(stsUser, iamUsersMap); err != nil {
 		return err
 	}
 	if isMinIOUsersSys {
-		if err := iamOS.loadUsers(false, iamUsersMap); err != nil {
+		if err := iamOS.loadUsers(regularUser, iamUsersMap); err != nil {
+			return err
+		}
+		if err := iamOS.loadUsers(srvAccUser, iamUsersMap); err != nil {
 			return err
 		}
 		if err := iamOS.loadGroups(iamGroupsMap); err != nil {
 			return err
 		}
 
-		if err := iamOS.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
+		if err := iamOS.loadMappedPolicies(regularUser, false, iamUserPolicyMap); err != nil {
 			return err
 		}
 	}
 	// load STS policy mappings
-	if err := iamOS.loadMappedPolicies(true, false, iamUserPolicyMap); err != nil {
+	if err := iamOS.loadMappedPolicies(stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 	// load policies mapped to groups
-	if err := iamOS.loadMappedPolicies(false, true, iamGroupPolicyMap); err != nil {
+	if err := iamOS.loadMappedPolicies(regularUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
@@ -510,12 +550,12 @@ func (iamOS *IAMObjectStore) savePolicyDoc(policyName string, p iampolicy.Policy
 	return iamOS.saveIAMConfig(&p, getPolicyDocPath(policyName))
 }
 
-func (iamOS *IAMObjectStore) saveMappedPolicy(name string, isSTS, isGroup bool, mp MappedPolicy) error {
-	return iamOS.saveIAMConfig(mp, getMappedPolicyPath(name, isSTS, isGroup))
+func (iamOS *IAMObjectStore) saveMappedPolicy(name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error {
+	return iamOS.saveIAMConfig(mp, getMappedPolicyPath(name, userType, isGroup))
 }
 
-func (iamOS *IAMObjectStore) saveUserIdentity(name string, isSTS bool, u UserIdentity) error {
-	return iamOS.saveIAMConfig(u, getUserIdentityPath(name, isSTS))
+func (iamOS *IAMObjectStore) saveUserIdentity(name string, userType IAMUserType, u UserIdentity) error {
+	return iamOS.saveIAMConfig(u, getUserIdentityPath(name, userType))
 }
 
 func (iamOS *IAMObjectStore) saveGroupInfo(name string, gi GroupInfo) error {
@@ -530,16 +570,16 @@ func (iamOS *IAMObjectStore) deletePolicyDoc(name string) error {
 	return err
 }
 
-func (iamOS *IAMObjectStore) deleteMappedPolicy(name string, isSTS, isGroup bool) error {
-	err := iamOS.deleteIAMConfig(getMappedPolicyPath(name, isSTS, isGroup))
+func (iamOS *IAMObjectStore) deleteMappedPolicy(name string, userType IAMUserType, isGroup bool) error {
+	err := iamOS.deleteIAMConfig(getMappedPolicyPath(name, userType, isGroup))
 	if err == errConfigNotFound {
 		err = errNoSuchPolicy
 	}
 	return err
 }
 
-func (iamOS *IAMObjectStore) deleteUserIdentity(name string, isSTS bool) error {
-	err := iamOS.deleteIAMConfig(getUserIdentityPath(name, isSTS))
+func (iamOS *IAMObjectStore) deleteUserIdentity(name string, userType IAMUserType) error {
+	err := iamOS.deleteIAMConfig(getUserIdentityPath(name, userType))
 	if err == errConfigNotFound {
 		err = errNoSuchUser
 	}
@@ -624,14 +664,15 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 }
 
 func (iamOS *IAMObjectStore) watch(sys *IAMSys) {
+	ctx := GlobalContext
 	watchDisk := func() {
 		for {
 			select {
-			case <-GlobalServiceDoneCh:
+			case <-ctx.Done():
 				return
 			case <-time.NewTimer(globalRefreshIAMInterval).C:
 				err := iamOS.loadAll(sys, nil)
-				logger.LogIf(context.Background(), err)
+				logger.LogIf(ctx, err)
 			}
 		}
 	}

cmd/iam.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"sync"
 
 	"github.com/minio/minio-go/v6/pkg/set"
@@ -52,6 +54,9 @@ const (
 	// IAM users directory.
 	iamConfigUsersPrefix = iamConfigPrefix + "/users/"
 
+	// IAM service accounts directory.
+	iamConfigServiceAccountsPrefix = iamConfigPrefix + "/service-accounts/"
+
 	// IAM groups directory.
 	iamConfigGroupsPrefix = iamConfigPrefix + "/groups/"
 
@@ -62,10 +67,11 @@ const (
 	iamConfigSTSPrefix = iamConfigPrefix + "/sts/"
 
 	// IAM Policy DB prefixes.
-	iamConfigPolicyDBPrefix         = iamConfigPrefix + "/policydb/"
-	iamConfigPolicyDBUsersPrefix    = iamConfigPolicyDBPrefix + "users/"
-	iamConfigPolicyDBSTSUsersPrefix = iamConfigPolicyDBPrefix + "sts-users/"
-	iamConfigPolicyDBGroupsPrefix   = iamConfigPolicyDBPrefix + "groups/"
+	iamConfigPolicyDBPrefix                = iamConfigPrefix + "/policydb/"
+	iamConfigPolicyDBUsersPrefix           = iamConfigPolicyDBPrefix + "users/"
+	iamConfigPolicyDBSTSUsersPrefix        = iamConfigPolicyDBPrefix + "sts-users/"
+	iamConfigPolicyDBServiceAccountsPrefix = iamConfigPolicyDBPrefix + "service-accounts/"
+	iamConfigPolicyDBGroupsPrefix          = iamConfigPolicyDBPrefix + "groups/"
 
 	// IAM identity file which captures identity credentials.
 	iamIdentityFile = "identity.json"
@@ -99,10 +105,15 @@ func getIAMFormatFilePath() string {
 	return iamConfigPrefix + SlashSeparator + iamFormatFile
 }
 
-func getUserIdentityPath(user string, isSTS bool) string {
-	basePath := iamConfigUsersPrefix
-	if isSTS {
+func getUserIdentityPath(user string, userType IAMUserType) string {
+	var basePath string
+	switch userType {
+	case srvAccUser:
+		basePath = iamConfigServiceAccountsPrefix
+	case stsUser:
 		basePath = iamConfigSTSPrefix
+	default:
+		basePath = iamConfigUsersPrefix
 	}
 	return pathJoin(basePath, user, iamIdentityFile)
 }
@@ -115,12 +126,15 @@ func getPolicyDocPath(name string) string {
 	return pathJoin(iamConfigPoliciesPrefix, name, iamPolicyFile)
 }
 
-func getMappedPolicyPath(name string, isSTS, isGroup bool) string {
-	switch {
-	case isSTS:
-		return pathJoin(iamConfigPolicyDBSTSUsersPrefix, name+".json")
-	case isGroup:
+func getMappedPolicyPath(name string, userType IAMUserType, isGroup bool) string {
+	if isGroup {
 		return pathJoin(iamConfigPolicyDBGroupsPrefix, name+".json")
+	}
+	switch userType {
+	case srvAccUser:
+		return pathJoin(iamConfigPolicyDBServiceAccountsPrefix, name+".json")
+	case stsUser:
+		return pathJoin(iamConfigPolicyDBSTSUsersPrefix, name+".json")
 	default:
 		return pathJoin(iamConfigPolicyDBUsersPrefix, name+".json")
 	}
@@ -180,6 +194,15 @@ type IAMSys struct {
 	store IAMStorageAPI
 }
 
+// IAMUserType represents a user type inside MinIO server
+type IAMUserType int
+
+const (
+	regularUser IAMUserType = iota
+	stsUser
+	srvAccUser
+)
+
 // IAMStorageAPI defines an interface for the IAM persistence layer
 type IAMStorageAPI interface {
 	migrateBackendFormat(ObjectLayer) error
@@ -187,14 +210,14 @@ type IAMStorageAPI interface {
 	loadPolicyDoc(policy string, m map[string]iampolicy.Policy) error
 	loadPolicyDocs(m map[string]iampolicy.Policy) error
 
-	loadUser(user string, isSTS bool, m map[string]auth.Credentials) error
-	loadUsers(isSTS bool, m map[string]auth.Credentials) error
+	loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error
+	loadUsers(userType IAMUserType, m map[string]auth.Credentials) error
 
 	loadGroup(group string, m map[string]GroupInfo) error
 	loadGroups(m map[string]GroupInfo) error
 
-	loadMappedPolicy(name string, isSTS, isGroup bool, m map[string]MappedPolicy) error
-	loadMappedPolicies(isSTS, isGroup bool, m map[string]MappedPolicy) error
+	loadMappedPolicy(name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
+	loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
 
 	loadAll(*IAMSys, ObjectLayer) error
 
@@ -203,13 +226,13 @@ type IAMStorageAPI interface {
 	deleteIAMConfig(path string) error
 
 	savePolicyDoc(policyName string, p iampolicy.Policy) error
-	saveMappedPolicy(name string, isSTS, isGroup bool, mp MappedPolicy) error
-	saveUserIdentity(name string, isSTS bool, u UserIdentity) error
+	saveMappedPolicy(name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error
+	saveUserIdentity(name string, userType IAMUserType, u UserIdentity) error
 	saveGroupInfo(group string, gi GroupInfo) error
 
 	deletePolicyDoc(policyName string) error
-	deleteMappedPolicy(name string, isSTS, isGroup bool) error
-	deleteUserIdentity(name string, isSTS bool) error
+	deleteMappedPolicy(name string, userType IAMUserType, isGroup bool) error
+	deleteUserIdentity(name string, userType IAMUserType) error
 	deleteGroupInfo(name string) error
 
 	watch(*IAMSys)
@@ -290,9 +313,9 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 	if globalEtcdClient == nil {
 		var err error
 		if isGroup {
-			err = sys.store.loadMappedPolicy(userOrGroup, false, isGroup, sys.iamGroupPolicyMap)
+			err = sys.store.loadMappedPolicy(userOrGroup, regularUser, isGroup, sys.iamGroupPolicyMap)
 		} else {
-			err = sys.store.loadMappedPolicy(userOrGroup, false, isGroup, sys.iamUserPolicyMap)
+			err = sys.store.loadMappedPolicy(userOrGroup, regularUser, isGroup, sys.iamUserPolicyMap)
 		}
 
 		// Ignore policy not mapped error
@@ -305,7 +328,7 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 }
 
 // LoadUser - reloads a specific user from backend disks or etcd.
-func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, isSTS bool) error {
+func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUserType) error {
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -314,11 +337,11 @@ func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, isSTS bool) er
 	}
 
 	if globalEtcdClient == nil {
-		err := sys.store.loadUser(accessKey, isSTS, sys.iamUsersMap)
+		err := sys.store.loadUser(accessKey, userType, sys.iamUsersMap)
 		if err != nil {
 			return err
 		}
-		err = sys.store.loadMappedPolicy(accessKey, isSTS, false, sys.iamUserPolicyMap)
+		err = sys.store.loadMappedPolicy(accessKey, userType, false, sys.iamUserPolicyMap)
 		// Ignore policy not mapped error
 		if err != nil && err != errConfigNotFound {
 			return err
@@ -360,8 +383,12 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	}
 
 	sys.store.watch(sys)
+	err := sys.store.loadAll(sys, objAPI)
 
-	return sys.store.loadAll(sys, objAPI)
+	// Invalidate the old cred after finishing IAM initialization
+	globalOldCred = auth.Credentials{}
+
+	return err
 }
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
@@ -393,7 +420,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 
 	// Delete user-policy mappings that will no longer apply
 	var usersToDel []string
-	var isUserSTS []bool
+	var usersType []IAMUserType
 	for u, mp := range sys.iamUserPolicyMap {
 		if mp.Policy == policyName {
 			usersToDel = append(usersToDel, u)
@@ -403,12 +430,15 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 				return errNoSuchUser
 			}
 			// User is from STS if the creds are temporary
-			isSTS := cr.IsTemp()
-			isUserSTS = append(isUserSTS, isSTS)
+			if cr.IsTemp() {
+				usersType = append(usersType, stsUser)
+			} else {
+				usersType = append(usersType, regularUser)
+			}
 		}
 	}
 	for i, u := range usersToDel {
-		sys.policyDBSet(u, "", isUserSTS[i], false)
+		sys.policyDBSet(u, "", usersType[i], false)
 	}
 
 	// Delete group-policy mappings that will no longer apply
@@ -419,7 +449,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 		}
 	}
 	for _, g := range groupsToDel {
-		sys.policyDBSet(g, "", false, true)
+		sys.policyDBSet(g, "", regularUser, true)
 	}
 
 	return err
@@ -523,8 +553,8 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
-	sys.store.deleteMappedPolicy(accessKey, false, false)
-	err := sys.store.deleteUserIdentity(accessKey, false)
+	sys.store.deleteMappedPolicy(accessKey, regularUser, false)
+	err := sys.store.deleteUserIdentity(accessKey, regularUser)
 	switch err.(type) {
 	case ObjectNotFound:
 		// ignore if user is already deleted.
@@ -534,6 +564,15 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	delete(sys.iamUsersMap, accessKey)
 	delete(sys.iamUserPolicyMap, accessKey)
 
+	for _, u := range sys.iamUsersMap {
+		if u.IsServiceAccount() {
+			if u.ParentUser == accessKey {
+				_ = sys.store.deleteUserIdentity(u.AccessKey, srvAccUser)
+				delete(sys.iamUsersMap, u.AccessKey)
+			}
+		}
+	}
+
 	return err
 }
 
@@ -565,7 +604,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 		}
 
 		mp := newMappedPolicy(policyName)
-		if err := sys.store.saveMappedPolicy(accessKey, true, false, mp); err != nil {
+		if err := sys.store.saveMappedPolicy(accessKey, stsUser, false, mp); err != nil {
 			return err
 		}
 
@@ -577,7 +616,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 	}
 
 	u := newUserIdentity(cred)
-	if err := sys.store.saveUserIdentity(accessKey, true, u); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, stsUser, u); err != nil {
 		return err
 	}
 
@@ -602,7 +641,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 	}
 
 	for k, v := range sys.iamUsersMap {
-		if !v.IsTemp() {
+		if !v.IsTemp() && !v.IsServiceAccount() {
 			users[k] = madmin.UserInfo{
 				PolicyName: sys.iamUserPolicyMap[k].Policy,
 				Status: func() madmin.AccountStatus {
@@ -636,6 +675,28 @@ func (sys *IAMSys) IsTempUser(name string) (bool, error) {
 	return creds.IsTemp(), nil
 }
 
+// IsServiceAccount - returns if given key is a service account
+func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil {
+		return false, "", errServerNotInitialized
+	}
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	creds, found := sys.iamUsersMap[name]
+	if !found {
+		return false, "", errNoSuchUser
+	}
+
+	if creds.IsServiceAccount() {
+		return true, creds.ParentUser, nil
+	}
+
+	return false, "", nil
+}
+
 // GetUserInfo - get info on a user.
 func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
@@ -717,7 +778,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		return errServerNotInitialized
 	}
 
-	if err := sys.store.saveUserIdentity(accessKey, false, uinfo); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, regularUser, uinfo); err != nil {
 		return err
 	}
 
@@ -725,6 +786,104 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 	return nil
 }
 
+// NewServiceAccount - create a new service account
+func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser, sessionPolicy string) (auth.Credentials, error) {
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	if len(sessionPolicy) > 16*1024 {
+		return auth.Credentials{}, fmt.Errorf("Session policy should not exceed 16*1024 characters")
+	}
+
+	policy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicy)))
+	if err != nil {
+		return auth.Credentials{}, err
+	}
+
+	// Version in policy must not be empty
+	if policy.Version == "" {
+		return auth.Credentials{}, fmt.Errorf("Invalid session policy version")
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	if sys.usersSysType != MinIOUsersSysType {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	if sys.store == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	if parentUser == globalActiveCred.AccessKey {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	cr, ok := sys.iamUsersMap[parentUser]
+	if !ok {
+		return auth.Credentials{}, errNoSuchUser
+	}
+
+	if cr.IsTemp() {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	m := make(map[string]interface{})
+	m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicy))
+	m[parentClaim] = parentUser
+	m[iamPolicyClaimName()] = "embedded-policy"
+
+	secret := globalActiveCred.SecretKey
+	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
+	if err != nil {
+		return auth.Credentials{}, err
+	}
+
+	cred.ParentUser = parentUser
+	u := newUserIdentity(cred)
+
+	if err := sys.store.saveUserIdentity(u.Credentials.AccessKey, srvAccUser, u); err != nil {
+		return auth.Credentials{}, err
+	}
+
+	sys.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+
+	return cred, nil
+}
+
+// GetServiceAccount - returns the credentials of the given service account
+func (sys *IAMSys) GetServiceAccount(ctx context.Context, serviceAccountAccessKey string) (auth.Credentials, error) {
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	if sys.usersSysType != MinIOUsersSysType {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	if sys.store == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	cr, ok := sys.iamUsersMap[serviceAccountAccessKey]
+	if !ok {
+		return auth.Credentials{}, errNoSuchUser
+	}
+
+	if !cr.IsServiceAccount() {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	return cr, nil
+}
+
 // SetUser - set user credentials and policy.
 func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
@@ -754,7 +913,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 		return errIAMActionNotAllowed
 	}
 
-	if err := sys.store.saveUserIdentity(accessKey, false, u); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, regularUser, u); err != nil {
 		return err
 	}
 
@@ -762,7 +921,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 
 	// Set policy if specified.
 	if uinfo.PolicyName != "" {
-		return sys.policyDBSet(accessKey, uinfo.PolicyName, false, false)
+		return sys.policyDBSet(accessKey, uinfo.PolicyName, regularUser, false)
 	}
 	return nil
 }
@@ -792,7 +951,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 	cred.SecretKey = secretKey
 	u := newUserIdentity(cred)
-	if err := sys.store.saveUserIdentity(accessKey, false, u); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, regularUser, u); err != nil {
 		return err
 	}
 
@@ -923,7 +1082,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 		// Remove the group from storage. First delete the
 		// mapped policy.
-		err := sys.store.deleteMappedPolicy(group, false, true)
+		err := sys.store.deleteMappedPolicy(group, regularUser, true)
 		// No-mapped-policy case is ignored.
 		if err != nil && err != errConfigNotFound {
 			return err
@@ -1068,12 +1227,12 @@ func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
 
 	// isSTS is always false when called via PolicyDBSet as policy
 	// is never set by an external API call for STS users.
-	return sys.policyDBSet(name, policy, false, isGroup)
+	return sys.policyDBSet(name, policy, regularUser, isGroup)
 }
 
 // policyDBSet - sets a policy for user in the policy db. Assumes that caller
 // has sys.Lock(). If policy == "", then policy mapping is removed.
-func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
+func (sys *IAMSys) policyDBSet(name, policy string, userType IAMUserType, isGroup bool) error {
 	if sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1099,7 +1258,7 @@ func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
 
 	// Handle policy mapping removal
 	if policy == "" {
-		if err := sys.store.deleteMappedPolicy(name, isSTS, isGroup); err != nil {
+		if err := sys.store.deleteMappedPolicy(name, userType, isGroup); err != nil {
 			return err
 		}
 		if !isGroup {
@@ -1112,7 +1271,7 @@ func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
 
 	// Handle policy mapping set/update
 	mp := newMappedPolicy(policy)
-	if err := sys.store.saveMappedPolicy(name, isSTS, isGroup, mp); err != nil {
+	if err := sys.store.saveMappedPolicy(name, userType, isGroup, mp); err != nil {
 		return err
 	}
 	if !isGroup {
@@ -1294,6 +1453,93 @@ func (sys *IAMSys) policyDBGet(name string, isGroup bool) ([]string, error) {
 	return result, nil
 }
 
+// IsAllowedServiceAccount - checks if the given service account is allowed to perform
+// actions. The permission of the parent user is checked first
+func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) bool {
+	// Now check if we have a subject claim
+	p, ok := args.Claims[parentClaim]
+	if ok {
+		parentInClaim, ok := p.(string)
+		if !ok {
+			// Reject malformed/malicious requests.
+			return false
+		}
+		// The parent claim in the session token should be equal
+		// to the parent detected in the backend
+		if parentInClaim != parent {
+			return false
+		}
+	}
+
+	// Check if the parent is allowed to perform this action, reject if not
+	parentUserPolicies, err := sys.PolicyDBGet(parent, false)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return false
+	}
+
+	if len(parentUserPolicies) == 0 {
+		return false
+	}
+
+	var availablePolicies []iampolicy.Policy
+
+	// Policies were found, evaluate all of them.
+	sys.RLock()
+	for _, pname := range parentUserPolicies {
+		p, found := sys.iamPolicyDocsMap[pname]
+		if found {
+			availablePolicies = append(availablePolicies, p)
+		}
+	}
+	sys.RUnlock()
+
+	if len(availablePolicies) == 0 {
+		return false
+	}
+
+	combinedPolicy := availablePolicies[0]
+	for i := 1; i < len(availablePolicies); i++ {
+		combinedPolicy.Statements = append(combinedPolicy.Statements,
+			availablePolicies[i].Statements...)
+	}
+
+	serviceAcc := args.AccountName
+	args.AccountName = parent
+	if !combinedPolicy.IsAllowed(args) {
+		return false
+	}
+	args.AccountName = serviceAcc
+
+	// Now check if we have a sessionPolicy.
+	spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
+	if ok {
+		spolicyStr, ok := spolicy.(string)
+		if !ok {
+			// Sub policy if set, should be a string reject
+			// malformed/malicious requests.
+			return false
+		}
+
+		// Check if policy is parseable.
+		subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+		if err != nil {
+			// Log any error in input session policy config.
+			logger.LogIf(context.Background(), err)
+			return false
+		}
+
+		// Policy without Version string value reject it.
+		if subPolicy.Version == "" {
+			return false
+		}
+
+		return subPolicy.IsAllowed(args)
+	}
+
+	return false
+}
+
 // IsAllowedSTS is meant for STS based temporary credentials,
 // which implements claims validation and verification other than
 // applying policies.
@@ -1444,6 +1690,17 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 		return sys.IsAllowedSTS(args)
 	}
 
+	// If the credential is for a service account, perform related check
+	ok, parentUser, err := sys.IsServiceAccount(args.AccountName)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return false
+	}
+	if ok {
+		return sys.IsAllowedServiceAccount(args, parentUser)
+	}
+
+	// Continue with the assumption of a regular user
 	policies, err := sys.PolicyDBGet(args.AccountName, false)
 	if err != nil {
 		logger.LogIf(context.Background(), err)

cmd/jwt/parser_test.go

@@ -27,7 +27,6 @@ import (
 	"time"
 
 	"github.com/dgrijalva/jwt-go"
-	jwtgo "github.com/dgrijalva/jwt-go"
 )
 
 var (
@@ -49,7 +48,7 @@ var jwtTestData = []struct {
 		"",
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -61,7 +60,7 @@ var jwtTestData = []struct {
 		"", // autogen
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 				"exp": float64(time.Now().Unix() - 100),
 			},
@@ -74,7 +73,7 @@ var jwtTestData = []struct {
 		"", // autogen
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 				"nbf": float64(time.Now().Unix() + 100),
 			},
@@ -87,7 +86,7 @@ var jwtTestData = []struct {
 		"", // autogen
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 				"nbf": float64(time.Now().Unix() + 100),
 				"exp": float64(time.Now().Unix() - 100),
@@ -101,7 +100,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.EhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -113,7 +112,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		nil,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -125,7 +124,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		emptyKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -137,7 +136,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		errorKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -149,7 +148,7 @@ var jwtTestData = []struct {
 		"",
 		defaultKeyFunc,
 		&StandardClaims{
-			StandardClaims: jwtgo.StandardClaims{
+			StandardClaims: jwt.StandardClaims{
 				ExpiresAt: time.Now().Add(time.Second * 10).Unix(),
 			},
 		},
@@ -160,7 +159,7 @@ var jwtTestData = []struct {
 
 func mapClaimsToken(claims *MapClaims) string {
 	claims.SetAccessKey("test")
-	j := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
+	j := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
 	tk, _ := j.SignedString([]byte("HelloSecret"))
 	return tk
 }
@@ -168,7 +167,7 @@ func mapClaimsToken(claims *MapClaims) string {
 func standardClaimsToken(claims *StandardClaims) string {
 	claims.AccessKey = "test"
 	claims.Subject = "test"
-	j := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
+	j := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
 	tk, _ := j.SignedString([]byte("HelloSecret"))
 	return tk
 }

cmd/lock-rest-server.go

@@ -225,8 +225,6 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 	return nlripMap
 }
 
-var lockMaintenanceTimeout = newDynamicTimeout(60*time.Second, time.Second)
-
 // lockMaintenance loops over locks that have been active for some time and checks back
 // with the original server whether it is still alive or not
 //
@@ -235,28 +233,19 @@ var lockMaintenanceTimeout = newDynamicTimeout(60*time.Second, time.Second)
 // - some network error (and server is up normally)
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
-func lockMaintenance(ctx context.Context, interval time.Duration, objAPI ObjectLayer) error {
-	// Lock to avoid concurrent lock maintenance loops
-	maintenanceLock := objAPI.NewNSLock(ctx, "system", "lock-maintenance-ops")
-	if err := maintenanceLock.GetLock(lockMaintenanceTimeout); err != nil {
-		return err
-	}
-	defer maintenanceLock.Unlock()
-
+func lockMaintenance(ctx context.Context, interval time.Duration) error {
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
 	for lendpoint, nlrips := range getLongLivedLocks(interval) {
+		nlripsMap := make(map[string]int, len(nlrips))
 		for _, nlrip := range nlrips {
 			// Locks are only held on first zone, make sure that
 			// we only look for ownership of locks from endpoints
 			// on first zone.
 			for _, endpoint := range globalEndpoints[0].Endpoints {
-				if endpoint.String() == lendpoint.String() {
-					continue
-				}
-
 				c := newLockAPI(endpoint)
 				if !c.IsOnline() {
+					nlripsMap[nlrip.name]++
 					continue
 				}
 
@@ -266,25 +255,37 @@ func lockMaintenance(ctx context.Context, interval time.Duration, objAPI ObjectL
 					UID:       nlrip.lri.UID,
 					Resources: []string{nlrip.name},
 				})
-
 				if err != nil {
+					nlripsMap[nlrip.name]++
 					c.Close()
 					continue
 				}
 
-				// For successful response, verify if lock was indeed active or stale.
-				if expired {
-					// The lock is no longer active at server that originated
-					// the lock, attempt to remove the lock.
-					globalLockServers[lendpoint].mutex.Lock()
-					// Purge the stale entry if it exists.
-					globalLockServers[lendpoint].removeEntryIfExists(nlrip)
-					globalLockServers[lendpoint].mutex.Unlock()
+				if !expired {
+					nlripsMap[nlrip.name]++
 				}
 
 				// Close the connection regardless of the call response.
 				c.Close()
 			}
+
+			// Read locks we assume quorum for be N/2 success
+			quorum := globalXLSetDriveCount / 2
+			if nlrip.lri.Writer {
+				// For write locks we need N/2+1 success
+				quorum = globalXLSetDriveCount/2 + 1
+			}
+
+			// less than the quorum, we have locks expired.
+			if nlripsMap[nlrip.name] < quorum {
+				// The lock is no longer active at server that originated
+				// the lock, attempt to remove the lock.
+				globalLockServers[lendpoint].mutex.Lock()
+				// Purge the stale entry if it exists.
+				globalLockServers[lendpoint].removeEntryIfExists(nlrip)
+				globalLockServers[lendpoint].mutex.Unlock()
+			}
+
 		}
 	}
 
@@ -293,12 +294,13 @@ func lockMaintenance(ctx context.Context, interval time.Duration, objAPI ObjectL
 
 // Start lock maintenance from all lock servers.
 func startLockMaintenance() {
-	var objAPI ObjectLayer
 	var ctx = context.Background()
 
 	// Wait until the object API is ready
+	// no need to start the lock maintenance
+	// if ObjectAPI is not initialized.
 	for {
-		objAPI = newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue
@@ -322,7 +324,7 @@ func startLockMaintenance() {
 			// "synchronous checks" between servers
 			duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
 			time.Sleep(duration)
-			if err := lockMaintenance(ctx, lockValidityCheckInterval, objAPI); err != nil {
+			if err := lockMaintenance(ctx, lockValidityCheckInterval); err != nil {
 				// Sleep right after an error.
 				duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
 				time.Sleep(duration)

cmd/logger/target/http/http.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
-	gohttp "net/http"
 	"strings"
 
 	xhttp "github.com/minio/minio/cmd/http"
@@ -41,7 +40,7 @@ type Target struct {
 	// User-Agent to be set on each log request sent to the `endpoint`
 	userAgent string
 	logKind   string
-	client    gohttp.Client
+	client    http.Client
 }
 
 func (h *Target) startHTTPLogger() {
@@ -54,7 +53,7 @@ func (h *Target) startHTTPLogger() {
 				continue
 			}
 
-			req, err := gohttp.NewRequest(http.MethodPost, h.endpoint, bytes.NewBuffer(logJSON))
+			req, err := http.NewRequest(http.MethodPost, h.endpoint, bytes.NewBuffer(logJSON))
 			if err != nil {
 				continue
 			}
@@ -78,12 +77,12 @@ func (h *Target) startHTTPLogger() {
 
 // New initializes a new logger target which
 // sends log over http to the specified endpoint
-func New(endpoint, userAgent, logKind string, transport *gohttp.Transport) *Target {
+func New(endpoint, userAgent, logKind string, transport *http.Transport) *Target {
 	h := Target{
 		endpoint:  endpoint,
 		userAgent: userAgent,
 		logKind:   strings.ToUpper(logKind),
-		client: gohttp.Client{
+		client: http.Client{
 			Transport: transport,
 		},
 		logCh: make(chan interface{}, 10000),

cmd/naughty-disk_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"io"
 	"sync"
 )
@@ -80,8 +81,8 @@ func (d *naughtyDisk) calcError() (err error) {
 func (d *naughtyDisk) SetDiskID(id string) {
 }
 
-func (d *naughtyDisk) CrawlAndGetDataUsage(endCh <-chan struct{}) (info DataUsageInfo, err error) {
-	return d.disk.CrawlAndGetDataUsage(endCh)
+func (d *naughtyDisk) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (info dataUsageCache, err error) {
+	return d.disk.CrawlAndGetDataUsage(ctx, cache)
 }
 
 func (d *naughtyDisk) DiskInfo() (info DiskInfo, err error) {
@@ -196,6 +197,13 @@ func (d *naughtyDisk) DeleteFileBulk(volume string, paths []string) ([]error, er
 	return errs, nil
 }
 
+func (d *naughtyDisk) DeletePrefixes(volume string, paths []string) ([]error, error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.DeletePrefixes(volume, paths)
+}
+
 func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) {
 	if err := d.calcError(); err != nil {
 		return err

cmd/notification.go

@@ -326,7 +326,7 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		for typ, data := range data {
 			// Send profiling data to zip as file
 			header, zerr := zip.FileInfoHeader(dummyFileInfo{
-				name:    fmt.Sprintf("profiling-%s-%s.pprof", client.host.String(), typ),
+				name:    fmt.Sprintf("profile-%s-%s", client.host.String(), typ),
 				size:    int64(len(data)),
 				mode:    0600,
 				modTime: UTCNow(),
@@ -757,8 +757,14 @@ func (sys *NotificationSys) ConfiguredTargetIDs() []event.TargetID {
 			}
 		}
 	}
-
-	return targetIDs
+	// Filter out targets configured via env
+	var tIDs []event.TargetID
+	for _, targetID := range targetIDs {
+		if !globalEnvTargetList.Exists(targetID) {
+			tIDs = append(tIDs, targetID)
+		}
+	}
+	return tIDs
 }
 
 // RemoveNotification - removes all notification configuration for bucket name.

cmd/object-api-common.go

@@ -147,78 +147,6 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 	return err
 }
 
-// Cleanup objects in bulk and recursively: each object will have a list of sub-files to delete in the backend
-func cleanupObjectsBulk(storage StorageAPI, volume string, objsPaths []string, errs []error) ([]error, error) {
-	// The list of files in disk to delete
-	var filesToDelete []string
-	// Map files to delete to the passed objsPaths
-	var filesToDeleteObjsIndexes []int
-
-	// Traverse and return the list of sub entries
-	var traverse func(string) ([]string, error)
-	traverse = func(entryPath string) ([]string, error) {
-		var output = make([]string, 0)
-		if !HasSuffix(entryPath, SlashSeparator) {
-			output = append(output, entryPath)
-			return output, nil
-		}
-		entries, err := storage.ListDir(volume, entryPath, -1, "")
-		if err != nil {
-			if err == errFileNotFound {
-				return nil, nil
-			}
-			return nil, err
-		}
-
-		for _, entry := range entries {
-			subEntries, err := traverse(pathJoin(entryPath, entry))
-			if err != nil {
-				return nil, err
-			}
-			output = append(output, subEntries...)
-		}
-		return output, nil
-	}
-
-	// Find and collect the list of files to remove associated
-	// to the passed objects paths
-	for idx, objPath := range objsPaths {
-		if errs[idx] != nil {
-			continue
-		}
-		output, err := traverse(retainSlash(pathJoin(objPath)))
-		if err != nil {
-			errs[idx] = err
-			continue
-		} else {
-			errs[idx] = nil
-		}
-		filesToDelete = append(filesToDelete, output...)
-		for i := 0; i < len(output); i++ {
-			filesToDeleteObjsIndexes = append(filesToDeleteObjsIndexes, idx)
-		}
-	}
-
-	// Reverse the list so remove can succeed
-	reverseStringSlice(filesToDelete)
-
-	dErrs, err := storage.DeleteFileBulk(volume, filesToDelete)
-	if err != nil {
-		return nil, err
-	}
-
-	// Map files deletion errors to the correspondent objects
-	for i := range dErrs {
-		if dErrs[i] != nil {
-			if errs[filesToDeleteObjsIndexes[i]] != nil {
-				errs[filesToDeleteObjsIndexes[i]] = dErrs[i]
-			}
-		}
-	}
-
-	return errs, nil
-}
-
 // Removes notification.xml for a given bucket, only used during DeleteBucket.
 func removeNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
 	// Verify bucket is valid.

cmd/object-api-datatypes.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"io"
+	"math"
 	"time"
 
 	"github.com/minio/minio/pkg/hash"
@@ -77,6 +78,11 @@ type objectHistogramInterval struct {
 	start, end int64
 }
 
+const (
+	// dataUsageBucketLen must be length of ObjectsHistogramIntervals
+	dataUsageBucketLen = 7
+)
+
 // ObjectsHistogramIntervals is the list of all intervals
 // of object sizes to be included in objects histogram.
 var ObjectsHistogramIntervals = []objectHistogramInterval{
@@ -86,20 +92,25 @@ var ObjectsHistogramIntervals = []objectHistogramInterval{
 	{"BETWEEN_10_MB_AND_64_MB", 1024 * 1024 * 10, 1024*1024*64 - 1},
 	{"BETWEEN_64_MB_AND_128_MB", 1024 * 1024 * 64, 1024*1024*128 - 1},
 	{"BETWEEN_128_MB_AND_512_MB", 1024 * 1024 * 128, 1024*1024*512 - 1},
-	{"GREATER_THAN_512_MB", 1024 * 1024 * 512, -1},
+	{"GREATER_THAN_512_MB", 1024 * 1024 * 512, math.MaxInt64},
 }
 
 // DataUsageInfo represents data usage stats of the underlying Object API
 type DataUsageInfo struct {
-	// The timestamp of when the data usage info is generated
+	// LastUpdate is the timestamp of when the data usage info was last updated.
+	// This does not indicate a full scan.
 	LastUpdate time.Time `json:"lastUpdate"`
 
 	ObjectsCount uint64 `json:"objectsCount"`
 	// Objects total size
-	ObjectsTotalSize      uint64            `json:"objectsTotalSize"`
+	ObjectsTotalSize uint64 `json:"objectsTotalSize"`
+
+	// ObjectsSizesHistogram contains information on objects across all buckets.
+	// See ObjectsHistogramIntervals.
 	ObjectsSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
 
-	BucketsCount uint64            `json:"bucketsCount"`
+	BucketsCount uint64 `json:"bucketsCount"`
+	// BucketsSizes is "bucket name" -> size.
 	BucketsSizes map[string]uint64 `json:"bucketsSizes"`
 }
 

cmd/object-api-errors.go

@@ -274,7 +274,7 @@ func (e BucketSSEConfigNotFound) Error() string {
 // BucketNameInvalid - bucketname provided is invalid.
 type BucketNameInvalid GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e BucketNameInvalid) Error() string {
 	return "Bucket name invalid: " + e.Bucket
 }
@@ -290,17 +290,17 @@ type ObjectNameTooLong GenericError
 // ObjectNamePrefixAsSlash - object name has a slash as prefix.
 type ObjectNamePrefixAsSlash GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e ObjectNameInvalid) Error() string {
 	return "Object name invalid: " + e.Bucket + "#" + e.Object
 }
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e ObjectNameTooLong) Error() string {
 	return "Object name too long: " + e.Bucket + "#" + e.Object
 }
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e ObjectNamePrefixAsSlash) Error() string {
 	return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
 }
@@ -308,7 +308,7 @@ func (e ObjectNamePrefixAsSlash) Error() string {
 // AllAccessDisabled All access to this object has been disabled
 type AllAccessDisabled GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e AllAccessDisabled) Error() string {
 	return "All access to this object has been disabled"
 }
@@ -316,7 +316,7 @@ func (e AllAccessDisabled) Error() string {
 // IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header.
 type IncompleteBody GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e IncompleteBody) Error() string {
 	return e.Bucket + "#" + e.Object + "has incomplete body"
 }

cmd/object-api-interface.go

@@ -59,7 +59,7 @@ type ObjectLayer interface {
 
 	// Storage operations.
 	Shutdown(context.Context) error
-	CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo
+	CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error
 	StorageInfo(ctx context.Context, local bool) StorageInfo // local queries only local disks
 
 	// Bucket operations.
@@ -101,9 +101,8 @@ type ObjectLayer interface {
 	ReloadFormat(ctx context.Context, dryRun bool) error
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)
 	HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error)
-	HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (madmin.HealResultItem, error)
-
-	HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) error
+	HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error)
+	HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) error
 
 	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
 

cmd/object-api-listobjects_test.go

@@ -45,6 +45,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		// Will not store any objects in this bucket,
 		// Its to test ListObjects on an empty bucket.
 		"empty-bucket",
+		// Listing the case where the marker > last object.
+		"test-bucket-single-object",
 	}
 	for _, bucket := range testBuckets {
 		err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
@@ -72,6 +74,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		{testBuckets[1], "obj1", "obj1", nil},
 		{testBuckets[1], "obj2", "obj2", nil},
 		{testBuckets[1], "temporary/0/", "", nil},
+		{testBuckets[3], "A/B", "contentstring", nil},
 	}
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
@@ -445,6 +448,11 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 				{Name: "temporary/0/"},
 			},
 		},
+		// ListObjectsResult-34 Listing with marker > last object should return empty
+		{
+			IsTruncated: false,
+			Objects:     []ObjectInfo{},
+		},
 	}
 
 	testCases := []struct {
@@ -559,6 +567,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		{"test-bucket-empty-dir", "", "", SlashSeparator, 10, resultCases[32], nil, true},
 		// Test listing a directory which contains an empty directory (64)
 		{"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true},
+		// Test listing with marker > last object such that response should be empty (65)
+		{"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true},
 	}
 
 	for i, testCase := range testCases {

cmd/object-api-utils.go

@@ -51,10 +51,6 @@ import (
 const (
 	// MinIO meta bucket.
 	minioMetaBucket = ".minio.sys"
-	// Background ops meta prefix
-	backgroundOpsMetaPrefix = "background-ops"
-	// MinIO Stats meta prefix.
-	minioMetaBackgroundOpsBucket = minioMetaBucket + SlashSeparator + backgroundOpsMetaPrefix
 	// Multipart meta prefix.
 	mpartMetaPrefix = "multipart"
 	// MinIO Multipart meta prefix.
@@ -77,7 +73,7 @@ func isMinioMetaBucketName(bucket string) bool {
 	return bucket == minioMetaBucket ||
 		bucket == minioMetaMultipartBucket ||
 		bucket == minioMetaTmpBucket ||
-		bucket == minioMetaBackgroundOpsBucket
+		bucket == dataUsageBucket
 }
 
 // IsValidBucketName verifies that a bucket name is in accordance with

cmd/object-handlers.go

@@ -1214,7 +1214,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if tags := r.Header.Get(http.CanonicalHeaderKey(xhttp.AmzObjectTagging)); tags != "" {
+	if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
 		metadata[xhttp.AmzObjectTagging], err = extractTags(ctx, tags)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -2981,5 +2981,5 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessNoContent(w)
 }

cmd/peer-rest-server.go

@@ -325,7 +325,12 @@ func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if err = globalIAMSys.LoadUser(objAPI, accessKey, temp); err != nil {
+	var userType = regularUser
+	if temp {
+		userType = stsUser
+	}
+
+	if err = globalIAMSys.LoadUser(objAPI, accessKey, userType); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}

cmd/posix-diskid-check.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"io"
 )
 
@@ -38,8 +39,8 @@ func (p *posixDiskIDCheck) IsOnline() bool {
 	return storedDiskID == p.diskID
 }
 
-func (p *posixDiskIDCheck) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
-	return p.storage.CrawlAndGetDataUsage(endCh)
+func (p *posixDiskIDCheck) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
+	return p.storage.CrawlAndGetDataUsage(ctx, cache)
 }
 
 func (p *posixDiskIDCheck) Hostname() string {
@@ -179,6 +180,13 @@ func (p *posixDiskIDCheck) DeleteFileBulk(volume string, paths []string) (errs [
 	return p.storage.DeleteFileBulk(volume, paths)
 }
 
+func (p *posixDiskIDCheck) DeletePrefixes(volume string, paths []string) (errs []error, err error) {
+	if p.isDiskStale() {
+		return nil, errDiskNotFound
+	}
+	return p.storage.DeletePrefixes(volume, paths)
+}
+
 func (p *posixDiskIDCheck) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	if p.isDiskStale() {
 		return errDiskNotFound

cmd/posix.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -44,7 +44,6 @@ import (
 	"github.com/minio/minio/pkg/disk"
 	xioutil "github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/mountinfo"
-	"github.com/ncw/directio"
 )
 
 const (
@@ -213,7 +212,7 @@ func newPosix(path string) (*posix, error) {
 		diskPath: path,
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := directio.AlignedBlock(readBlockSize)
+				b := disk.AlignedBlock(readBlockSize)
 				return &b
 			},
 		},
@@ -339,8 +338,8 @@ func (s *posix) waitForLowActiveIO() {
 	}
 }
 
-func (s *posix) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
-	dataUsageInfo := updateUsage(s.diskPath, endCh, s.waitForLowActiveIO, func(item Item) (int64, error) {
+func (s *posix) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
+	dataUsageInfo, err := updateUsage(ctx, s.diskPath, cache, s.waitForLowActiveIO, func(item Item) (int64, error) {
 		// Look for `xl.json' at the leaf.
 		if !strings.HasSuffix(item.Path, SlashSeparator+xlMetaJSONFile) {
 			// if no xl.json found, skip the file.
@@ -354,14 +353,20 @@ func (s *posix) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, erro
 
 		meta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaBuf)
 		if err != nil {
-			return 0, errSkipFile
+			return 0, nil
 		}
 
 		return meta.Stat.Size, nil
 	})
-
-	dataUsageInfo.LastUpdate = UTCNow()
-	atomic.StoreUint64(&s.totalUsed, dataUsageInfo.ObjectsTotalSize)
+	if err != nil {
+		return dataUsageInfo, err
+	}
+	dataUsageInfo.Info.LastUpdate = time.Now()
+	total := dataUsageInfo.sizeRecursive(dataUsageInfo.Info.Name)
+	if total == nil {
+		total = &dataUsageEntry{}
+	}
+	atomic.StoreUint64(&s.totalUsed, uint64(total.Size))
 	return dataUsageInfo, nil
 }
 
@@ -670,13 +675,16 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
 	ch = make(chan FileInfo)
 	go func() {
 		defer close(ch)
-		listDir := func(volume, dirPath, dirEntry string) (entries []string) {
+		listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) {
 			entries, err := s.ListDir(volume, dirPath, -1, leafFile)
 			if err != nil {
 				return
 			}
+			if len(entries) == 0 {
+				return true, nil
+			}
 			sort.Strings(entries)
-			return filterMatchingPrefix(entries, dirEntry)
+			return false, filterMatchingPrefix(entries, dirEntry)
 		}
 
 		walkResultCh := startTreeWalk(context.Background(), volume, dirPath, marker, recursive, listDir, endWalkCh)
@@ -1265,16 +1273,28 @@ func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 	}, nil
 }
 
-// deleteFile deletes a file path if its empty. If it's successfully deleted,
-// it will recursively move up the tree, deleting empty parent directories
-// until it finds one with files in it. Returns nil for a non-empty directory.
-func deleteFile(basePath, deletePath string) error {
-	if basePath == deletePath {
+// deleteFile deletes a file or a directory if its empty unless recursive
+// is set to true. If the target is successfully deleted, it will recursively
+// move up the tree, deleting empty parent directories until it finds one
+// with files in it. Returns nil for a non-empty directory even when
+// recursive is set to false.
+func deleteFile(basePath, deletePath string, recursive bool) error {
+	if basePath == "" || deletePath == "" {
+		return nil
+	}
+	basePath = filepath.Clean(basePath)
+	deletePath = filepath.Clean(deletePath)
+	if !strings.HasPrefix(deletePath, basePath) || deletePath == basePath {
 		return nil
 	}
 
-	// Attempt to remove path.
-	if err := os.Remove((deletePath)); err != nil {
+	var err error
+	if recursive {
+		err = os.RemoveAll(deletePath)
+	} else {
+		err = os.Remove(deletePath)
+	}
+	if err != nil {
 		switch {
 		case isSysErrNotEmpty(err):
 			// Ignore errors if the directory is not empty. The server relies on
@@ -1297,12 +1317,58 @@ func deleteFile(basePath, deletePath string) error {
 	deletePath = strings.TrimSuffix(deletePath, SlashSeparator)
 	deletePath = slashpath.Dir(deletePath)
 
-	// Delete parent directory. Errors for parent directories shouldn't trickle down.
-	deleteFile(basePath, deletePath)
+	// Delete parent directory obviously not recursively. Errors for
+	// parent directories shouldn't trickle down.
+	deleteFile(basePath, deletePath, false)
 
 	return nil
 }
 
+// DeletePrefixes forcibly deletes all the contents of a set of specified paths.
+// Parent directories are automatically removed if they become empty. err can
+// bil nil while errs can contain some errors for corresponding objects. No error
+// is set if a specified prefix path does not exist.
+func (s *posix) DeletePrefixes(volume string, paths []string) (errs []error, err error) {
+	atomic.AddInt32(&s.activeIOCount, 1)
+	defer func() {
+		atomic.AddInt32(&s.activeIOCount, -1)
+	}()
+
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return nil, err
+	}
+
+	// Stat a volume entry.
+	_, err = os.Stat(volumeDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, errVolumeNotFound
+		} else if os.IsPermission(err) {
+			return nil, errVolumeAccessDenied
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		}
+		return nil, err
+	}
+
+	errs = make([]error, len(paths))
+	// Following code is needed so that we retain SlashSeparator
+	// suffix if any in path argument.
+	for idx, path := range paths {
+		filePath := pathJoin(volumeDir, path)
+		errs[idx] = checkPathLength(filePath)
+		if errs[idx] != nil {
+			continue
+		}
+		// Delete file or a directory recursively, delete parent
+		// directory as well if its empty.
+		errs[idx] = deleteFile(volumeDir, filePath, true)
+	}
+
+	return
+}
+
 // DeleteFile - delete a file at path.
 func (s *posix) DeleteFile(volume, path string) (err error) {
 	atomic.AddInt32(&s.activeIOCount, 1)
@@ -1316,7 +1382,7 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 	}
 
 	// Stat a volume entry.
-	_, err = os.Stat((volumeDir))
+	_, err = os.Stat(volumeDir)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
@@ -1331,12 +1397,12 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 	// Following code is needed so that we retain SlashSeparator suffix if any in
 	// path argument.
 	filePath := pathJoin(volumeDir, path)
-	if err = checkPathLength((filePath)); err != nil {
+	if err = checkPathLength(filePath); err != nil {
 		return err
 	}
 
 	// Delete file and delete parent directory as well if its empty.
-	return deleteFile(volumeDir, filePath)
+	return deleteFile(volumeDir, filePath, false)
 }
 
 func (s *posix) DeleteFileBulk(volume string, paths []string) (errs []error, err error) {
@@ -1373,7 +1439,7 @@ func (s *posix) DeleteFileBulk(volume string, paths []string) (errs []error, err
 			continue
 		}
 		// Delete file and delete parent directory as well if its empty.
-		errs[idx] = deleteFile(volumeDir, filePath)
+		errs[idx] = deleteFile(volumeDir, filePath, false)
 	}
 	return
 }
@@ -1460,7 +1526,7 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 
 	// Remove parent dir of the source file if empty
 	if parentDir := slashpath.Dir(srcFilePath); isDirEmpty(parentDir) {
-		deleteFile(srcVolumeDir, parentDir)
+		deleteFile(srcVolumeDir, parentDir, false)
 	}
 
 	return nil

cmd/server-main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -361,6 +362,9 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	httpServer := xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{handler}, getCert)
+	httpServer.BaseContext = func(listener net.Listener) context.Context {
+		return GlobalContext
+	}
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start()
 	}()
@@ -441,9 +445,6 @@ func serverMain(ctx *cli.Context) {
 		logger.StartupMessage(color.RedBold(msg))
 	}
 
-	// Set uptime time after object layer has initialized.
-	globalBootTime = UTCNow()
-
 	handleSignals()
 }
 

cmd/server-startup-msg.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/logger"
 	color "github.com/minio/minio/pkg/color"
 	xnet "github.com/minio/minio/pkg/net"
 )
@@ -91,13 +92,13 @@ func printStartupSafeModeMessage(apiEndpoints []string, err error) {
 			mcMessage := fmt.Sprintf("> mc.exe config host add %s %s %s %s --api s3v4", alias,
 				endPoint, cred.AccessKey, cred.SecretKey)
 			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-			mcMessage = fmt.Sprintf("> mc.exe admin config --help")
+			mcMessage = "> mc.exe admin config --help"
 			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		} else {
 			mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s --api s3v4", alias,
 				endPoint, cred.AccessKey, cred.SecretKey)
 			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-			mcMessage = fmt.Sprintf("$ mc admin config --help")
+			mcMessage = "$ mc admin config --help"
 			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		}
 	}
@@ -249,8 +250,12 @@ func printObjectAPIMsg() {
 // Get formatted disk/storage info message.
 func getStorageInfoMsgSafeMode(storageInfo StorageInfo) string {
 	var msg string
+	var mcMessage string
 	if storageInfo.Backend.Type == BackendErasure {
-		diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum())
+		if storageInfo.Backend.OfflineDisks.Sum() > 0 {
+			mcMessage = "Run `mc admin info` to look for server/disk info`"
+		}
+		diskInfo := fmt.Sprintf(" %d Online, %d Offline.%s ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum(), mcMessage)
 		msg += color.Red("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo)
 	}
 	return msg
@@ -259,8 +264,12 @@ func getStorageInfoMsgSafeMode(storageInfo StorageInfo) string {
 // Get formatted disk/storage info message.
 func getStorageInfoMsg(storageInfo StorageInfo) string {
 	var msg string
+	var mcMessage string
 	if storageInfo.Backend.Type == BackendErasure {
-		diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum())
+		if storageInfo.Backend.OfflineDisks.Sum() > 0 {
+			mcMessage = "Use `mc admin info` to look for server/disk info"
+		}
+		diskInfo := fmt.Sprintf(" %d Online, %d Offline.%s ", storageInfo.Backend.OnlineDisks.Sum(), storageInfo.Backend.OfflineDisks.Sum(), mcMessage)
 		msg += color.Blue("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo)
 	}
 	return msg
@@ -269,6 +278,9 @@ func getStorageInfoMsg(storageInfo StorageInfo) string {
 // Prints startup message of storage capacity and erasure information.
 func printStorageInfo(storageInfo StorageInfo) {
 	if msg := getStorageInfoMsg(storageInfo); msg != "" {
+		if globalCLIContext.Quiet {
+			logger.Info(msg)
+		}
 		logStartupMessage(msg)
 	}
 }

cmd/service.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"os"
 	"os/exec"
 	"syscall"
@@ -35,11 +36,22 @@ const (
 var globalServiceSignalCh chan serviceSignal
 
 // GlobalServiceDoneCh - Global service done channel.
-var GlobalServiceDoneCh chan struct{}
+var GlobalServiceDoneCh <-chan struct{}
+
+// GlobalContext context that is canceled when server is requested to shut down.
+var GlobalContext context.Context
+
+// cancelGlobalContext can be used to indicate server shutdown.
+var cancelGlobalContext context.CancelFunc
 
 // Initialize service mutex once.
 func init() {
-	GlobalServiceDoneCh = make(chan struct{})
+	initGlobalContext()
+}
+
+func initGlobalContext() {
+	GlobalContext, cancelGlobalContext = context.WithCancel(context.Background())
+	GlobalServiceDoneCh = GlobalContext.Done()
 	globalServiceSignalCh = make(chan serviceSignal)
 }
 

cmd/signals.go

@@ -57,7 +57,7 @@ func handleSignals() {
 		}
 
 		// send signal to various go-routines that they need to quit.
-		close(GlobalServiceDoneCh)
+		cancelGlobalContext()
 
 		if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
 			oerr = objAPI.Shutdown(context.Background())

cmd/signature-v4-parser.go

@@ -108,7 +108,11 @@ func parseCredentialHeader(credElement string, region string, stype serviceType)
 
 	}
 	if credElements[2] != string(stype) {
-		return ch, ErrInvalidService
+		switch stype {
+		case serviceSTS:
+			return ch, ErrInvalidServiceSTS
+		}
+		return ch, ErrInvalidServiceS3
 	}
 	cred.scope.service = credElements[2]
 	if credElements[3] != "aws4_request" {

cmd/signature-v4-parser_test.go

@@ -151,7 +151,7 @@ func TestParseCredentialHeader(t *testing.T) {
 				"ABCD",
 				"ABCD"),
 			expectedCredentials: credentialHeader{},
-			expectedErrCode:     ErrInvalidService,
+			expectedErrCode:     ErrInvalidServiceS3,
 		},
 		// Test Case - 7.
 		// Test case with invalid region.

cmd/storage-interface.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"io"
 )
 
@@ -32,7 +33,7 @@ type StorageAPI interface {
 	SetDiskID(id string)
 
 	DiskInfo() (info DiskInfo, err error)
-	CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error)
+	CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error)
 
 	// Volume operations.
 	MakeVol(volume string) (err error)
@@ -55,6 +56,7 @@ type StorageAPI interface {
 	StatFile(volume string, path string) (file FileInfo, err error)
 	DeleteFile(volume string, path string) (err error)
 	DeleteFileBulk(volume string, paths []string) (errs []error, err error)
+	DeletePrefixes(volume string, paths []string) (errs []error, err error)
 	VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error
 
 	// Write all data, syncs the data to disk.

cmd/storage-rest-client.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"bufio"
 	"bytes"
 	"context"
 	"crypto/tls"
@@ -152,26 +151,25 @@ func (client *storageRESTClient) Hostname() string {
 	return client.endpoint.Host
 }
 
-func (client *storageRESTClient) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
-	respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage, nil, nil, -1)
+func (client *storageRESTClient) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
+	b := cache.serialize()
+	respBody, err := client.call(storageRESTMethodCrawlAndGetDataUsage,
+		url.Values{},
+		bytes.NewBuffer(b), int64(len(b)))
 	defer http.DrainBody(respBody)
 	if err != nil {
-		return DataUsageInfo{}, err
+		return cache, err
 	}
-	reader := bufio.NewReader(respBody)
-	for {
-		b, err := reader.ReadByte()
-		if err != nil {
-			return DataUsageInfo{}, err
-		}
-		if b != ' ' {
-			reader.UnreadByte()
-			break
-		}
+	reader, err := waitForHTTPResponse(respBody)
+	if err != nil {
+		return cache, err
 	}
-	var usageInfo DataUsageInfo
-	err = gob.NewDecoder(reader).Decode(&usageInfo)
-	return usageInfo, err
+	b, err = ioutil.ReadAll(reader)
+	if err != nil {
+		return cache, err
+	}
+	var newCache dataUsageCache
+	return newCache, newCache.deserialize(b)
 }
 
 func (client *storageRESTClient) SetDiskID(id string) {
@@ -414,13 +412,54 @@ func (client *storageRESTClient) DeleteFileBulk(volume string, paths []string) (
 
 	respBody, err := client.call(storageRESTMethodDeleteFileBulk, values, &buffer, -1)
 	defer http.DrainBody(respBody)
+	if err != nil {
+		return nil, err
+	}
 
+	reader, err := waitForHTTPResponse(respBody)
 	if err != nil {
 		return nil, err
 	}
 
 	dErrResp := &DeleteFileBulkErrsResp{}
-	if err = gob.NewDecoder(respBody).Decode(dErrResp); err != nil {
+	if err = gob.NewDecoder(reader).Decode(dErrResp); err != nil {
+		return nil, err
+	}
+
+	for _, dErr := range dErrResp.Errs {
+		errs = append(errs, toStorageErr(dErr))
+	}
+
+	return errs, nil
+}
+
+// DeletePrefixes - deletes prefixes in bulk.
+func (client *storageRESTClient) DeletePrefixes(volume string, paths []string) (errs []error, err error) {
+	if len(paths) == 0 {
+		return errs, err
+	}
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+
+	var buffer bytes.Buffer
+	for _, path := range paths {
+		buffer.WriteString(path)
+		buffer.WriteString("\n")
+	}
+
+	respBody, err := client.call(storageRESTMethodDeletePrefixes, values, &buffer, -1)
+	defer http.DrainBody(respBody)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := waitForHTTPResponse(respBody)
+	if err != nil {
+		return nil, err
+	}
+
+	dErrResp := &DeletePrefixesErrsResp{}
+	if err = gob.NewDecoder(reader).Decode(dErrResp); err != nil {
 		return nil, err
 	}
 
@@ -457,16 +496,9 @@ func (client *storageRESTClient) VerifyFile(volume, path string, size int64, alg
 	if err != nil {
 		return err
 	}
-	reader := bufio.NewReader(respBody)
-	for {
-		b, err := reader.ReadByte()
-		if err != nil {
-			return err
-		}
-		if b != ' ' {
-			reader.UnreadByte()
-			break
-		}
+	reader, err := waitForHTTPResponse(respBody)
+	if err != nil {
+		return err
 	}
 	verifyResp := &VerifyFileResp{}
 	if err = gob.NewDecoder(reader).Decode(verifyResp); err != nil {

cmd/storage-rest-common.go

@@ -17,7 +17,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v14" // DeleteFileBulk API change
+	storageRESTVersion       = "v16" // CrawlAndGetDataUsageHandler API change
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )
@@ -42,6 +42,7 @@ const (
 	storageRESTMethodWalk           = "/walk"
 	storageRESTMethodDeleteFile     = "/deletefile"
 	storageRESTMethodDeleteFileBulk = "/deletefilebulk"
+	storageRESTMethodDeletePrefixes = "/deleteprefixes"
 	storageRESTMethodRenameFile     = "/renamefile"
 	storageRESTMethodVerifyFile     = "/verifyfile"
 )

cmd/storage-rest-server.go

@@ -24,6 +24,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os/user"
 	"path"
@@ -138,16 +139,28 @@ func (s *storageRESTServer) CrawlAndGetDataUsageHandler(w http.ResponseWriter, r
 	}
 
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
-	doneCh := sendWhiteSpaceToHTTPResponse(w)
-	usageInfo, err := s.storage.CrawlAndGetDataUsage(GlobalServiceDoneCh)
-	<-doneCh
+	b, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	var cache dataUsageCache
+	err = cache.deserialize(b)
+	if err != nil {
+		logger.LogIf(r.Context(), err)
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	done := keepHTTPResponseAlive(w)
+	usageInfo, err := s.storage.CrawlAndGetDataUsage(r.Context(), cache)
+	done()
 
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-
-	gob.NewEncoder(w).Encode(usageInfo)
+	w.Write(usageInfo.serialize())
 	w.(http.Flusher).Flush()
 }
 
@@ -508,20 +521,69 @@ func (s *storageRESTServer) DeleteFileBulkHandler(w http.ResponseWriter, r *http
 		return
 	}
 
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	encoder := gob.NewEncoder(w)
+	done := keepHTTPResponseAlive(w)
 	errs, err := s.storage.DeleteFileBulk(volume, filePaths)
+	done()
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
 
-	derrsResp := &DeleteFileBulkErrsResp{Errs: make([]error, len(errs))}
+	dErrsResp := &DeleteFileBulkErrsResp{Errs: make([]error, len(errs))}
 	for idx, err := range errs {
 		if err != nil {
-			derrsResp.Errs[idx] = StorageErr(err.Error())
+			dErrsResp.Errs[idx] = StorageErr(err.Error())
 		}
 	}
 
-	gob.NewEncoder(w).Encode(derrsResp)
+	encoder.Encode(dErrsResp)
+	w.(http.Flusher).Flush()
+}
+
+// DeletePrefixesErrsResp - collection of delete errors
+// for bulk prefixes deletes
+type DeletePrefixesErrsResp struct {
+	Errs []error
+}
+
+// DeletePrefixesHandler - delete a set of a prefixes.
+func (s *storageRESTServer) DeletePrefixesHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := r.URL.Query()
+	volume := vars.Get(storageRESTVolume)
+
+	bio := bufio.NewScanner(r.Body)
+	var prefixes []string
+	for bio.Scan() {
+		prefixes = append(prefixes, bio.Text())
+	}
+
+	if err := bio.Err(); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	encoder := gob.NewEncoder(w)
+	done := keepHTTPResponseAlive(w)
+	errs, err := s.storage.DeletePrefixes(volume, prefixes)
+	done()
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	dErrsResp := &DeletePrefixesErrsResp{Errs: make([]error, len(errs))}
+	for idx, err := range errs {
+		if err != nil {
+			dErrsResp.Errs[idx] = StorageErr(err.Error())
+		}
+	}
+	encoder.Encode(dErrsResp)
 	w.(http.Flusher).Flush()
 }
 
@@ -541,11 +603,15 @@ func (s *storageRESTServer) RenameFileHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
-// Send whitespace to the client to avoid timeouts with long storage
+// keepHTTPResponseAlive can be used to avoid timeouts with long storage
 // operations, such as bitrot verification or data usage crawling.
-func sendWhiteSpaceToHTTPResponse(w http.ResponseWriter) <-chan struct{} {
+// Every 10 seconds a space character is sent.
+// The returned function should always be called to release resources.
+// waitForHTTPResponse should be used to the receiving side.
+func keepHTTPResponseAlive(w http.ResponseWriter) func() {
 	doneCh := make(chan struct{})
 	go func() {
+		defer close(doneCh)
 		ticker := time.NewTicker(time.Second * 10)
 		for {
 			select {
@@ -553,13 +619,38 @@ func sendWhiteSpaceToHTTPResponse(w http.ResponseWriter) <-chan struct{} {
 				w.Write([]byte(" "))
 				w.(http.Flusher).Flush()
 			case doneCh <- struct{}{}:
+				w.Write([]byte{0})
 				ticker.Stop()
 				return
 			}
 		}
-
 	}()
-	return doneCh
+	return func() {
+		// Indicate we are ready to write.
+		<-doneCh
+		// Wait for channel to be closed so we don't race on writes.
+		<-doneCh
+	}
+}
+
+// waitForHTTPResponse will wait for responses where keepHTTPResponseAlive
+// has been used.
+// The returned reader contains the payload.
+func waitForHTTPResponse(respBody io.Reader) (io.Reader, error) {
+	reader := bufio.NewReader(respBody)
+	for {
+		b, err := reader.ReadByte()
+		if err != nil {
+			return nil, err
+		}
+		if b != ' ' {
+			if b != 0 {
+				reader.UnreadByte()
+			}
+			break
+		}
+	}
+	return reader, nil
 }
 
 // VerifyFileResp - VerifyFile()'s response.
@@ -601,9 +692,9 @@ func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
 	}
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 	encoder := gob.NewEncoder(w)
-	doneCh := sendWhiteSpaceToHTTPResponse(w)
+	done := keepHTTPResponseAlive(w)
 	err = s.storage.VerifyFile(volume, filePath, size, BitrotAlgorithmFromString(algoStr), hash, int64(shardSize))
-	<-doneCh
+	done()
 	vresp := &VerifyFileResp{}
 	if err != nil {
 		vresp.Err = StorageErr(err.Error())
@@ -665,6 +756,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpointZones EndpointZones
 				Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount, storageRESTLeafFile)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalk).HandlerFunc(httpTraceHdrs(server.WalkHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTMarkerPath, storageRESTRecursive, storageRESTLeafFile)...)
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeletePrefixes).HandlerFunc(httpTraceHdrs(server.DeletePrefixesHandler)).
+				Queries(restQueries(storageRESTVolume)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFileBulk).HandlerFunc(httpTraceHdrs(server.DeleteFileBulkHandler)).

cmd/streaming-signature-v4.go

@@ -102,7 +102,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 
 	// Extract date, if not present throw error.
 	var dateStr string
-	if dateStr = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); dateStr == "" {
+	if dateStr = req.Header.Get("x-amz-date"); dateStr == "" {
 		if dateStr = r.Header.Get("Date"); dateStr == "" {
 			return cred, "", "", time.Time{}, ErrMissingDateHeader
 		}

cmd/sts-errors.go

@@ -29,6 +29,14 @@ import (
 // writeSTSErrorRespone writes error headers
 func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, errCode STSErrorCode, errCtxt error) {
 	err := stsErrCodes.ToSTSErr(errCode)
+	if err.Code == "InternalError" {
+		aerr := getAPIError(APIErrorCode(errCode))
+		if aerr.Code != "InternalError" {
+			err.Code = aerr.Code
+			err.Description = aerr.Description
+			err.HTTPStatusCode = aerr.HTTPStatusCode
+		}
+	}
 	// Generate error response.
 	stsErrorResponse := STSErrorResponse{}
 	stsErrorResponse.Error.Code = err.Code
@@ -73,12 +81,12 @@ type STSErrorCode int
 // Error codes, non exhaustive list - http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithSAML.html
 const (
 	ErrSTSNone STSErrorCode = iota
+	ErrSTSInvalidService
 	ErrSTSAccessDenied
 	ErrSTSMissingParameter
 	ErrSTSInvalidParameterValue
 	ErrSTSWebIdentityExpiredToken
 	ErrSTSClientGrantsExpiredToken
-	ErrSTSInvalidAccessKey
 	ErrSTSInvalidClientGrantsToken
 	ErrSTSMalformedPolicyDocument
 	ErrSTSNotInitialized
@@ -128,11 +136,6 @@ var stsErrCodes = stsErrorCodeMap{
 		Description:    "The client grants token that was passed could not be validated by MinIO.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrSTSInvalidAccessKey: {
-		Code:           "InvalidClientTokenId",
-		Description:    "The security token included in the request is invalid.",
-		HTTPStatusCode: http.StatusForbidden,
-	},
 	ErrSTSMalformedPolicyDocument: {
 		Code:           "MalformedPolicyDocument",
 		Description:    "The request was rejected because the policy document was malformed.",

cmd/sts-handlers.go

@@ -58,6 +58,9 @@ const (
 	expClaim = "exp"
 	subClaim = "sub"
 
+	// JWT claim to check the parent user
+	parentClaim = "parent"
+
 	// LDAP claim keys
 	ldapUser   = "ldapUser"
 	ldapGroups = "ldapGroups"
@@ -116,17 +119,11 @@ func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (user auth.Creden
 	case authTypeSigned:
 		s3Err := isReqAuthenticated(ctx, r, globalServerRegion, serviceSTS)
 		if STSErrorCode(s3Err) != ErrSTSNone {
-			if s3Err == ErrInvalidAccessKeyID {
-				return user, ErrSTSInvalidAccessKey
-			}
 			return user, STSErrorCode(s3Err)
 		}
 		var owner bool
 		user, owner, s3Err = getReqAccessKeyV4(r, globalServerRegion, serviceSTS)
 		if STSErrorCode(s3Err) != ErrSTSNone {
-			if s3Err == ErrInvalidAccessKeyID {
-				return user, ErrSTSInvalidAccessKey
-			}
 			return user, STSErrorCode(s3Err)
 		}
 		// Root credentials are not allowed to use STS API

cmd/test-utils_test.go

@@ -419,22 +419,6 @@ func resetGlobalConfigPath() {
 	globalConfigDir = &ConfigDir{path: ""}
 }
 
-func resetGlobalServiceDoneCh() {
-	// Repeatedly send on the service done channel, so that
-	// listening go-routines will quit. This works better than
-	// closing the channel - closing introduces a new race, as the
-	// current thread writes to the variable, and other threads
-	// listening on it, read from it.
-loop:
-	for {
-		select {
-		case GlobalServiceDoneCh <- struct{}{}:
-		default:
-			break loop
-		}
-	}
-}
-
 // sets globalObjectAPI to `nil`.
 func resetGlobalObjectAPI() {
 	globalObjLayerMutex.Lock()
@@ -499,7 +483,8 @@ func resetGlobalIAMSys() {
 func resetTestGlobals() {
 	// close any indefinitely running go-routines from previous
 	// tests.
-	resetGlobalServiceDoneCh()
+	cancelGlobalContext()
+	initGlobalContext()
 	// set globalObjectAPI to `nil`.
 	resetGlobalObjectAPI()
 	// Reset config path set.

cmd/tree-walk.go

@@ -56,10 +56,10 @@ func filterMatchingPrefix(entries []string, prefixEntry string) []string {
 }
 
 // ListDirFunc - "listDir" function of type listDirFunc returned by listDirFactory() - explained below.
-type ListDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string)
+type ListDirFunc func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string)
 
 // treeWalk walks directory tree recursively pushing TreeWalkResult into the channel as and when it encounters files.
-func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir ListDirFunc, resultCh chan TreeWalkResult, endWalkCh <-chan struct{}, isEnd bool) (totalNum int, treeErr error) {
+func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir ListDirFunc, resultCh chan TreeWalkResult, endWalkCh <-chan struct{}, isEnd bool) (emptyDir bool, treeErr error) {
 	// Example:
 	// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
 	// called with prefixDir="one/two/three/four/" and marker="five.txt"
@@ -75,10 +75,10 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 		}
 	}
 
-	entries := listDir(bucket, prefixDir, entryPrefixMatch)
+	emptyDir, entries := listDir(bucket, prefixDir, entryPrefixMatch)
 	// For an empty list return right here.
-	if len(entries) == 0 {
-		return 0, nil
+	if emptyDir {
+		return true, nil
 	}
 
 	// example:
@@ -90,7 +90,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 	entries = entries[idx:]
 	// For an empty list after search through the entries, return right here.
 	if len(entries) == 0 {
-		return 0, nil
+		return false, nil
 	}
 
 	for i, entry := range entries {
@@ -123,16 +123,16 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 			// markIsEnd is passed to this entry's treeWalk() so that treeWalker.end can be marked
 			// true at the end of the treeWalk stream.
 			markIsEnd := i == len(entries)-1 && isEnd
-			totalFound, err := doTreeWalk(ctx, bucket, pentry, prefixMatch, markerArg, recursive,
+			emptyDir, err := doTreeWalk(ctx, bucket, pentry, prefixMatch, markerArg, recursive,
 				listDir, resultCh, endWalkCh, markIsEnd)
 			if err != nil {
-				return 0, err
+				return false, err
 			}
 
 			// A nil totalFound means this is an empty directory that
 			// needs to be sent to the result channel, otherwise continue
 			// to the next entry.
-			if totalFound > 0 {
+			if !emptyDir {
 				continue
 			}
 		}
@@ -141,13 +141,13 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 		isEOF := ((i == len(entries)-1) && isEnd)
 		select {
 		case <-endWalkCh:
-			return 0, errWalkAbort
+			return false, errWalkAbort
 		case resultCh <- TreeWalkResult{entry: pentry, end: isEOF}:
 		}
 	}
 
 	// Everything is listed.
-	return len(entries), nil
+	return false, nil
 }
 
 // Initiate a new treeWalk in a goroutine.

cmd/tree-walk_test.go

@@ -260,7 +260,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file1" from fsDir1.
-	entries := listDir(volume, "", "")
+	_, entries := listDir(volume, "", "")
 	if len(entries) != 2 {
 		t.Fatal("Expected the number of entries to be 2")
 	}
@@ -278,7 +278,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file2" from fsDir2.
-	entries = listDir(volume, "", "")
+	_, entries = listDir(volume, "", "")
 	if len(entries) != 1 {
 		t.Fatal("Expected the number of entries to be 1")
 	}

cmd/typed-errors.go

@@ -50,9 +50,6 @@ var errRPCAPIVersionUnsupported = errors.New("Unsupported rpc API version")
 // errServerTimeMismatch - server times are too far apart.
 var errServerTimeMismatch = errors.New("Server times are too far apart")
 
-// errOperationTimedOut
-var errOperationTimedOut = errors.New("Operation timed out")
-
 // errInvalidBucketName - bucket name is reserved for MinIO, usually
 // returned for 'minio', '.minio.sys', buckets with capital letters.
 var errInvalidBucketName = errors.New("The specified bucket is not valid")

cmd/web-handlers.go

@@ -690,6 +690,17 @@ next:
 				}
 			}
 			if authErr == errNoAuthToken {
+				// Check if object is allowed to be deleted anonymously
+				if !globalPolicySys.IsAllowed(policy.Args{
+					Action:          policy.DeleteObjectAction,
+					BucketName:      args.BucketName,
+					ConditionValues: getConditionValues(r, "", "", nil),
+					IsOwner:         false,
+					ObjectName:      objectName,
+				}) {
+					return toJSONError(ctx, errAccessDenied)
+				}
+
 				// Check if object is allowed to be deleted anonymously
 				if globalPolicySys.IsAllowed(policy.Args{
 					Action:          policy.BypassGovernanceRetentionAction,
@@ -710,16 +721,29 @@ next:
 			continue
 		}
 
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     claims.AccessKey,
-			Action:          iampolicy.DeleteObjectAction,
-			BucketName:      args.BucketName,
-			ConditionValues: getConditionValues(r, "", claims.AccessKey, claims.Map()),
-			IsOwner:         owner,
-			ObjectName:      objectName,
-			Claims:          claims.Map(),
-		}) {
-			return toJSONError(ctx, errAccessDenied)
+		if authErr == errNoAuthToken {
+			// Check if object is allowed to be deleted anonymously
+			if !globalPolicySys.IsAllowed(policy.Args{
+				Action:          iampolicy.DeleteObjectAction,
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, "", "", nil),
+				IsOwner:         false,
+				ObjectName:      objectName,
+			}) {
+				return toJSONError(ctx, errAccessDenied)
+			}
+		} else {
+			if !globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     claims.AccessKey,
+				Action:          iampolicy.DeleteObjectAction,
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, "", claims.AccessKey, claims.Map()),
+				IsOwner:         owner,
+				ObjectName:      objectName,
+				Claims:          claims.Map(),
+			}) {
+				return toJSONError(ctx, errAccessDenied)
+			}
 		}
 
 		// For directories, list the contents recursively and remove.

cmd/web-handlers_test.go

@@ -449,7 +449,7 @@ func testListBucketsWebHandler(obj ObjectLayer, instanceType string, t TestErrHa
 		t.Fatalf("Cannot find the bucket already created by MakeBucket")
 	}
 	if listBucketsReply.Buckets[0].Name != bucketName {
-		t.Fatalf("Found another bucket other than already created by MakeBucket")
+		t.Fatalf("Found another bucket %q other than already created by MakeBucket", listBucketsReply.Buckets[0].Name)
 	}
 }
 

cmd/xl-sets.go

@@ -235,7 +235,7 @@ func (s *xlSets) connectDisks() {
 // monitorAndConnectEndpoints this is a monitoring loop to keep track of disconnected
 // endpoints by reconnecting them and making sure to place them into right position in
 // the set topology, this monitoring happens at a given monitoring interval.
-func (s *xlSets) monitorAndConnectEndpoints(monitorInterval time.Duration) {
+func (s *xlSets) monitorAndConnectEndpoints(ctx context.Context, monitorInterval time.Duration) {
 
 	ticker := time.NewTicker(monitorInterval)
 	// Stop the timer.
@@ -243,7 +243,7 @@ func (s *xlSets) monitorAndConnectEndpoints(monitorInterval time.Duration) {
 
 	for {
 		select {
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		case <-s.disksConnectDoneCh:
 			return
@@ -332,7 +332,7 @@ func newXLSets(endpoints Endpoints, format *formatXLV3, setCount int, drivesPerS
 	s.connectDisksWithQuorum()
 
 	// Start the disk monitoring and connect routine.
-	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
+	go s.monitorAndConnectEndpoints(GlobalContext, defaultMonitorConnectEndpointInterval)
 
 	go s.maintainMRFList()
 	go s.healMRFRoutine()
@@ -445,8 +445,8 @@ func (s *xlSets) StorageInfo(ctx context.Context, local bool) StorageInfo {
 	return storageInfo
 }
 
-func (s *xlSets) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-	return DataUsageInfo{}
+func (s *xlSets) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+	return NotImplemented{}
 }
 
 // Shutdown shutsdown all erasure coded sets in parallel
@@ -1327,7 +1327,7 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
 	s.connectDisks()
 
 	// Restart monitoring loop to monitor reformatted disks again.
-	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
+	go s.monitorAndConnectEndpoints(GlobalContext, defaultMonitorConnectEndpointInterval)
 
 	return nil
 }
@@ -1516,7 +1516,7 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 		s.connectDisks()
 
 		// Restart our monitoring loop to start monitoring newly formatted disks.
-		go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
+		go s.monitorAndConnectEndpoints(GlobalContext, defaultMonitorConnectEndpointInterval)
 	}
 
 	return res, nil
@@ -1593,8 +1593,8 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun, remove b
 }
 
 // HealObject - heals inconsistent object on a hashedSet based on object name.
-func (s *xlSets) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (madmin.HealResultItem, error) {
-	return s.getHashedSet(object).HealObject(ctx, bucket, object, dryRun, remove, scanMode)
+func (s *xlSets) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	return s.getHashedSet(object).HealObject(ctx, bucket, object, opts)
 }
 
 // Lists all buckets which need healing.
@@ -1642,11 +1642,10 @@ func (s *xlSets) Walk(ctx context.Context, bucket, prefix string, results chan<-
 				return
 			}
 
-			if quorumCount != s.drivesPerSet {
-				return
+			if quorumCount >= s.drivesPerSet/2 {
+				results <- entry.ToObjectInfo() // Read quorum exists proceed
 			}
-
-			results <- entry.ToObjectInfo()
+			// skip entries which do not have quorum
 		}
 	}()
 
@@ -1655,7 +1654,7 @@ func (s *xlSets) Walk(ctx context.Context, bucket, prefix string, results chan<-
 
 // HealObjects - Heal all objects recursively at a specified prefix, any
 // dangling objects deleted as well automatically.
-func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObject healObjectFn) error {
+func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject healObjectFn) error {
 	endWalkCh := make(chan struct{})
 	defer close(endWalkCh)
 
@@ -1669,7 +1668,7 @@ func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObj
 			break
 		}
 
-		if quorumCount == s.drivesPerSet {
+		if quorumCount == s.drivesPerSet && opts.ScanMode == madmin.HealNormalScan {
 			// Skip good entries.
 			continue
 		}
@@ -1764,6 +1763,10 @@ func (s *xlSets) healMRFRoutine() {
 	// Wait until background heal state is initialized
 	var bgSeq *healSequence
 	for {
+		if globalBackgroundHealState == nil {
+			time.Sleep(time.Second)
+			continue
+		}
 		var ok bool
 		bgSeq, ok = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 		if ok {

cmd/xl-v1-healing.go

@@ -695,7 +695,7 @@ func isObjectDangling(metaArr []xlMetaV1, errs []error, dataErrs []error) (valid
 }
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.
-func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool, remove bool, scanMode madmin.HealScanMode) (hr madmin.HealResultItem, err error) {
+func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (hr madmin.HealResultItem, err error) {
 	// Create context that also contains information about the object and bucket.
 	// The top level handler might not have this information.
 	reqInfo := logger.GetReqInfo(ctx)
@@ -709,7 +709,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 
 	// Healing directories handle it separately.
 	if HasSuffix(object, SlashSeparator) {
-		return xl.healObjectDir(healCtx, bucket, object, dryRun)
+		return xl.healObjectDir(healCtx, bucket, object, opts.DryRun)
 	}
 
 	storageDisks := xl.getDisks()
@@ -724,7 +724,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 		if m.Erasure.DataBlocks == 0 {
 			writeQuorum = len(xl.getDisks())/2 + 1
 		}
-		if !dryRun && remove {
+		if !opts.DryRun && opts.Remove {
 			xl.deleteObject(healCtx, bucket, object, writeQuorum, false)
 		}
 		err = reduceReadQuorumErrs(ctx, errs, nil, writeQuorum-1)
@@ -753,7 +753,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 				if m.Erasure.DataBlocks == 0 {
 					writeQuorum = len(storageDisks)/2 + 1
 				}
-				if !dryRun && remove {
+				if !opts.DryRun && opts.Remove {
 					xl.deleteObject(ctx, bucket, object, writeQuorum, false)
 				}
 			}
@@ -762,5 +762,5 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 	}
 
 	// Heal the object.
-	return xl.healObject(healCtx, bucket, object, partsMetadata, errs, latestXLMeta, dryRun, remove, scanMode)
+	return xl.healObject(healCtx, bucket, object, partsMetadata, errs, latestXLMeta, opts.DryRun, opts.Remove, opts.ScanMode)
 }

cmd/xl-v1-healing_test.go

@@ -121,7 +121,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	_, err = objLayer.HealObject(context.Background(), bucket, object, false, false, madmin.HealNormalScan)
+	_, err = objLayer.HealObject(context.Background(), bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
@@ -144,7 +144,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
-	_, err = objLayer.HealObject(context.Background(), bucket, object, false, true, madmin.HealDeepScan)
+	_, err = objLayer.HealObject(context.Background(), bucket, object, madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
 	if err != nil {
 		t.Errorf("Expected nil but received %v", err)
 	}
@@ -170,7 +170,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
-	_, err = objLayer.HealObject(context.Background(), bucket, object, false, true, madmin.HealDeepScan)
+	_, err = objLayer.HealObject(context.Background(), bucket, object, madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
 	if err != nil {
 		t.Errorf("Expected nil but received %v", err)
 	}
@@ -190,7 +190,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
-	_, err = objLayer.HealObject(context.Background(), bucket, object, false, true, madmin.HealDeepScan)
+	_, err = objLayer.HealObject(context.Background(), bucket, object, madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
 	if err != nil {
 		if _, ok := err.(ObjectNotFound); !ok {
 			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
@@ -263,7 +263,7 @@ func TestHealObjectXL(t *testing.T) {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	_, err = obj.HealObject(context.Background(), bucket, object, false, false, madmin.HealNormalScan)
+	_, err = obj.HealObject(context.Background(), bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
@@ -285,7 +285,7 @@ func TestHealObjectXL(t *testing.T) {
 	z.zones[0].xlDisksMu.Unlock()
 
 	// Try healing now, expect to receive errDiskNotFound.
-	_, err = obj.HealObject(context.Background(), bucket, object, false, false, madmin.HealDeepScan)
+	_, err = obj.HealObject(context.Background(), bucket, object, madmin.HealOpts{ScanMode: madmin.HealDeepScan})
 	// since majority of xl.jsons are not available, object quorum can't be read properly and error will be errXLReadQuorum
 	if _, ok := err.(InsufficientReadQuorum); !ok {
 		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)
@@ -333,7 +333,7 @@ func TestHealEmptyDirectoryXL(t *testing.T) {
 	}
 
 	// Heal the object
-	hr, err := obj.HealObject(context.Background(), bucket, object, false, false, madmin.HealNormalScan)
+	hr, err := obj.HealObject(context.Background(), bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
@@ -357,7 +357,7 @@ func TestHealEmptyDirectoryXL(t *testing.T) {
 	}
 
 	// Heal the same object again
-	hr, err = obj.HealObject(context.Background(), bucket, object, false, false, madmin.HealNormalScan)
+	hr, err = obj.HealObject(context.Background(), bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
 		t.Fatalf("Failed to heal object - %v", err)
 	}

cmd/xl-v1-list-objects-heal.go

@@ -20,6 +20,7 @@ import (
 	"context"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // This is not implemented/needed anymore, look for xl-sets.ListBucketHeal()
@@ -29,7 +30,7 @@ func (xl xlObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
 }
 
 // This is not implemented/needed anymore, look for xl-sets.HealObjects()
-func (xl xlObjects) HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) error {
+func (xl xlObjects) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) error {
 	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }

cmd/xl-v1-list-objects.go

@@ -25,7 +25,7 @@ import (
 // disks - used for doing disk.ListDir()
 func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
 	// Returns sorted merged entries from all the disks.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, mergedEntries []string) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
@@ -34,7 +34,7 @@ func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
 			var newEntries []string
 			var err error
 			entries, err = disk.ListDir(bucket, prefixDir, -1, xlMetaJSONFile)
-			if err != nil {
+			if err != nil || len(entries) == 0 {
 				continue
 			}
 
@@ -54,7 +54,12 @@ func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
 				sort.Strings(mergedEntries)
 			}
 		}
-		return filterMatchingPrefix(mergedEntries, prefixEntry)
+
+		if len(mergedEntries) == 0 {
+			return true, nil
+		}
+
+		return false, filterMatchingPrefix(mergedEntries, prefixEntry)
 	}
 	return listDir
 }

cmd/xl-v1-multipart.go

@@ -765,7 +765,7 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
 }
 
 // Clean-up the old multipart uploads. Should be run in a Go routine.
-func (xl xlObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+func (xl xlObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/xl-v1-multipart_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"sync"
 	"testing"
 	"time"
 )
@@ -35,10 +36,6 @@ func TestXLCleanupStaleMultipartUploads(t *testing.T) {
 	z := obj.(*xlZones)
 	xl := z.zones[0].sets[0]
 
-	// Close the go-routine, we are going to
-	// manually start it and test in this test case.
-	GlobalServiceDoneCh <- struct{}{}
-
 	bucketName := "bucket"
 	objectName := "object"
 	var opts ObjectOptions
@@ -49,19 +46,30 @@ func TestXLCleanupStaleMultipartUploads(t *testing.T) {
 		t.Fatal("Unexpected err: ", err)
 	}
 
-	go xl.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, GlobalServiceDoneCh)
+	// Create a context we can cancel.
+	ctx, cancel := context.WithCancel(context.Background())
 
-	// Wait for 40ms such that - we have given enough time for
-	// cleanup routine to kick in.
-	time.Sleep(40 * time.Millisecond)
+	var cleanupWg sync.WaitGroup
+	cleanupWg.Add(1)
+	go func() {
+		defer cleanupWg.Done()
+		xl.cleanupStaleMultipartUploads(context.Background(), time.Millisecond, 0, ctx.Done())
+	}()
 
-	// Close the routine we do not need it anymore.
-	GlobalServiceDoneCh <- struct{}{}
+	// Wait for 100ms such that - we have given enough time for cleanup routine to kick in.
+	// Flaky on slow systems :/
+	time.Sleep(100 * time.Millisecond)
+
+	// Exit cleanup..
+	cancel()
+	cleanupWg.Wait()
 
 	// Check if upload id was already purged.
 	if err = obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
+	} else {
+		t.Error("Item was not cleaned up.")
 	}
 }

cmd/xl-v1-object.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"net/http"
 	"path"
+	"sync"
 
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -781,19 +782,26 @@ func (xl xlObjects) doDeleteObjects(ctx context.Context, bucket string, objects
 	// Initialize list of errors.
 	var opErrs = make([]error, len(disks))
 	var delObjErrs = make([][]error, len(disks))
+	var wg = sync.WaitGroup{}
 
 	// Remove objects in bulk for each disk
-	for index, disk := range disks {
-		if disk == nil {
-			opErrs[index] = errDiskNotFound
+	for i, d := range disks {
+		if d == nil {
+			opErrs[i] = errDiskNotFound
 			continue
 		}
-		delObjErrs[index], opErrs[index] = cleanupObjectsBulk(disk, minioMetaTmpBucket, tmpObjs, errs)
-		if opErrs[index] == errVolumeNotFound || opErrs[index] == errFileNotFound {
-			opErrs[index] = nil
-		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			delObjErrs[index], opErrs[index] = disk.DeletePrefixes(minioMetaTmpBucket, tmpObjs)
+			if opErrs[index] == errVolumeNotFound || opErrs[index] == errFileNotFound {
+				opErrs[index] = nil
+			}
+		}(i, d)
 	}
 
+	wg.Wait()
+
 	// Return errors if any during deletion
 	if err := reduceWriteQuorumErrs(ctx, opErrs, objectOpIgnoredErrs, len(disks)/2+1); err != nil {
 		return nil, err
@@ -805,9 +813,14 @@ func (xl xlObjects) doDeleteObjects(ctx context.Context, bucket string, objects
 			continue
 		}
 		listErrs := make([]error, len(disks))
+		// Iterate over disks to fetch the error
+		// of deleting of the current object
 		for i := range delObjErrs {
+			// delObjErrs[i] is not nil when disks[i] is also not nil
 			if delObjErrs[i] != nil {
-				listErrs[i] = delObjErrs[i][objIndex]
+				if delObjErrs[i][objIndex] != errFileNotFound {
+					listErrs[i] = delObjErrs[i][objIndex]
+				}
 			}
 		}
 		errs[objIndex] = reduceWriteQuorumErrs(ctx, listErrs, objectOpIgnoredErrs, writeQuorums[objIndex])

cmd/xl-v1-object_test.go

@@ -414,7 +414,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = xl.HealObject(context.Background(), bucket, object, false, false, madmin.HealNormalScan)
+	_, err = xl.HealObject(context.Background(), bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -443,7 +443,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = xl.HealObject(context.Background(), bucket, object, false, false, madmin.HealDeepScan)
+	_, err = xl.HealObject(context.Background(), bucket, object, madmin.HealOpts{ScanMode: madmin.HealDeepScan})
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/xl-v1.go

@@ -18,8 +18,10 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"sort"
 	"sync"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
@@ -195,45 +197,175 @@ func (xl xlObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
 	return &Metrics{}, NotImplemented{}
 }
 
-// CrawlAndGetDataUsage picks three random disks to crawl and get data usage
-func (xl xlObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-	var randomDisks []StorageAPI
+// CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
+// Updates are sent on a regular basis and the caller *must* consume them.
+func (xl xlObjects) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+	cache := make(chan dataUsageCache, 1)
+	defer close(cache)
+	buckets, err := xl.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+	go func() {
+		for update := range cache {
+			updates <- update.dui(update.Info.Name, buckets)
+		}
+	}()
+	return xl.crawlAndGetDataUsage(ctx, buckets, cache)
+}
+
+// CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
+// Updates are sent on a regular basis and the caller *must* consume them.
+func (xl xlObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, updates chan<- dataUsageCache) error {
+	var disks []StorageAPI
+
 	for _, d := range xl.getLoadBalancedDisks() {
 		if d == nil || !d.IsOnline() {
 			continue
 		}
-		randomDisks = append(randomDisks, d)
-		if len(randomDisks) >= 3 {
-			break
+		disks = append(disks, d)
+	}
+	if len(disks) == 0 || len(buckets) == 0 {
+		return nil
+	}
+
+	// Load bucket totals
+	oldCache := dataUsageCache{}
+	err := oldCache.load(ctx, xl, dataUsageCacheName)
+	if err != nil {
+		return err
+	}
+
+	// New cache..
+	cache := dataUsageCache{
+		Info: dataUsageCacheInfo{
+			Name:      dataUsageRoot,
+			NextCycle: oldCache.Info.NextCycle,
+		},
+		Cache: make(map[dataUsageHash]dataUsageEntry, len(oldCache.Cache)),
+	}
+
+	// Put all buckets into channel.
+	bucketCh := make(chan BucketInfo, len(buckets))
+	// Add new buckets first
+	for _, b := range buckets {
+		if oldCache.find(b.Name) == nil {
+			bucketCh <- b
+		}
+	}
+	// Add existing buckets.
+	for _, b := range buckets {
+		e := oldCache.find(b.Name)
+		if e != nil {
+			bucketCh <- b
+			cache.replace(b.Name, dataUsageRoot, *e)
 		}
 	}
 
-	var dataUsageResults = make([]DataUsageInfo, len(randomDisks))
+	close(bucketCh)
+	buckets = nil
+	bucketResults := make(chan dataUsageEntryInfo, len(disks))
 
-	var wg sync.WaitGroup
-	for i := 0; i < len(randomDisks); i++ {
-		wg.Add(1)
-		go func(index int, disk StorageAPI) {
-			defer wg.Done()
-			var err error
-			dataUsageResults[index], err = disk.CrawlAndGetDataUsage(endCh)
-			if err != nil {
-				logger.LogIf(ctx, err)
+	// Start async collector/saver.
+	// This goroutine owns the cache.
+	var saverWg sync.WaitGroup
+	saverWg.Add(1)
+	go func() {
+		const updateTime = 30 * time.Second
+		t := time.NewTicker(updateTime)
+		defer t.Stop()
+		defer saverWg.Done()
+		var lastSave time.Time
+
+	saveLoop:
+		for {
+			select {
+			case <-ctx.Done():
+				// Return without saving.
+				return
+			case <-t.C:
+				if cache.Info.LastUpdate.Equal(lastSave) {
+					continue
+				}
+				logger.LogIf(ctx, cache.save(ctx, xl, dataUsageCacheName))
+				updates <- cache.clone()
+				lastSave = cache.Info.LastUpdate
+			case v, ok := <-bucketResults:
+				if !ok {
+					break saveLoop
+				}
+				cache.replace(v.Name, v.Parent, v.Entry)
+				cache.Info.LastUpdate = time.Now()
 			}
-		}(i, randomDisks[i])
+		}
+		// Save final state...
+		cache.Info.NextCycle++
+		cache.Info.LastUpdate = time.Now()
+		logger.LogIf(ctx, cache.save(ctx, xl, dataUsageCacheName))
+		updates <- cache
+	}()
+
+	// Start one crawler per disk
+	var wg sync.WaitGroup
+	wg.Add(len(disks))
+	for i := range disks {
+		go func(i int) {
+			defer wg.Done()
+			disk := disks[i]
+
+			for bucket := range bucketCh {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
+
+				// Load cache for bucket
+				cacheName := pathJoin(bucket.Name, dataUsageCacheName)
+				cache := dataUsageCache{}
+				logger.LogIf(ctx, cache.load(ctx, xl, cacheName))
+				if cache.Info.Name == "" {
+					cache.Info.Name = bucket.Name
+				}
+				if cache.Info.Name != bucket.Name {
+					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
+					cache.Info = dataUsageCacheInfo{
+						Name:       bucket.Name,
+						LastUpdate: time.Time{},
+						NextCycle:  0,
+					}
+				}
+
+				// Calc usage
+				before := cache.Info.LastUpdate
+				cache, err = disk.CrawlAndGetDataUsage(ctx, cache)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					if cache.Info.LastUpdate.After(before) {
+						logger.LogIf(ctx, cache.save(ctx, xl, cacheName))
+					}
+					continue
+				}
+
+				var root dataUsageEntry
+				if r := cache.root(); r != nil {
+					root = cache.flatten(*r)
+				}
+				bucketResults <- dataUsageEntryInfo{
+					Name:   cache.Info.Name,
+					Parent: dataUsageRoot,
+					Entry:  root,
+				}
+				// Save cache
+				logger.LogIf(ctx, cache.save(ctx, xl, cacheName))
+			}
+		}(i)
 	}
 	wg.Wait()
+	close(bucketResults)
+	saverWg.Wait()
 
-	var dataUsageInfo = dataUsageResults[0]
-	// Pick the crawling result of the disk which has the most
-	// number of objects in it.
-	for i := 1; i < len(dataUsageResults); i++ {
-		if dataUsageResults[i].ObjectsCount > dataUsageInfo.ObjectsCount {
-			dataUsageInfo = dataUsageResults[i]
-		}
-	}
-
-	return dataUsageInfo
+	return nil
 }
 
 // IsReady - No Op.

cmd/xl-zones.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -215,44 +216,104 @@ func (z *xlZones) StorageInfo(ctx context.Context, local bool) StorageInfo {
 	return storageInfo
 }
 
-func (z *xlZones) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-	var aggDataUsageInfo = struct {
-		sync.Mutex
-		DataUsageInfo
-	}{}
-
-	aggDataUsageInfo.ObjectsSizesHistogram = make(map[string]uint64)
-	aggDataUsageInfo.BucketsSizes = make(map[string]uint64)
-
+func (z *xlZones) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	var wg sync.WaitGroup
+	var mu sync.Mutex
+	var results []dataUsageCache
+	var firstErr error
+	var knownBuckets = make(map[string]struct{}) // used to deduplicate buckets.
+	var allBuckets []BucketInfo
+
+	// Collect for each set in zones.
 	for _, z := range z.zones {
 		for _, xlObj := range z.sets {
+			// Add new buckets.
+			buckets, err := xlObj.ListBuckets(ctx)
+			if err != nil {
+				return err
+			}
+			for _, b := range buckets {
+				if _, ok := knownBuckets[b.Name]; ok {
+					continue
+				}
+				allBuckets = append(allBuckets, b)
+				knownBuckets[b.Name] = struct{}{}
+			}
 			wg.Add(1)
-			go func(xl *xlObjects) {
-				defer wg.Done()
-				info := xl.CrawlAndGetDataUsage(ctx, endCh)
-
-				aggDataUsageInfo.Lock()
-				aggDataUsageInfo.ObjectsCount += info.ObjectsCount
-				aggDataUsageInfo.ObjectsTotalSize += info.ObjectsTotalSize
-				if aggDataUsageInfo.BucketsCount < info.BucketsCount {
-					aggDataUsageInfo.BucketsCount = info.BucketsCount
+			results = append(results, dataUsageCache{})
+			go func(i int, xl *xlObjects) {
+				updates := make(chan dataUsageCache, 1)
+				defer close(updates)
+				// Start update collector.
+				go func() {
+					defer wg.Done()
+					for info := range updates {
+						mu.Lock()
+						results[i] = info
+						mu.Unlock()
+					}
+				}()
+				// Start crawler. Blocks until done.
+				err := xl.crawlAndGetDataUsage(ctx, buckets, updates)
+				if err != nil {
+					mu.Lock()
+					if firstErr == nil {
+						firstErr = err
+					}
+					// Cancel remaining...
+					cancel()
+					mu.Unlock()
+					return
 				}
-				for k, v := range info.ObjectsSizesHistogram {
-					aggDataUsageInfo.ObjectsSizesHistogram[k] += v
-				}
-				for k, v := range info.BucketsSizes {
-					aggDataUsageInfo.BucketsSizes[k] += v
-				}
-				aggDataUsageInfo.Unlock()
-
-			}(xlObj)
+			}(len(results)-1, xlObj)
 		}
 	}
-	wg.Wait()
+	updateCloser := make(chan chan struct{})
+	go func() {
+		updateTicker := time.NewTicker(30 * time.Second)
+		defer updateTicker.Stop()
+		var lastUpdate time.Time
+		update := func() {
+			mu.Lock()
+			defer mu.Unlock()
 
-	aggDataUsageInfo.LastUpdate = UTCNow()
-	return aggDataUsageInfo.DataUsageInfo
+			// We need to merge since we will get the same buckets from each zone.
+			// Therefore to get the exact bucket sizes we must merge before we can convert.
+			allMerged := dataUsageCache{Info: dataUsageCacheInfo{Name: dataUsageRoot}}
+			for _, info := range results {
+				if info.Info.LastUpdate.IsZero() {
+					// Not filled yet.
+					return
+				}
+				allMerged.merge(info)
+			}
+			if allMerged.root() != nil && allMerged.Info.LastUpdate.After(lastUpdate) {
+				updates <- allMerged.dui(allMerged.Info.Name, allBuckets)
+				lastUpdate = allMerged.Info.LastUpdate
+			}
+		}
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case v := <-updateCloser:
+				update()
+				close(v)
+				return
+			case <-updateTicker.C:
+				update()
+			}
+		}
+	}()
+
+	wg.Wait()
+	ch := make(chan struct{})
+	updateCloser <- ch
+	<-ch
+
+	return firstErr
 }
 
 // This function is used to undo a successful MakeBucket operation.
@@ -1325,11 +1386,11 @@ func (z *xlZones) Walk(ctx context.Context, bucket, prefix string, results chan<
 				return
 			}
 
-			if quorumCount != zoneDrivesPerSet[zoneIndex] {
-				continue
+			if quorumCount >= zoneDrivesPerSet[zoneIndex]/2 {
+				results <- entry.ToObjectInfo() // Read quorum exists proceed
 			}
 
-			results <- entry.ToObjectInfo()
+			// skip entries which do not have quorum
 		}
 	}()
 
@@ -1338,7 +1399,7 @@ func (z *xlZones) Walk(ctx context.Context, bucket, prefix string, results chan<
 
 type healObjectFn func(string, string) error
 
-func (z *xlZones) HealObjects(ctx context.Context, bucket, prefix string, healObject healObjectFn) error {
+func (z *xlZones) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject healObjectFn) error {
 	var zonesEntryChs [][]FileInfoCh
 
 	endWalkCh := make(chan struct{})
@@ -1367,7 +1428,7 @@ func (z *xlZones) HealObjects(ctx context.Context, bucket, prefix string, healOb
 			break
 		}
 
-		if quorumCount == zoneDrivesPerSet[zoneIndex] {
+		if quorumCount == zoneDrivesPerSet[zoneIndex] && opts.ScanMode == madmin.HealNormalScan {
 			// Skip good entries.
 			continue
 		}
@@ -1383,7 +1444,7 @@ func (z *xlZones) HealObjects(ctx context.Context, bucket, prefix string, healOb
 	return nil
 }
 
-func (z *xlZones) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (madmin.HealResultItem, error) {
+func (z *xlZones) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	// Lock the object before healing. Use read lock since healing
 	// will only regenerate parts & xl.json of outdated disks.
 	objectLock := z.NewNSLock(ctx, bucket, object)
@@ -1393,10 +1454,10 @@ func (z *xlZones) HealObject(ctx context.Context, bucket, object string, dryRun,
 	defer objectLock.RUnlock()
 
 	if z.SingleZone() {
-		return z.zones[0].HealObject(ctx, bucket, object, dryRun, remove, scanMode)
+		return z.zones[0].HealObject(ctx, bucket, object, opts)
 	}
 	for _, zone := range z.zones {
-		result, err := zone.HealObject(ctx, bucket, object, dryRun, remove, scanMode)
+		result, err := zone.HealObject(ctx, bucket, object, opts)
 		if err != nil {
 			if isErrObjectNotFound(err) {
 				continue

docs/config/README.md

@@ -235,8 +235,22 @@ This behavior is consistent across all keys, each key self documents itself with
 
 ## Environment only settings (not in config)
 
-#### Worm
-Enable this to turn on Write-Once-Read-Many. By default it is set to `off`. Set ``MINIO_WORM=on`` environment variable to enable WORM mode.
+#### Usage crawler
+Data usage crawler is enabled by default, following ENVs allow for more staggered delay in terms of usage calculation. 
+
+The crawler adapts to the system speed and completely pauses when the system is under load. It is possible to adjust the speed of the crawler and thereby the latency of updates being reflected. The delays between each operation of the crawl can be adjusted by the `MINIO_DISK_USAGE_CRAWL_DELAY` environment variable. By default the value is `10`. This means the crawler will sleep *10x* the time each operation takes.
+
+This will in most setups make the crawler slow enough to not impact overall system performance. Setting `MINIO_DISK_USAGE_CRAWL_DELAY` to a *lower* value will make the crawler faster and setting it to 0 will make the crawler run at full speed (not recommended). Setting it to a higher value will make the crawler slower, further consume less resources.
+
+Example: Following setting will decrease the crawler speed by a factor of 3, reducing the system resource use, but increasing the latency of updates being reflected.
+
+```sh
+export MINIO_DISK_USAGE_CRAWL_DELAY=30
+minio server /data
+```
+
+#### Worm (deprecated)
+Enable this to turn on Write-Once-Read-Many. By default it is set to `off`. Set ``MINIO_WORM=on`` environment variable to enable WORM mode. This ENV setting is not recommended anymore, please use Object Locking and Object Retention APIs documented [here](https://github.com/minio/minio/tree/master/docs/retention).
 
 Example:
 

docs/kms/README.md

@@ -401,7 +401,7 @@ export MINIO_KMS_KES_ENDPOINT=https://localhost:7373
 export MINIO_KMS_KES_KEY_FILE=minio.key
 export MINIO_KMS_KES_CERT_FILE=minio.cert
 export MINIO_KMS_KES_KEY_NAME=minio-key-1
-export MINIO_KMS_KES_CAPATH=kes-tls.crt
+export MINIO_KMS_KES_CA_PATH=kes-tls.crt
 ```
 > The `MINIO_KMS_KES_CAPATH` is only required since we use self-signed certificates.
 

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # 9001 through 9004.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+    image: minio/minio:RELEASE.2020-03-14T02-21-58Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+        image: minio/minio:RELEASE.2020-03-14T02-21-58Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -22,7 +22,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+        image: minio/minio:RELEASE.2020-03-14T02-21-58Z
         args:
         - server
         - http://minio-{0...3}.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -24,7 +24,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+        image: minio/minio:RELEASE.2020-03-14T02-21-58Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -32,7 +32,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2020-03-06T22-23-56Z
+        image: minio/minio:RELEASE.2020-03-14T02-21-58Z
         args:
         - server
         - /data

docs/sts/keycloak.md

@@ -39,15 +39,19 @@ $ go run docs/sts/web-identity.go -cid account -csec 072e7f00-4289-469c-9ab2-bbe
 2018/12/26 17:49:36 listening on http://localhost:8888/
 ```
 
-This will open the login page of keycloak, upon successful login, STS credentials will be printed on the screen, for example
+This will open the login page of keycloak, upon successful login, STS credentials along with any buckets discovered using the credentials will be printed on the screen, for example:
 
 ```
-##### Credentials
 {
-	"accessKey": "6N2BALX7ELO827DXS3GK",
-	"secretKey": "23JKqAD+um8ObHqzfIh+bfqwG9V8qs9tFY6MqeFR",
-	"expiration": "2019-10-01T07:22:34Z",
-	"sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI2TjJCQUxYN0VMTzgyN0RYUzNHSyIsImFjciI6IjAiLCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjoxNTY5OTEwNTUyLCJhenAiOiJhY2NvdW50IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJleHAiOjE1Njk5MTQ1NTQsImlhdCI6MTU2OTkxMDk1NCwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgxL2F1dGgvcmVhbG1zL2RlbW8iLCJqdGkiOiJkOTk4YTBlZS01NDk2LTQ4OWYtYWJlMi00ZWE5MjJiZDlhYWYiLCJuYmYiOjAsInBvbGljeSI6InJlYWR3cml0ZSIsInByZWZlcnJlZF91c2VybmFtZSI6Im5ld3VzZXIxIiwic2Vzc2lvbl9zdGF0ZSI6IjJiYTAyYTI2LWE5MTUtNDUxNC04M2M1LWE0YjgwYjc4ZTgxNyIsInN1YiI6IjY4ZmMzODVhLTA5MjItNGQyMS04N2U5LTZkZTdhYjA3Njc2NSIsInR5cCI6IklEIn0._UG_-ZHgwdRnsp0gFdwChb7VlbPs-Gr_RNUz9EV7TggCD59qjCFAKjNrVHfOSVkKvYEMe0PvwfRKjnJl3A_mBA"
+  "buckets": [
+    "bucket-x"
+  ],
+  "credentials": {
+    "AccessKeyID": "6N2BALX7ELO827DXS3GK",
+    "SecretAccessKey": "23JKqAD+um8ObHqzfIh+bfqwG9V8qs9tFY6MqeFR+xxx",
+    "SessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI2TjJCQUxYN0VMTzgyN0RYUzNHSyIsImFjciI6IjAiLCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjoxNTY5OTEwNTUyLCJhenAiOiJhY2NvdW50IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJleHAiOjE1Njk5MTQ1NTQsImlhdCI6MTU2OTkxMDk1NCwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgxL2F1dGgvcmVhbG1zL2RlbW8iLCJqdGkiOiJkOTk4YTBlZS01NDk2LTQ4OWYtYWJlMi00ZWE5MjJiZDlhYWYiLCJuYmYiOjAsInBvbGljeSI6InJlYWR3cml0ZSIsInByZWZlcnJlZF91c2VybmFtZSI6Im5ld3VzZXIxIiwic2Vzc2lvbl9zdGF0ZSI6IjJiYTAyYTI2LWE5MTUtNDUxNC04M2M1LWE0YjgwYjc4ZTgxNyIsInN1YiI6IjY4ZmMzODVhLTA5MjItNGQyMS04N2U5LTZkZTdhYjA3Njc2NSIsInR5cCI6IklEIn0._UG_-ZHgwdRnsp0gFdwChb7VlbPs-Gr_RNUz9EV7TggCD59qjCFAKjNrVHfOSVkKvYEMe0PvwfRKjnJl3A_mBA"",
+    "SignerType": 1
+  }
 }
 ```
 

docs/sts/web-identity.go

@@ -143,6 +143,7 @@ func main() {
 
 	ddoc, err := parseDiscoveryDoc(configEndpoint)
 	if err != nil {
+		log.Println(fmt.Errorf("Failed to parse OIDC discovery document %s", err))
 		fmt.Println(err)
 		return
 	}
@@ -163,10 +164,16 @@ func main() {
 	state := randomState()
 
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("%s %s", r.Method, r.RequestURI)
+		if r.RequestURI != "/" {
+			http.NotFound(w, r)
+			return
+		}
 		http.Redirect(w, r, config.AuthCodeURL(state), http.StatusFound)
 	})
 
 	http.HandleFunc("/oauth2/callback", func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("%s %s", r.Method, r.RequestURI)
 		if r.URL.Query().Get("state") != state {
 			http.Error(w, "state did not match", http.StatusBadRequest)
 			return
@@ -189,13 +196,11 @@ func main() {
 
 		sts, err := credentials.NewSTSWebIdentity(stsEndpoint, getWebTokenExpiry)
 		if err != nil {
+			log.Println(fmt.Errorf("Could not get STS credentials: %s", err))
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
 
-		// Uncomment this to use MinIO API operations by initializing minio
-		// client with obtained credentials.
-
 		opts := &minio.Options{
 			Creds:        sts,
 			BucketLookup: minio.BucketLookupAuto,
@@ -203,23 +208,40 @@ func main() {
 
 		u, err := url.Parse(stsEndpoint)
 		if err != nil {
+			log.Println(fmt.Errorf("Failed to parse STS Endpoint: %s", err))
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
 
 		clnt, err := minio.NewWithOptions(u.Host, opts)
 		if err != nil {
+			log.Println(fmt.Errorf("Error while initializing Minio client, %s", err))
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
 		buckets, err := clnt.ListBuckets()
 		if err != nil {
+			log.Println(fmt.Errorf("Error while listing buckets, %s", err))
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
+		creds, _ := sts.Get()
+
+		bucketNames := []string{}
+
 		for _, bucket := range buckets {
-			log.Println(bucket)
+			log.Println(fmt.Sprintf("Bucket discovered: %s", bucket.Name))
+			bucketNames = append(bucketNames, bucket.Name)
 		}
+		response := make(map[string]interface{})
+		response["credentials"] = creds
+		response["buckets"] = bucketNames
+		c, err := json.MarshalIndent(response, "", "\t")
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		w.Write(c)
 	})
 
 	address := fmt.Sprintf("localhost:%v", port)

docs/zh_CN/distributed/README.md

@@ -9,21 +9,17 @@
 ### 数据保护
 
 
-分布式Minio采用 [erasure code](https://docs.min.io/cn/minio-erasure-code-quickstart-guide)来防范多个节点宕机和[位衰减`bit rot`](https://github.com/minio/minio/blob/master/docs/zh_CN/erasure/README.md#what-is-bit-rot-protection)。  
+分布式Minio采用 [纠删码](https://docs.min.io/cn/minio-erasure-code-quickstart-guide)来防范多个节点宕机和[位衰减`bit rot`](https://github.com/minio/minio/blob/master/docs/zh_CN/erasure/README.md#what-is-bit-rot-protection)。  
 
-分布式Minio至少需要4个节点，使用分布式Minio自动引入了纠删码功能。
+分布式Minio至少需要4个硬盘，使用分布式Minio自动引入了纠删码功能。
 
 ### 高可用
 
-单机Minio服务存在单点故障，相反，如果是一个N节点的分布式Minio,只要有N/2节点在线，你的数据就是安全的。不过你需要至少有N/2+1个节点   [Quorum](https://github.com/minio/dsync#lock-process) 来创建新的对象。
+单机Minio服务存在单点故障，相反，如果是一个有N块硬盘的分布式Minio,只要有N/2硬盘在线，你的数据就是安全的。不过你需要至少有N/2+1个硬盘来创建新的对象。
 
-例如，一个8节点的Minio集群，每个节点一块盘，就算4个节点宕机，这个集群仍然是可读的，不过你需要5个节点才能写数据。
+例如，一个16节点的Minio集群，每个节点16块硬盘，就算8台服務器宕机，这个集群仍然是可读的，不过你需要9台服務器才能写数据。
 
-### 限制
-
-分布式Minio单租户存在最少4个盘最多16个盘的限制（受限于纠删码）。这种限制确保了Minio的简洁，同时仍拥有伸缩性。如果你需要搭建一个多租户环境，你可以轻松的使用编排工具（Kubernetes）来管理多个Minio实例。
-
-注意，只要遵守分布式Minio的限制，你可以组合不同的节点和每个节点几块盘。比如，你可以使用2个节点，每个节点4块盘，也可以使用4个节点，每个节点两块盘，诸如此类。
+注意，只要遵守分布式Minio的限制，你可以组合不同的节点和每个节点几块硬盘。比如，你可以使用2个节点，每个节点4块硬盘，也可以使用4个节点，每个节点两块硬盘，诸如此类。
 
 ### 一致性