fix: data usage crawler env handling, usage-cache.bin location (#9163)

canonicalize the ENVs such that we can bring these ENVs 
as part of the config values, as a subsequent change.

- fix location of per bucket usage to `.minio.sys/buckets/<bucket_name>/usage-cache.bin`
- fix location of the overall usage in `json` at `.minio.sys/buckets/.usage.json`
  (avoid conflicts with a bucket named `usage.json` )
- fix location of the overall usage in `msgp` at `.minio.sys/buckets/.usage.bin`
  (avoid conflicts with a bucket named `usage.bin`

Dockerfile

@@ -22,6 +22,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
+COPY CREDITS /third_party/
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \

Dockerfile.arm.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm32v7/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.arm64.release

@@ -0,0 +1,41 @@
+FROM golang:1.13-alpine as builder
+
+WORKDIR /home
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+ENV GO111MODULE on
+
+RUN  \
+     apk add --no-cache git 'curl>7.61.0' && \
+     git clone https://github.com/minio/minio && \
+     curl -L https://github.com/balena-io/qemu/releases/download/v3.0.0%2Bresin/qemu-3.0.0+resin-arm.tar.gz | tar zxvf - -C . && mv qemu-3.0.0+resin-arm/qemu-arm-static . 
+
+FROM arm64v8/alpine:3.10
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
+COPY --from=builder /home/qemu-arm-static /usr/bin/qemu-arm-static
+
+ENV MINIO_UPDATE off
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.min.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Dockerfile.dev

@@ -4,6 +4,7 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \

Dockerfile.release

@@ -13,6 +13,7 @@ FROM alpine:3.10
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY CREDITS /third_party/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \

Makefile

@@ -22,7 +22,7 @@ getdeps:
 ifeq ($(GOARCH),s390x)
 	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
 else
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2019.2.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2020.1.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
 endif
 	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
 

browser/app/js/browser/Login.js

@@ -148,7 +148,8 @@ export class Login extends React.Component {
                   <OpenIDLoginButton
                     className="btn openid-btn"
                     clientId={this.state.clientId}
-                    authorizationEndpoint={this.state.discoveryDoc.authorization_endpoint}
+                    authEp={this.state.discoveryDoc.authorization_endpoint}
+                    authScopes={this.state.discoveryDoc.scopes_supported}
                   >
                     Log in with OpenID
                   </OpenIDLoginButton>

browser/app/js/browser/OpenIDLogin.js

@@ -66,6 +66,7 @@ export class OpenIDLogin extends React.Component {
 
       const authURL = buildOpenIDAuthURL(
         this.state.discoveryDoc.authorization_endpoint,
+        this.state.discoveryDoc.scopes_supported,
         redirectURI,
         this.state.clientID,
         nonce

browser/app/js/browser/OpenIDLoginButton.js

@@ -27,7 +27,7 @@ export class OpenIDLoginButton extends React.Component {
 
   handleClick(event) {
     event.stopPropagation()
-    const { authorizationEndpoint, clientId } = this.props
+    const { authEp, authScopes, clientId } = this.props
 
     let redirectURI = window.location.href.split("#")[0]
     if (redirectURI.endsWith('/')) {
@@ -40,7 +40,7 @@ export class OpenIDLoginButton extends React.Component {
     const nonce = getRandomString(16)
     storage.setItem(OPEN_ID_NONCE_KEY, nonce)
 
-    const authURL = buildOpenIDAuthURL(authorizationEndpoint, redirectURI, clientId, nonce)
+    const authURL = buildOpenIDAuthURL(authEp, authScopes, redirectURI, clientId, nonce)
     window.location = authURL
   }
 

browser/app/js/browser/utils.js

@@ -16,14 +16,13 @@
 
 export const OPEN_ID_NONCE_KEY = 'openIDKey'
 
-export const buildOpenIDAuthURL = (authorizationEndpoint, redirectURI, clientID, nonce) => {
+export const buildOpenIDAuthURL = (authEp, authScopes, redirectURI, clientID, nonce) => {
   const params = new URLSearchParams()
   params.set("response_type", "id_token")
-  params.set("scope", "openid")
+  params.set("scope", authScopes.join(" "))
   params.set("client_id", clientID)
   params.set("redirect_uri", redirectURI)
   params.set("nonce", nonce)
 
-  return `${authorizationEndpoint}?${params.toString()}`
+  return `${authEp}?${params.toString()}`
 }
-

browser/staticcheck.conf

@@ -0,0 +1 @@
+checks = ["all", "-ST1005", "-ST1000", "-SA4000", "-SA9004", "-SA1019", "-SA1008", "-U1000", "-ST1003", "-ST1018"]

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64"
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
 }
 
 function _build() {

buildscripts/verify-healing.sh

@@ -74,7 +74,7 @@ function __init__()
     echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
 }
 
-function perform_test_1() {
+function perform_test() {
     minio_pids=( $(start_minio_3_node 60) )
     for pid in "${minio_pids[@]}"; do
         if ! kill "$pid"; then
@@ -86,106 +86,14 @@ function perform_test_1() {
             purge "$WORK_DIR"
             exit 1
         fi
+        # forcibly killing, to proceed further properly.
+        kill -9 "$pid"
     done
 
-    echo "Testing in Distributed Erasure setup healing test case 1"
-    echo "Remove the contents of the disks belonging to '2' erasure set"
+    echo "Testing Distributed Erasure setup healing of drives"
+    echo "Remove the contents of the disks belonging to '${1}' erasure set"
 
-    rm -rf ${WORK_DIR}/2/*/
-
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            for i in $(seq 1 3); do
-                echo "server$i log:"
-                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
-            done
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
-    done
-
-    rv=$(check_online)
-    if [ "$rv" == "1" ]; then
-        for pid in "${minio_pids[@]}"; do
-            kill -9 "$pid"
-        done
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
-}
-
-function perform_test_2() {
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            for i in $(seq 1 3); do
-                echo "server$i log:"
-                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
-            done
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
-    done
-
-    echo "Testing in Distributed Erasure setup healing test case 2"
-    echo "Remove the contents of the disks belonging to '1' erasure set"
-
-    rm -rf ${WORK_DIR}/1/*/
-
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            for i in $(seq 1 3); do
-                echo "server$i log:"
-                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
-            done
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
-    done
-
-    rv=$(check_online)
-    if [ "$rv" == "1" ]; then
-        for pid in "${minio_pids[@]}"; do
-            kill -9 "$pid"
-        done
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
-}
-
-function perform_test_3() {
-    minio_pids=( $(start_minio_3_node 60) )
-    for pid in "${minio_pids[@]}"; do
-        if ! kill "$pid"; then
-            for i in $(seq 1 3); do
-                echo "server$i log:"
-                cat "${WORK_DIR}/dist-minio-$[8000+$i].log"
-            done
-            echo "FAILED"
-            purge "$WORK_DIR"
-            exit 1
-        fi
-    done
-
-    echo "Testing in Distributed Erasure setup healing test case 3"
-    echo "Remove the contents of the disks belonging to '3' erasure set"
-
-    rm -rf ${WORK_DIR}/3/*/
+    rm -rf ${WORK_DIR}/${1}/*/
 
     minio_pids=( $(start_minio_3_node 60) )
     for pid in "${minio_pids[@]}"; do
@@ -198,6 +106,9 @@ function perform_test_3() {
             purge "$WORK_DIR"
             exit 1
         fi
+        # forcibly killing, to proceed further properly.
+        # if the previous kill is taking time.
+        kill -9 "$pid"
     done
 
     rv=$(check_online)
@@ -217,9 +128,9 @@ function perform_test_3() {
 
 function main()
 {
-    perform_test_1
-    perform_test_2
-    perform_test_3
+    perform_test "2"
+    perform_test "1"
+    perform_test "3"
 }
 
 ( __init__ "$@" && main "$@" )

cmd/admin-handlers-users.go

@@ -378,6 +378,120 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// AddServiceAccount - PUT /minio/admin/v2/add-service-account?parentUser=<parent_user_accesskey>
+func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AddServiceAccount")
+
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	password := cred.SecretKey
+	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	var createReq madmin.AddServiceAccountReq
+	if err = json.Unmarshal(configBytes, &createReq); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	if createReq.Parent == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+
+	// Disallow creating service accounts for the root user
+	if createReq.Parent == globalActiveCred.AccessKey {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddServiceAccountInvalidArgument), r.URL)
+		return
+	}
+
+	creds, err := globalIAMSys.NewServiceAccount(ctx, createReq.Parent, createReq.Policy)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(creds.AccessKey, false) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	var createResp = madmin.AddServiceAccountResp{
+		Credentials: creds,
+	}
+
+	data, err := json.Marshal(createResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
+// GetServiceAccount - GET /minio/admin/v2/get-service-account?accessKey=<access_key>
+func (a adminAPIHandlers) GetServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetServiceAccount")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars["accessKey"]
+
+	creds, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(creds)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	password := globalActiveCred.SecretKey
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
 // InfoCannedPolicy - GET /minio/admin/v2/info-canned-policy?name={policyName}
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")

cmd/admin-handlers.go

@@ -141,9 +141,9 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		if runtime.GOOS == "windows" {
-			u.Path = path.Dir(u.Path) + "minio.exe"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio.exe"
 		} else {
-			u.Path = path.Dir(u.Path) + "minio"
+			u.Path = path.Dir(u.Path) + SlashSeparator + "minio"
 		}
 
 		updateURL = u.String()
@@ -982,13 +982,24 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 
 	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{}
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
 	for _, state := range bgHealStates {
 		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
 			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
 		}
-
 	}
 
 	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
@@ -1398,10 +1409,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		} else if ldapConn == nil {
 			ldap.Status = "Not Configured"
 		} else {
+			// Close ldap connection to avoid leaks.
+			ldapConn.Close()
 			ldap.Status = "online"
 		}
-		// Close ldap connection to avoid leaks.
-		defer ldapConn.Close()
 	}
 
 	log, audit := fetchLoggerInfo(cfg)

cmd/admin-handlers_test.go

@@ -263,6 +263,7 @@ func TestAdminServerInfo(t *testing.T) {
 	if err != nil {
 		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
+
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.

cmd/admin-heal-ops.go

@@ -106,12 +106,12 @@ func initHealState() *allHealState {
 		healSeqMap: make(map[string]*healSequence),
 	}
 
-	go healState.periodicHealSeqsClean()
+	go healState.periodicHealSeqsClean(GlobalContext)
 
 	return healState
 }
 
-func (ahs *allHealState) periodicHealSeqsClean() {
+func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
 	ticker := time.NewTicker(time.Minute * 5)
@@ -127,7 +127,7 @@ func (ahs *allHealState) periodicHealSeqsClean() {
 				}
 			}
 			ahs.Unlock()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			// server could be restarting - need
 			// to exit immediately
 			return
@@ -369,7 +369,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	return &healSequence{
 		bucket:         bucket,
@@ -575,8 +575,6 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	h.lastHealActivity = UTCNow()
-
 	bucketsOnly := true // heal buckets only, not objects.
 	if err := h.healItems(bucketsOnly); err != nil {
 		logger.LogIf(h.ctx, err)
@@ -605,7 +603,7 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			h.lastHealActivity = UTCNow()
 		case <-h.traverseAndHealDoneCh:
 			return nil
-		case <-GlobalServiceDoneCh:
+		case <-h.ctx.Done():
 			return nil
 		}
 	}
@@ -632,11 +630,6 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 		return err
 	}
 
-	// Start healing the background ops prefix.
-	if err := h.healMinioSysMeta(backgroundOpsMetaPrefix)(); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
-
 	// Heal buckets and objects
 	return h.healBuckets(bucketsOnly)
 }
@@ -673,7 +666,7 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, func(bucket string, object string) error {
+		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket string, object string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -767,7 +760,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.healObject); err != nil {
+	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.settings, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil

cmd/admin-router.go

@@ -110,6 +110,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
+		// Service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
+		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/get-service-account").HandlerFunc(httpTraceHdrs(adminAPI.GetServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+
 		// Info policy IAM
 		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 

cmd/api-errors.go

@@ -122,7 +122,8 @@ const (
 	ErrMissingCredTag
 	ErrCredMalformed
 	ErrInvalidRegion
-	ErrInvalidService
+	ErrInvalidServiceS3
+	ErrInvalidServiceSTS
 	ErrInvalidRequestVersion
 	ErrMissingSignTag
 	ErrMissingSignHeadersTag
@@ -332,6 +333,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrAddServiceAccountInvalidArgument
 	ErrPostPolicyConditionInvalidFormat
 )
 
@@ -652,9 +654,14 @@ var errorCodes = errorCodeMap{
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
 	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
 	// Need changes to make sure variable messages can be constructed.
-	ErrInvalidService: {
-		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+	ErrInvalidServiceS3: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidServiceSTS: {
+		Code:           "AuthorizationParametersError",
+		Description:    "Error parsing the Credential parameter; incorrect service. This endpoint belongs to \"sts\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
@@ -1573,6 +1580,12 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrAddServiceAccountInvalidArgument: {
+		Code:           "XMinioInvalidArgument",
+		Description:    "New service accounts for admin access key is not allowed",
+		HTTPStatusCode: http.StatusConflict,
+	},
+
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
 		Description:    "Invalid according to Policy: Policy Condition failed",
@@ -1641,7 +1654,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case crypto.ErrKMSAuthLogin:
 		apiErr = ErrKMSAuthFailure
-	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
+	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
 		apiErr = ErrSlowDown
@@ -1769,6 +1782,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrOverlappingFilterNotification
 	case *event.ErrUnsupportedConfiguration:
 		apiErr = ErrUnsupportedNotification
+	case OperationTimedOut:
+		apiErr = ErrOperationTimedOut
 	case BackendDown:
 		apiErr = ErrBackendDown
 	case ObjectNameTooLong:

cmd/api-headers.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -81,6 +82,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
+	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
+		w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts)))
+	}
+
 	if objInfo.ContentType != "" {
 		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}

cmd/background-heal-ops.go

@@ -186,5 +186,5 @@ func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOp
 	if objectAPI == nil {
 		return madmin.HealResultItem{}, errServerNotInitialized
 	}
-	return objectAPI.HealObject(ctx, bucket, object, opts.DryRun, opts.Remove, opts.ScanMode)
+	return objectAPI.HealObject(ctx, bucket, object, opts)
 }

cmd/bucket-handlers.go

@@ -342,13 +342,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Content-Length is required and should be non-zero
-	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-	if r.ContentLength <= 0 {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
 	// Content-Md5 is requied should be set
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
@@ -356,27 +349,21 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Allocate incoming content length bytes.
-	var deleteXMLBytes []byte
-	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
-	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
-		deleteXMLBytes = make([]byte, maxBodySize)
-	} else {
-		deleteXMLBytes = make([]byte, r.ContentLength)
-	}
-
-	// Read incoming body XML bytes.
-	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
-		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
+	// Content-Length is required and should be non-zero
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if r.ContentLength <= 0 {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
+	// The max. XML contains 100000 object names (each at most 1024 bytes long) + XML overhead
+	const maxBodySize = 2 * 100000 * 1024
+
 	// Unmarshal list of keys to be deleted.
 	deleteObjects := &DeleteObjectsRequest{}
-	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
+	if err := xmlDecoder(r.Body, deleteObjects, maxBodySize); err != nil {
 		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/common-main.go

@@ -22,6 +22,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"net"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -226,10 +227,22 @@ func handleCommonEnvVars() {
 		globalConfigEncrypted = true
 	}
 
+	if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
+		oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate the old credentials inherited from the shell environment")
+		}
+		globalOldCred = oldCred
+		os.Unsetenv(config.EnvAccessKeyOld)
+		os.Unsetenv(config.EnvSecretKeyOld)
+	}
+
 	globalWORMEnabled, err = config.LookupWorm()
 	if err != nil {
 		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
 	}
+
 }
 
 func logStartupMessage(msg string) {

cmd/config-current.go

@@ -26,7 +26,6 @@ import (
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
 	"github.com/minio/minio/cmd/config/etcd"
-	xetcd "github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
@@ -304,13 +303,13 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	etcdCfg, err := xetcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
+	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 	}
 
 	if etcdCfg.Enabled {
-		globalEtcdClient, err = xetcd.New(etcdCfg)
+		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 		}
@@ -428,6 +427,10 @@ func lookupConfigs(s config.Config) {
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
+	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalServiceDoneCh, NewCustomHTTPTransport())
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
+	}
 }
 
 // Help - return sub-system level help

cmd/config-encrypted.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 	"unicode/utf8"
 
@@ -29,7 +28,6 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -88,11 +86,6 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	doneCh = make(chan struct{})
 	defer close(doneCh)
 
@@ -106,7 +99,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		select {
 		case <-retryTimerCh:
 			// Migrate IAM configuration
-			if err = migrateConfigPrefixToEncrypted(objAPI, activeCredOld, encrypted); err != nil {
+			if err = migrateConfigPrefixToEncrypted(objAPI, globalOldCred, encrypted); err != nil {
 				if err == errDiskNotFound ||
 					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
 					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
@@ -164,21 +157,6 @@ func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
 	return data, err
 }
 
-func getOldCreds() (activeCredOld auth.Credentials, err error) {
-	accessKeyOld := env.Get(config.EnvAccessKeyOld, "")
-	secretKeyOld := env.Get(config.EnvSecretKeyOld, "")
-	if accessKeyOld != "" && secretKeyOld != "" {
-		activeCredOld, err = auth.CreateCredentials(accessKeyOld, secretKeyOld)
-		if err != nil {
-			return activeCredOld, err
-		}
-		// Once we have obtained the rotating creds
-		os.Unsetenv(config.EnvAccessKeyOld)
-		os.Unsetenv(config.EnvSecretKeyOld)
-	}
-	return activeCredOld, nil
-}
-
 func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	defer cancel()
@@ -206,20 +184,15 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 	}
 
-	activeCredOld, err := getOldCreds()
-	if err != nil {
-		return err
-	}
-
 	if encrypted {
 		// No key rotation requested, and backend is
 		// already encrypted. We proceed without migration.
-		if !activeCredOld.IsValid() {
+		if !globalOldCred.IsValid() {
 			return nil
 		}
 
 		// No real reason to rotate if old and new creds are same.
-		if activeCredOld.Equal(globalActiveCred) {
+		if globalOldCred.Equal(globalActiveCred) {
 			return nil
 		}
 
@@ -254,8 +227,8 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 
 		var data []byte
 		// Is rotating of creds requested?
-		if activeCredOld.IsValid() {
-			data, err = decryptData(cdata, activeCredOld, globalActiveCred)
+		if globalOldCred.IsValid() {
+			data, err = decryptData(cdata, globalOldCred, globalActiveCred)
 			if err != nil {
 				if err == madmin.ErrMaliciousData {
 					return config.ErrInvalidRotatingCredentialsBackendEncrypted(nil)
@@ -285,7 +258,7 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		}
 	}
 
-	if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
+	if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
 

cmd/config/cache/config.go

@@ -28,13 +28,15 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled bool     `json:"-"`
-	Drives  []string `json:"drives"`
-	Expiry  int      `json:"expiry"`
-	MaxUse  int      `json:"maxuse"`
-	Quota   int      `json:"quota"`
-	Exclude []string `json:"exclude"`
-	After   int      `json:"after"`
+	Enabled       bool     `json:"-"`
+	Drives        []string `json:"drives"`
+	Expiry        int      `json:"expiry"`
+	MaxUse        int      `json:"maxuse"`
+	Quota         int      `json:"quota"`
+	Exclude       []string `json:"exclude"`
+	After         int      `json:"after"`
+	WatermarkLow  int      `json:"watermark_low"`
+	WatermarkHigh int      `json:"watermark_high"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -64,6 +66,15 @@ func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
 	if _cfg.After < 0 {
 		return errors.New("cache after value should not be less than 0")
 	}
+	if _cfg.WatermarkLow < 0 || _cfg.WatermarkLow > 100 {
+		return errors.New("config low watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkHigh < 0 || _cfg.WatermarkHigh > 100 {
+		return errors.New("config high watermark value should be between 0 and 100")
+	}
+	if _cfg.WatermarkLow > 0 && (_cfg.WatermarkLow >= _cfg.WatermarkHigh) {
+		return errors.New("config low watermark value should be less than high watermark")
+	}
 	return nil
 }
 

cmd/config/cache/help.go

@@ -56,5 +56,17 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         WatermarkLow,
+			Description: `% of cache use at which to stop cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         WatermarkHigh,
+			Description: `% of cache use at which to start cache eviction`,
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -26,25 +26,31 @@ import (
 
 // Cache ENVs
 const (
-	Drives  = "drives"
-	Exclude = "exclude"
-	Expiry  = "expiry"
-	MaxUse  = "maxuse"
-	Quota   = "quota"
-	After   = "after"
+	Drives        = "drives"
+	Exclude       = "exclude"
+	Expiry        = "expiry"
+	MaxUse        = "maxuse"
+	Quota         = "quota"
+	After         = "after"
+	WatermarkLow  = "watermark_low"
+	WatermarkHigh = "watermark_high"
 
-	EnvCacheDrives  = "MINIO_CACHE_DRIVES"
-	EnvCacheExclude = "MINIO_CACHE_EXCLUDE"
-	EnvCacheExpiry  = "MINIO_CACHE_EXPIRY"
-	EnvCacheMaxUse  = "MINIO_CACHE_MAXUSE"
-	EnvCacheQuota   = "MINIO_CACHE_QUOTA"
-	EnvCacheAfter   = "MINIO_CACHE_AFTER"
+	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry        = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse        = "MINIO_CACHE_MAXUSE"
+	EnvCacheQuota         = "MINIO_CACHE_QUOTA"
+	EnvCacheAfter         = "MINIO_CACHE_AFTER"
+	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
+	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
-	DefaultExpiry = "90"
-	DefaultQuota  = "80"
-	DefaultAfter  = "0"
+	DefaultExpiry        = "90"
+	DefaultQuota         = "80"
+	DefaultAfter         = "0"
+	DefaultWaterMarkLow  = "70"
+	DefaultWaterMarkHigh = "80"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -70,6 +76,14 @@ var (
 			Key:   After,
 			Value: DefaultAfter,
 		},
+		config.KV{
+			Key:   WatermarkLow,
+			Value: DefaultWaterMarkLow,
+		},
+		config.KV{
+			Key:   WatermarkHigh,
+			Value: DefaultWaterMarkHigh,
+		},
 	}
 )
 
@@ -87,7 +101,6 @@ func Enabled(kvs config.KVS) bool {
 // variables and merge them with provided CacheConfiguration.
 func LookupConfig(kvs config.KVS) (Config, error) {
 	cfg := Config{}
-
 	if err := config.CheckValidKeys(config.CacheSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
@@ -154,5 +167,33 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		}
 	}
 
+	if lowWMStr := env.Get(EnvCacheWatermarkLow, kvs.Get(WatermarkLow)); lowWMStr != "" {
+		cfg.WatermarkLow, err = strconv.Atoi(lowWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+		// WatermarkLow should be a valid percentage.
+		if cfg.WatermarkLow < 0 || cfg.WatermarkLow > 100 {
+			err := errors.New("config min watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkLow(err)
+		}
+	}
+
+	if highWMStr := env.Get(EnvCacheWatermarkHigh, kvs.Get(WatermarkHigh)); highWMStr != "" {
+		cfg.WatermarkHigh, err = strconv.Atoi(highWMStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+
+		// MaxWatermark should be a valid percentage.
+		if cfg.WatermarkHigh < 0 || cfg.WatermarkHigh > 100 {
+			err := errors.New("config high watermark value should be between 0 and 100")
+			return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+		}
+	}
+	if cfg.WatermarkLow > cfg.WatermarkHigh {
+		err := errors.New("config high watermark value should be greater than low watermark value")
+		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
+	}
 	return cfg, nil
 }

cmd/config/errors-utils.go

@@ -46,7 +46,7 @@ func (u Err) Clone() Err {
 	}
 }
 
-// Return the error message
+// Error returns the error message
 func (u Err) Error() string {
 	if u.detail == "" {
 		if u.msg != "" {

cmd/config/errors.go

@@ -72,6 +72,18 @@ var (
 		"MINIO_CACHE_AFTER: Valid cache after value must be 0 or greater",
 	)
 
+	ErrInvalidCacheWatermarkLow = newErrFn(
+		"Invalid cache low watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_LOW: Valid cache low watermark value must be between 0-100",
+	)
+
+	ErrInvalidCacheWatermarkHigh = newErrFn(
+		"Invalid cache high watermark value",
+		"Please check the passed value",
+		"MINIO_CACHE_WATERMARK_HIGH: Valid cache high watermark value must be between 0-100",
+	)
+
 	ErrInvalidCacheEncryptionKey = newErrFn(
 		"Invalid cache encryption master key value",
 		"Please check the passed value",

cmd/config/etcd/dns/etcd_dns.go

@@ -60,11 +60,7 @@ func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
 			if record.Key == "" {
 				continue
 			}
-			if _, ok := srvRecords[record.Key]; ok {
-				srvRecords[record.Key] = append(srvRecords[record.Key], record)
-			} else {
-				srvRecords[record.Key] = []SrvRecord{record}
-			}
+			srvRecords[record.Key] = append(srvRecords[record.Key], record)
 		}
 	}
 	return srvRecords, nil

cmd/daily-lifecycle-ops.go

@@ -104,12 +104,12 @@ func startDailyLifecycle() {
 	}
 }
 
-var lifecycleTimeout = newDynamicTimeout(60*time.Second, time.Second)
+var lifecycleLockTimeout = newDynamicTimeout(60*time.Second, time.Second)
 
 func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 	// Lock to avoid concurrent lifecycle ops from other nodes
 	sweepLock := objAPI.NewNSLock(ctx, "system", "daily-lifecycle-ops")
-	if err := sweepLock.GetLock(lifecycleTimeout); err != nil {
+	if err := sweepLock.GetLock(lifecycleLockTimeout); err != nil {
 		return err
 	}
 	defer sweepLock.Unlock()
@@ -133,25 +133,35 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 		}
 		commonPrefix := lcp(prefixes)
 
-		// List all objects and calculate lifecycle action based on object name & object modtime
-		marker := ""
+		// Allocate new results channel to receive ObjectInfo.
+		objInfoCh := make(chan ObjectInfo)
+
+		// Walk through all objects
+		if err := objAPI.Walk(ctx, bucket.Name, commonPrefix, objInfoCh); err != nil {
+			return err
+		}
+
 		for {
-			res, err := objAPI.ListObjects(ctx, bucket.Name, commonPrefix, marker, "", maxObjectList)
-			if err != nil {
-				continue
-			}
 			var objects []string
-			for _, obj := range res.Objects {
+			for obj := range objInfoCh {
+				if len(objects) == maxObjectList {
+					// Reached maximum delete requests, attempt a delete for now.
+					break
+				}
+
 				// Find the action that need to be executed
-				action := l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime)
-				switch action {
-				case lifecycle.DeleteAction:
+				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
 					objects = append(objects, obj.Name)
-				default:
-					// Do nothing, for now.
 				}
 			}
 
+			// Nothing to do.
+			if len(objects) == 0 {
+				break
+			}
+
+			waitForLowHTTPReq(int32(globalEndpoints.Nodes()))
+
 			// Deletes a list of objects.
 			deleteErrs, err := objAPI.DeleteObjects(ctx, bucket.Name, objects)
 			if err != nil {
@@ -173,12 +183,6 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 					})
 				}
 			}
-
-			if !res.IsTruncated {
-				// We are done here, proceed to next bucket.
-				break
-			}
-			marker = res.NextMarker
 		}
 	}
 

cmd/data-usage-cache.go

@@ -0,0 +1,517 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/tinylib/msgp/msgp"
+)
+
+const dataUsageHashLen = 8
+
+//go:generate msgp -file $GOFILE -unexported
+
+// dataUsageHash is the hash type used.
+type dataUsageHash uint64
+
+// sizeHistogram is a size histogram.
+type sizeHistogram [dataUsageBucketLen]uint64
+
+//msgp:tuple dataUsageEntry
+type dataUsageEntry struct {
+	// These fields do no include any children.
+	Size     int64
+	Objects  uint64
+	ObjSizes sizeHistogram
+
+	Children dataUsageHashMap
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageEntryInfo struct {
+	Name   string
+	Parent string
+	Entry  dataUsageEntry
+}
+
+type dataUsageCacheInfo struct {
+	// Name of the bucket. Also root element.
+	Name       string
+	LastUpdate time.Time
+	NextCycle  uint8
+}
+
+// merge other data usage entry into this, excluding children.
+func (e *dataUsageEntry) merge(other dataUsageEntry) {
+	e.Objects += other.Objects
+	e.Size += other.Size
+	for i, v := range other.ObjSizes[:] {
+		e.ObjSizes[i] += v
+	}
+}
+
+// mod returns true if the hash mod cycles == cycle.
+func (h dataUsageHash) mod(cycle uint8, cycles uint8) bool {
+	return uint8(h)%cycles == cycle%cycles
+}
+
+// addChildString will add a child based on its name.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChildString(name string) {
+	e.addChild(hashPath(name))
+}
+
+// addChild will add a child based on its hash.
+// If it already exists it will not be added again.
+func (e *dataUsageEntry) addChild(hash dataUsageHash) {
+	if _, ok := e.Children[hash]; ok {
+		return
+	}
+	if e.Children == nil {
+		e.Children = make(dataUsageHashMap, 1)
+	}
+	e.Children[hash] = struct{}{}
+}
+
+// find a path in the cache.
+// Returns nil if not found.
+func (d *dataUsageCache) find(path string) *dataUsageEntry {
+	due, ok := d.Cache[hashPath(path)]
+	if !ok {
+		return nil
+	}
+	return &due
+}
+
+// dui converts the flattened version of the path to DataUsageInfo.
+func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
+	e := d.find(path)
+	if e == nil {
+		return DataUsageInfo{LastUpdate: UTCNow()}
+	}
+	flat := d.flatten(*e)
+	return DataUsageInfo{
+		LastUpdate:            d.Info.LastUpdate,
+		ObjectsCount:          flat.Objects,
+		ObjectsTotalSize:      uint64(flat.Size),
+		ObjectsSizesHistogram: flat.ObjSizes.asMap(),
+		BucketsCount:          uint64(len(e.Children)),
+		BucketsSizes:          d.pathSizes(buckets),
+	}
+}
+
+// replace will add or replace an entry in the cache.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
+	hash := hashPath(path)
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != "" {
+		phash := hashPath(parent)
+		p := d.Cache[phash]
+		p.addChild(hash)
+		d.Cache[phash] = p
+	}
+}
+
+// replaceHashed add or replaces an entry to the cache based on its hash.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) {
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	d.Cache[hash] = e
+	if parent != nil {
+		p := d.Cache[*parent]
+		p.addChild(hash)
+		d.Cache[*parent] = p
+	}
+}
+
+// StringAll returns a detailed string representation of all entries in the cache.
+func (d *dataUsageCache) StringAll() string {
+	s := fmt.Sprintf("info:%+v\n", d.Info)
+	for k, v := range d.Cache {
+		s += fmt.Sprintf("\t%v: %+v\n", k, v)
+	}
+	return strings.TrimSpace(s)
+}
+
+// String returns a human readable representation of the string.
+func (h dataUsageHash) String() string {
+	return fmt.Sprintf("%x", uint64(h))
+}
+
+// flatten all children of the root into the root element and return it.
+func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
+	for id := range root.Children {
+		e := d.Cache[id]
+		if len(e.Children) > 0 {
+			e = d.flatten(e)
+		}
+		root.merge(e)
+	}
+	root.Children = nil
+	return root
+}
+
+// add a size to the histogram.
+func (h *sizeHistogram) add(size int64) {
+	// Fetch the histogram interval corresponding
+	// to the passed object size.
+	for i, interval := range ObjectsHistogramIntervals {
+		if size >= interval.start && size <= interval.end {
+			h[i]++
+			break
+		}
+	}
+}
+
+// asMap returns the map as a map[string]uint64.
+func (h *sizeHistogram) asMap() map[string]uint64 {
+	res := make(map[string]uint64, 7)
+	for i, count := range h {
+		res[ObjectsHistogramIntervals[i].name] = count
+	}
+	return res
+}
+
+// pathSizes returns the path sizes as a map.
+func (d *dataUsageCache) pathSizes(buckets []BucketInfo) map[string]uint64 {
+	var dst = make(map[string]uint64, len(buckets))
+	for _, bucket := range buckets {
+		e := d.find(bucket.Name)
+		if e == nil {
+			continue
+		}
+		flat := d.flatten(*e)
+		dst[bucket.Name] = uint64(flat.Size)
+	}
+	return dst
+}
+
+// sizeRecursive returns the path as a flattened entry.
+func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
+	root := d.find(path)
+	if root == nil || len(root.Children) == 0 {
+		return root
+	}
+	flat := d.flatten(*root)
+	return &flat
+}
+
+// dataUsageCache contains a cache of data usage entries.
+//msgp:ignore dataUsageCache
+type dataUsageCache struct {
+	Info  dataUsageCacheInfo
+	Cache map[dataUsageHash]dataUsageEntry
+}
+
+// root returns the root of the cache.
+func (d *dataUsageCache) root() *dataUsageEntry {
+	return d.find(d.Info.Name)
+}
+
+// rootHash returns the root of the cache.
+func (d *dataUsageCache) rootHash() dataUsageHash {
+	return hashPath(d.Info.Name)
+}
+
+// clone returns a copy of the cache with no references to the existing.
+func (d *dataUsageCache) clone() dataUsageCache {
+	clone := dataUsageCache{
+		Info:  d.Info,
+		Cache: make(map[dataUsageHash]dataUsageEntry, len(d.Cache)),
+	}
+	for k, v := range d.Cache {
+		clone.Cache[k] = v
+	}
+	return clone
+}
+
+// merge root of other into d.
+// children of root will be flattened before being merged.
+// Last update time will be set to the last updated.
+func (d *dataUsageCache) merge(other dataUsageCache) {
+	existingRoot := d.root()
+	otherRoot := other.root()
+	if existingRoot == nil && otherRoot == nil {
+		return
+	}
+	if otherRoot == nil {
+		return
+	}
+	if existingRoot == nil {
+		*d = other.clone()
+		return
+	}
+	if other.Info.LastUpdate.After(d.Info.LastUpdate) {
+		d.Info.LastUpdate = other.Info.LastUpdate
+	}
+	existingRoot.merge(*otherRoot)
+	eHash := d.rootHash()
+	for key := range otherRoot.Children {
+		entry := other.Cache[key]
+		flat := other.flatten(entry)
+		existing := d.Cache[key]
+		// If not found, merging simply adds.
+		existing.merge(flat)
+		d.replaceHashed(key, &eHash, existing)
+	}
+}
+
+// load the cache content with name from minioMetaBackgroundOpsBucket.
+// Only backend errors are returned as errors.
+// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name string) error {
+	var buf bytes.Buffer
+	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
+	if err != nil {
+		if !isErrObjectNotFound(err) {
+			return toObjectErr(err, dataUsageBucket, name)
+		}
+		*d = dataUsageCache{}
+		return nil
+	}
+	err = d.deserialize(buf.Bytes())
+	if err != nil {
+		*d = dataUsageCache{}
+		logger.LogIf(ctx, err)
+	}
+	return nil
+}
+
+// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
+func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name string) error {
+	b := d.serialize()
+	size := int64(len(b))
+	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)
+	if err != nil {
+		return err
+	}
+
+	_, err = store.PutObject(ctx,
+		dataUsageBucket,
+		name,
+		NewPutObjReader(r, nil, nil),
+		ObjectOptions{})
+	return err
+}
+
+// dataUsageCacheVer indicates the cache version.
+// Bumping the cache version will drop data from previous versions
+// and write new data with the new version.
+const dataUsageCacheVer = 1
+
+// serialize the contents of the cache.
+func (d *dataUsageCache) serialize() []byte {
+	// Alloc pessimistically
+	// dataUsageCacheVer
+	due := dataUsageEntry{}
+	msgLen := 1
+	msgLen += d.Info.Msgsize()
+	// len(d.Cache)
+	msgLen += binary.MaxVarintLen64
+	// Hashes (one for key, assume 1 child/node)
+	msgLen += len(d.Cache) * dataUsageHashLen * 2
+	msgLen += len(d.Cache) * due.Msgsize()
+
+	// Create destination buffer...
+	dst := make([]byte, 0, msgLen)
+
+	var n int
+	tmp := make([]byte, 1024)
+	// byte: version.
+	dst = append(dst, dataUsageCacheVer)
+	// Info...
+	dst, err := d.Info.MarshalMsg(dst)
+	if err != nil {
+		panic(err)
+	}
+	n = binary.PutUvarint(tmp, uint64(len(d.Cache)))
+	dst = append(dst, tmp[:n]...)
+
+	for k, v := range d.Cache {
+		// Put key
+		binary.LittleEndian.PutUint64(tmp[:dataUsageHashLen], uint64(k))
+		dst = append(dst, tmp[:8]...)
+		tmp, err = v.MarshalMsg(tmp[:0])
+		if err != nil {
+			panic(err)
+		}
+		// key, value pairs.
+		dst = append(dst, tmp...)
+
+	}
+	return dst
+}
+
+// deserialize the supplied byte slice into the cache.
+func (d *dataUsageCache) deserialize(b []byte) error {
+	if len(b) < 1 {
+		return io.ErrUnexpectedEOF
+	}
+	switch b[0] {
+	case 1:
+	default:
+		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
+	}
+	b = b[1:]
+
+	// Info...
+	b, err := d.Info.UnmarshalMsg(b)
+	if err != nil {
+		return err
+	}
+	cacheLen, n := binary.Uvarint(b)
+	if n <= 0 {
+		return fmt.Errorf("dataUsageCache: reading cachelen, n <= 0 ")
+	}
+	b = b[n:]
+	d.Cache = make(map[dataUsageHash]dataUsageEntry, cacheLen)
+
+	for i := 0; i < int(cacheLen); i++ {
+		if len(b) <= dataUsageHashLen {
+			return io.ErrUnexpectedEOF
+		}
+		k := binary.LittleEndian.Uint64(b[:dataUsageHashLen])
+		b = b[dataUsageHashLen:]
+		var v dataUsageEntry
+		b, err = v.UnmarshalMsg(b)
+		if err != nil {
+			return err
+		}
+		d.Cache[dataUsageHash(k)] = v
+	}
+	return nil
+}
+
+// Trim this from start+end of hashes.
+var hashPathCutSet = dataUsageRoot
+
+func init() {
+	if dataUsageRoot != string(filepath.Separator) {
+		hashPathCutSet = dataUsageRoot + string(filepath.Separator)
+	}
+}
+
+// hashPath calculates a hash of the provided string.
+func hashPath(data string) dataUsageHash {
+	if data != dataUsageRoot {
+		data = strings.Trim(data, hashPathCutSet)
+	}
+	data = path.Clean(data)
+	return dataUsageHash(xxhash.Sum64String(data))
+}
+
+//msgp:ignore dataUsageEntryInfo
+type dataUsageHashMap map[dataUsageHash]struct{}
+
+// MarshalMsg implements msgp.Marshaler
+func (d dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, d.Msgsize())
+
+	// Write bin header manually
+	const mbin32 uint8 = 0xc6
+	sz := uint32(len(d)) * dataUsageHashLen
+	o = append(o, mbin32, byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz))
+
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		o = append(o, tmp[:]...)
+	}
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (d dataUsageHashMap) Msgsize() (s int) {
+	s = 5 + len(d)*dataUsageHashLen
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (d *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var hashes []byte
+	hashes, bts, err = msgp.ReadBytesZC(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "dataUsageHashMap")
+		return
+	}
+
+	var dst = make(dataUsageHashMap, len(hashes)/dataUsageHashLen)
+	for len(hashes) >= dataUsageHashLen {
+		dst[dataUsageHash(binary.LittleEndian.Uint64(hashes[:dataUsageHashLen]))] = struct{}{}
+		hashes = hashes[dataUsageHashLen:]
+	}
+	*d = dst
+	o = bts
+	return
+}
+
+func (d *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadBytesHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var dst = make(dataUsageHashMap, zb0001)
+	var tmp [8]byte
+	for i := uint32(0); i < zb0001; i++ {
+		_, err = io.ReadFull(dc, tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err, "dataUsageHashMap")
+			return
+		}
+		dst[dataUsageHash(binary.LittleEndian.Uint64(tmp[:]))] = struct{}{}
+	}
+	return nil
+}
+func (d dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteBytesHeader(uint32(len(d)) * dataUsageHashLen)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	var tmp [dataUsageHashLen]byte
+	for k := range d {
+		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
+		_, err = en.Write(tmp[:])
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+	}
+	return nil
+}

cmd/data-usage-cache_gen.go

@@ -0,0 +1,447 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, err = dc.ReadUint8()
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "Name"
+	err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "LastUpdate"
+	err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.LastUpdate)
+	if err != nil {
+		err = msgp.WrapError(err, "LastUpdate")
+		return
+	}
+	// write "NextCycle"
+	err = en.Append(0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(z.NextCycle)
+	if err != nil {
+		err = msgp.WrapError(err, "NextCycle")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "Name"
+	o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "LastUpdate"
+	o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
+	o = msgp.AppendTime(o, z.LastUpdate)
+	// string "NextCycle"
+	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
+	o = msgp.AppendUint8(o, z.NextCycle)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "LastUpdate":
+			z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LastUpdate")
+				return
+			}
+		case "NextCycle":
+			z.NextCycle, bts, err = msgp.ReadUint8Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NextCycle")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageCacheInfo) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint8Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, err = dc.ReadInt64()
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.DecodeMsg(dc)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// array header, size 4
+	err = en.Append(0x94)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.Size)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	err = en.WriteUint64(z.Objects)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		err = en.WriteUint64(z.ObjSizes[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	err = z.Children.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// array header, size 4
+	o = append(o, 0x94)
+	o = msgp.AppendInt64(o, z.Size)
+	o = msgp.AppendUint64(o, z.Objects)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z.ObjSizes {
+		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
+	}
+	o, err = z.Children.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != 4 {
+		err = msgp.ArrayError{Wanted: 4, Got: zb0001}
+		return
+	}
+	z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Objects")
+		return
+	}
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjSizes")
+		return
+	}
+	if zb0002 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
+		return
+	}
+	for za0001 := range z.ObjSizes {
+		z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, "ObjSizes", za0001)
+			return
+		}
+	}
+	bts, err = z.Children.UnmarshalMsg(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Children")
+		return
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntry) Msgsize() (s int) {
+	s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 uint64
+		zb0001, err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteUint64(uint64(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint64(o, uint64(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 uint64
+		zb0001, bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = dataUsageHash(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageHash) Msgsize() (s int) {
+	s = msgp.Uint64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *sizeHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *sizeHistogram) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *sizeHistogram) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *sizeHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *sizeHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size))
+	return
+}

cmd/data-usage-cache_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageCacheInfo(t *testing.T) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageCacheInfo Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageCacheInfo{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageCacheInfo(b *testing.B) {
+	v := dataUsageCacheInfo{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshaldataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageEntry(t *testing.T) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageEntry Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageEntry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageEntry(b *testing.B) {
+	v := dataUsageEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalsizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodesizeHistogram(t *testing.T) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodesizeHistogram Msgsize() is inaccurate")
+	}
+
+	vn := sizeHistogram{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodesizeHistogram(b *testing.B) {
+	v := sizeHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage.go

@@ -20,25 +20,42 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"os"
-	"path/filepath"
-	"sync"
+	"path"
+	"strconv"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
 )
 
 const (
-	dataUsageObjName       = "data-usage"
-	dataUsageCrawlInterval = 12 * time.Hour
+	dataUsageObjName         = ".usage.json"
+	dataUsageCacheName       = ".usage-cache.bin"
+	envDataUsageCrawlConf    = "MINIO_DISK_USAGE_CRAWL_ENABLE"
+	envDataUsageCrawlDelay   = "MINIO_DISK_USAGE_CRAWL_DELAY"
+	envDataUsageCrawlDebug   = "MINIO_DISK_USAGE_CRAWL_DEBUG"
+	dataUsageSleepPerFolder  = 1 * time.Millisecond
+	dataUsageSleepDefMult    = 10.0
+	dataUsageUpdateDirCycles = 16
+	dataUsageRoot            = SlashSeparator
+	dataUsageBucket          = minioMetaBucket + SlashSeparator + bucketMetaPrefix
+	dataUsageStartDelay      = 5 * time.Minute // Time to wait on startup and between cycles.
 )
 
+// initDataUsageStats will start the crawler unless disabled.
 func initDataUsageStats() {
-	go runDataUsageInfoUpdateRoutine()
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+		go runDataUsageInfoUpdateRoutine()
+	}
 }
 
+// runDataUsageInfoUpdateRoutine will contain the main crawler.
 func runDataUsageInfoUpdateRoutine() {
 	// Wait until the object layer is ready
 	var objAPI ObjectLayer
@@ -51,38 +68,16 @@ func runDataUsageInfoUpdateRoutine() {
 		break
 	}
 
-	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
+	runDataUsageInfo(GlobalContext, objAPI)
 }
 
-// timeToNextCrawl returns the duration until next crawl should occur
-// this is validated by verifying the LastUpdate time.
-func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
-	if err != nil {
-		// Upon an error wait for like 10
-		// seconds to start the crawler.
-		return 10 * time.Second
-	}
-	// File indeed doesn't exist when LastUpdate is zero
-	// so we have never crawled, start crawl right away.
-	if dataUsageInfo.LastUpdate.IsZero() {
-		return 1 * time.Second
-	}
-	waitDuration := dataUsageInfo.LastUpdate.Sub(UTCNow())
-	if waitDuration > dataUsageCrawlInterval {
-		// Waited long enough start crawl in a 1 second
-		return 1 * time.Second
-	}
-	// No crawling needed, ask the routine to wait until
-	// the daily interval 12hrs - delta between last update
-	// with current time.
-	return dataUsageCrawlInterval - waitDuration
-}
+var dataUsageLockTimeout = lifecycleLockTimeout
 
-func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
+func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only 1 crawler is running on the cluster.
 	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
 	for {
-		err := locker.GetLock(newDynamicTimeout(time.Millisecond, time.Millisecond))
+		err := locker.GetLock(dataUsageLockTimeout)
 		if err != nil {
 			time.Sleep(5 * time.Minute)
 			continue
@@ -91,47 +86,51 @@ func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan stru
 		// data usage calculator role for its lifetime.
 		break
 	}
-
 	for {
-		wait := timeToCrawl(ctx, objAPI)
 		select {
-		case <-endCh:
+		case <-ctx.Done():
 			locker.Unlock()
 			return
-		case <-time.NewTimer(wait).C:
-			// Crawl only when no previous crawl has occurred,
-			// or its been too long since last crawl.
-			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
+			// Wait before starting next cycle and wait on startup.
+		case <-time.NewTimer(dataUsageStartDelay).C:
+			results := make(chan DataUsageInfo, 1)
+			go storeDataUsageInBackend(ctx, objAPI, results)
+			err := objAPI.CrawlAndGetDataUsage(ctx, results)
+			close(results)
 			logger.LogIf(ctx, err)
 		}
 	}
 }
 
-func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
-	dataUsageJSON, err := json.Marshal(dataUsageInfo)
-	if err != nil {
-		return err
-	}
+// storeDataUsageInBackend will store all objects sent on the gui channel until closed.
+func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
+	for dataUsageInfo := range gui {
+		dataUsageJSON, err := json.Marshal(dataUsageInfo)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		size := int64(len(dataUsageJSON))
+		r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
 
-	size := int64(len(dataUsageJSON))
-	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
-	if err != nil {
-		return err
+		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+		logger.LogIf(ctx, err)
 	}
-
-	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
-	return err
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	var dataUsageInfoJSON bytes.Buffer
 
-	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
+	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
 	if err != nil {
 		if isErrObjectNotFound(err) {
 			return DataUsageInfo{}, nil
 		}
-		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
+		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
 	}
 
 	var dataUsageInfo DataUsageInfo
@@ -152,60 +151,315 @@ type Item struct {
 
 type getSizeFn func(item Item) (int64, error)
 
-func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO func(), getSize getSizeFn) DataUsageInfo {
-	var dataUsageInfo = DataUsageInfo{
-		BucketsSizes:          make(map[string]uint64),
-		ObjectsSizesHistogram: make(map[string]uint64),
+type cachedFolder struct {
+	name   string
+	parent *dataUsageHash
+}
+
+type folderScanner struct {
+	root               string
+	getSize            getSizeFn
+	oldCache           dataUsageCache
+	newCache           dataUsageCache
+	waitForLowActiveIO func()
+
+	dataUsageCrawlMult  float64
+	dataUsageCrawlDebug bool
+
+	newFolders      []cachedFolder
+	existingFolders []cachedFolder
+}
+
+// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
+// sleep is limited to max 1 second.
+func sleepDuration(d time.Duration, x float64) {
+	// Don't sleep for really small amount of time
+	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
+		if d > time.Second {
+			d = time.Second
+		}
+		time.Sleep(d)
+	}
+}
+
+// scanQueuedLevels will scan the provided folders.
+// Files found in the folders will be added to f.newCache.
+// If final is provided folders will be put into f.newFolders or f.existingFolders.
+// If final is not provided the folders found are returned from the function.
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+	var nextFolders []cachedFolder
+	done := ctx.Done()
+	for _, folder := range folders {
+		select {
+		case <-done:
+			return nil, ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+
+		cache := dataUsageEntry{}
+		thisHash := hashPath(folder.name)
+
+		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+			// Parse
+			entName = path.Clean(path.Join(folder.name, entName))
+			bucket, _ := path2BucketObjectWithBasePath(f.root, entName)
+			if bucket == "" {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
+				}
+				return nil
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
+				}
+				return nil
+			}
+
+			select {
+			case <-done:
+				return ctx.Err()
+			default:
+			}
+
+			if typ&os.ModeDir != 0 {
+				h := hashPath(entName)
+				_, exists := f.oldCache.Cache[h]
+				cache.addChildString(entName)
+
+				this := cachedFolder{name: entName, parent: &thisHash}
+				cache.addChild(h)
+				if final {
+					if exists {
+						f.existingFolders = append(f.existingFolders, this)
+					} else {
+						f.newFolders = append(f.newFolders, this)
+					}
+				} else {
+					nextFolders = append(nextFolders, this)
+				}
+				return nil
+			}
+			f.waitForLowActiveIO()
+			// Dynamic time delay.
+			t := UTCNow()
+
+			// Get file size, ignore errors.
+			size, err := f.getSize(Item{Path: path.Join(f.root, entName), Typ: typ})
+
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			if err == errSkipFile {
+				return nil
+			}
+			logger.LogIf(ctx, err)
+			cache.Size += size
+			cache.Objects++
+			cache.ObjSizes.add(size)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		f.newCache.replaceHashed(thisHash, folder.parent, cache)
+	}
+	return nextFolders, nil
+}
+
+// deepScanFolder will deep scan a folder and return the size if no error occurs.
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
+	var cache dataUsageEntry
+
+	done := ctx.Done()
+
+	var addDir func(entName string, typ os.FileMode) error
+	var dirStack = []string{f.root, folder}
+
+	addDir = func(entName string, typ os.FileMode) error {
+		select {
+		case <-done:
+			return ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		if typ&os.ModeDir != 0 {
+			dirStack = append(dirStack, entName)
+			err := readDirFn(path.Join(dirStack...), addDir)
+			dirStack = dirStack[:len(dirStack)-1]
+			sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
+			return err
+		}
+
+		// Dynamic time delay.
+		t := UTCNow()
+
+		// Get file size, ignore errors.
+		dirStack = append(dirStack, entName)
+		fileName := path.Join(dirStack...)
+		dirStack = dirStack[:len(dirStack)-1]
+
+		size, err := f.getSize(Item{Path: fileName, Typ: typ})
+
+		// Don't sleep for really small amount of time
+		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
+		if err == errSkipFile {
+			return nil
+		}
+		logger.LogIf(ctx, err)
+		cache.Size += size
+		cache.Objects++
+		cache.ObjSizes.add(size)
+		return nil
+	}
+	err := readDirFn(path.Join(dirStack...), addDir)
+	if err != nil {
+		return nil, err
+	}
+	return &cache, nil
+}
+
+// updateUsage will crawl the basepath+cache.Info.Name and return an updated cache.
+// The returned cache will always be valid, but may not be updated from the existing.
+// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// If the supplied context is canceled the function will return at the first chance.
+func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+	t := UTCNow()
+
+	dataUsageDebug := env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn
+	defer func() {
+		if dataUsageDebug {
+			logger.Info(color.Green("updateUsage")+" Crawl time at %s: %v", basePath, time.Since(t))
+		}
+	}()
+
+	if cache.Info.Name == "" {
+		cache.Info.Name = dataUsageRoot
 	}
 
-	numWorkers := 4
-	var mutex sync.Mutex // Mutex to update dataUsageInfo
+	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		delayMult = dataUsageSleepDefMult
+	}
 
-	fastWalk(basePath, numWorkers, doneCh, func(path string, typ os.FileMode) error {
-		// Wait for I/O to go down.
-		waitForLowActiveIO()
+	s := folderScanner{
+		root:                basePath,
+		getSize:             getSize,
+		oldCache:            cache,
+		newCache:            dataUsageCache{Info: cache.Info},
+		waitForLowActiveIO:  waitForLowActiveIO,
+		newFolders:          nil,
+		existingFolders:     nil,
+		dataUsageCrawlMult:  delayMult,
+		dataUsageCrawlDebug: dataUsageDebug,
+	}
 
-		bucket, entry := path2BucketObjectWithBasePath(basePath, path)
-		if bucket == "" {
-			return nil
+	if s.dataUsageCrawlDebug {
+		logger.Info(color.Green("runDataUsageInfo:") + " Starting crawler master")
+	}
+
+	done := ctx.Done()
+	var flattenLevels = 3
+
+	// If we are scanning inside a bucket reduce depth by 1.
+	if cache.Info.Name != dataUsageRoot {
+		flattenLevels--
+	}
+
+	var logPrefix, logSuffix string
+	if s.dataUsageCrawlDebug {
+		logPrefix = color.Green("data-usage: ")
+		logSuffix = color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Cycle: %v"+logSuffix, cache.Info.NextCycle)
+	}
+
+	// Always scan flattenLevels deep. Cache root is level 0.
+	todo := []cachedFolder{{name: cache.Info.Name}}
+	for i := 0; i < flattenLevels; i++ {
+		if s.dataUsageCrawlDebug {
+			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
 		}
-
-		if isReservedOrInvalidBucket(bucket, false) {
-			return filepath.SkipDir
-		}
-
-		if entry == "" && typ&os.ModeDir != 0 {
-			mutex.Lock()
-			dataUsageInfo.BucketsCount++
-			dataUsageInfo.BucketsSizes[bucket] = 0
-			mutex.Unlock()
-			return nil
-		}
-
-		mutex.Lock()
-		defer mutex.Unlock()
-
-		if typ&os.ModeDir != 0 {
-			return nil
-		}
-
-		t := time.Now()
-		size, err := getSize(Item{path, typ})
-		// Use the response time of the getSize call to guess system load.
-		// Sleep equivalent time.
-		if d := time.Since(t); d > 100*time.Microsecond {
-			time.Sleep(d)
+		select {
+		case <-done:
+			return cache, ctx.Err()
+		default:
 		}
+		var err error
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
 		if err != nil {
-			return errSkipFile
+			// No useful information...
+			return cache, err
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+	}
+
+	// Add new folders first
+	for _, folder := range s.newFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replace(folder.name, "", *du)
+		// Add to parent manually
+		if folder.parent != nil {
+			parent := s.newCache.Cache[*folder.parent]
+			parent.addChildString(folder.name)
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+	}
+
+	// Do selective scanning of existing folders.
+	for _, folder := range s.existingFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		h := hashPath(folder.name)
+		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
+			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
+			continue
 		}
 
-		dataUsageInfo.ObjectsCount++
-		dataUsageInfo.ObjectsTotalSize += uint64(size)
-		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
-		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
-		return nil
-	})
+		// Update on this cycle...
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replaceHashed(h, folder.parent, *du)
+	}
 
-	return dataUsageInfo
+	s.newCache.Info.LastUpdate = UTCNow()
+	s.newCache.Info.NextCycle++
+	return s.newCache, nil
 }

cmd/data-usage_test.go

@@ -0,0 +1,651 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type usageTestFile struct {
+	name string
+	size int
+}
+
+func TestDataUsageUpdate(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageUpdate")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+
+	got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1322310,
+			flatten: true,
+			objs:    8,
+			oSizes:  sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:    "/dir1/dira",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:    "/dir1/dira/",
+			flatten: true,
+			size:    1300010,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "/dir1/dira",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:   "/dir1/dira/",
+			size:   300000,
+			objs:   2,
+			oSizes: sizeHistogram{0: 0, 1: 2},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    1363315,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    1300010,
+			flatten: true,
+			objs:    4,
+			oSizes:  sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		flatten    bool
+		oSizes     sizeHistogram
+	}{
+		{
+			path:    "/",
+			size:    363515,
+			flatten: true,
+			objs:    14,
+			oSizes:  sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:    "/dir1/dira",
+			size:    300210,
+			objs:    4,
+			flatten: true,
+			oSizes:  sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if w.flatten {
+				*e = got.flatten(*e)
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func TestDataUsageUpdatePrefix(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUpdateUsagePrefix")
+	if err != nil {
+		t.Skip(err)
+	}
+	base = filepath.Join(base, "bucket")
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "bucket/rootfile", size: 10000},
+		{name: "bucket/rootfile2", size: 10000},
+		{name: "bucket/dir1/d1file", size: 2000},
+		{name: "bucket/dir2/d2file", size: 300},
+		{name: "bucket/dir1/dira/dafile", size: 100000},
+		{name: "bucket/dir1/dira/dbfile", size: 200000},
+		{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test dirs
+	var want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1322310,
+			objs:   8,
+			oSizes: sizeHistogram{0: 2, 1: 6},
+		},
+		{
+			path:   "bucket/",
+			size:   20000,
+			objs:   2,
+			oSizes: sizeHistogram{1: 2},
+		},
+		{
+			path:   "bucket/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{1: 1},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:   "bucket/dir1/dira/",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/newfolder/afile",
+			size: 4,
+		},
+		{
+			name: "bucket/newfolder/anotherone",
+			size: 1,
+		},
+		{
+			name: "bucket/newfolder/anemptyone",
+			size: 0,
+		},
+		{
+			name: "bucket/dir1/fileindir1",
+			size: 20000,
+		},
+		{
+			name: "bucket/dir1/dirc/fileindirc",
+			size: 20000,
+		},
+		{
+			name: "bucket/rootfile3",
+			size: 1000,
+		},
+	}
+	createUsageTestFiles(t, base, files)
+	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   1363315,
+			objs:   14,
+			oSizes: sizeHistogram{0: 6, 1: 8},
+		},
+		{
+			path:   "bucket/",
+			size:   21000,
+			objs:   3,
+			oSizes: sizeHistogram{0: 1, 1: 2},
+		},
+		{
+			path:   "bucket/newfolder",
+			size:   5,
+			objs:   3,
+			oSizes: sizeHistogram{0: 3},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
+		},
+		{
+			path:  "bucket/nonexistying",
+			isNil: true,
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+
+	files = []usageTestFile{
+		{
+			name: "bucket/dir1/dira/dirasub/fileindira2",
+			size: 200,
+		},
+	}
+
+	createUsageTestFiles(t, base, files)
+	err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Changed dir must be picked up in this many cycles.
+	for i := 0; i < dataUsageUpdateDirCycles; i++ {
+		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	want = []struct {
+		path       string
+		isNil      bool
+		size, objs int
+		oSizes     sizeHistogram
+	}{
+		{
+			path:   "flat",
+			size:   363515,
+			objs:   14,
+			oSizes: sizeHistogram{0: 7, 1: 7},
+		},
+		{
+			path:   "bucket/dir1/dira",
+			size:   300210,
+			objs:   4,
+			oSizes: sizeHistogram{0: 2, 1: 2},
+		},
+	}
+
+	for _, w := range want {
+		t.Run(w.path, func(t *testing.T) {
+			e := got.find(w.path)
+			if w.path == "flat" {
+				f := got.flatten(*got.root())
+				e = &f
+			}
+			if w.isNil {
+				if e != nil {
+					t.Error("want nil, got", e)
+				}
+				return
+			}
+			if e == nil {
+				t.Fatal("got nil result")
+			}
+			if e.Size != int64(w.size) {
+				t.Error("got size", e.Size, "want", w.size)
+			}
+			if e.Objects != uint64(w.objs) {
+				t.Error("got objects", e.Objects, "want", w.objs)
+			}
+			if e.ObjSizes != w.oSizes {
+				t.Error("got histogram", e.ObjSizes, "want", w.oSizes)
+			}
+		})
+	}
+}
+
+func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) {
+	for _, f := range files {
+		err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestDataUsageCacheSerialize(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDataUsageCacheSerialize")
+	if err != nil {
+		t.Skip(err)
+	}
+	defer os.RemoveAll(base)
+	var files = []usageTestFile{
+		{name: "rootfile", size: 10000},
+		{name: "rootfile2", size: 10000},
+		{name: "dir1/d1file", size: 2000},
+		{name: "dir2/d2file", size: 300},
+		{name: "dir1/dira/dafile", size: 100000},
+		{name: "dir1/dira/dbfile", size: 200000},
+		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+	}
+	createUsageTestFiles(t, base, files)
+
+	getSize := func(item Item) (i int64, err error) {
+		if item.Typ&os.ModeDir == 0 {
+			s, err := os.Stat(item.Path)
+			if err != nil {
+				return 0, err
+			}
+			return s.Size(), nil
+		}
+		return 0, nil
+	}
+	want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	b := want.serialize()
+	var got dataUsageCache
+	err = got.deserialize(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got.Info.LastUpdate.IsZero() {
+		t.Error("lastupdate not set")
+	}
+
+	if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) {
+		t.Fatalf("deserialize mismatch\nwant: %+v\ngot:  %+v", want, got)
+	}
+}

cmd/disk-cache-backend.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -36,7 +37,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/sio"
-	"github.com/ncw/directio"
+	"go.uber.org/atomic"
 )
 
 const (
@@ -44,7 +45,7 @@ const (
 	cacheMetaJSONFile = "cache.json"
 	cacheDataFile     = "part.1"
 	cacheMetaVersion  = "1.0.0"
-
+	cacheExpiryDays   = time.Duration(90 * time.Hour * 24) // defaults to 90 days
 	// SSECacheEncrypted is the metadata key indicating that the object
 	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
 	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
@@ -126,15 +127,15 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 type diskCache struct {
 	dir      string // caching directory
 	quotaPct int    // max usage in %
-	expiry   int    // cache expiry in days
 	// mark false if drive is offline
 	online bool
 	// mutex to protect updates to online variable
-	onlineMutex *sync.RWMutex
-	// purge() listens on this channel to start the cache-purge process
-	purgeChan chan struct{}
-	pool      sync.Pool
-	after     int // minimum accesses before an object is cached.
+	onlineMutex   *sync.RWMutex
+	pool          sync.Pool
+	after         int // minimum accesses before an object is cached.
+	lowWatermark  int
+	highWatermark int
+	gcCounter     atomic.Uint64
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
@@ -142,21 +143,21 @@ type diskCache struct {
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(dir string, expiry int, quotaPct, after int) (*diskCache, error) {
+func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
-		dir:         dir,
-		expiry:      expiry,
-		quotaPct:    quotaPct,
-		after:       after,
-		purgeChan:   make(chan struct{}),
-		online:      true,
-		onlineMutex: &sync.RWMutex{},
+		dir:           dir,
+		quotaPct:      quotaPct,
+		after:         after,
+		lowWatermark:  lowWatermark,
+		highWatermark: highWatermark,
+		online:        true,
+		onlineMutex:   &sync.RWMutex{},
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := directio.AlignedBlock(int(cacheBlkSize))
+				b := disk.AlignedBlock(int(cacheBlkSize))
 				return &b
 			},
 		},
@@ -168,12 +169,12 @@ func newDiskCache(dir string, expiry int, quotaPct, after int) (*diskCache, erro
 	return &cache, nil
 }
 
-// Returns if the disk usage is low.
-// Disk usage is low if usage is < 80% of cacheMaxDiskUsagePct
-// Ex. for a 100GB disk, if maxUsage is configured as 70% then cacheMaxDiskUsagePct is 70G
-// hence disk usage is low if the disk usage is less than 56G (because 80% of 70G is 56G)
+// diskUsageLow() returns true if disk usage falls below the low watermark w.r.t configured cache quota.
+// Ex. for a 100GB disk, if quota is configured as 70%  and watermark_low = 80% and
+// watermark_high = 90% then garbage collection starts when 63% of disk is used and
+// stops when disk usage drops to 56%
 func (c *diskCache) diskUsageLow() bool {
-	minUsage := c.quotaPct * 80 / 100
+	gcStopPct := c.quotaPct * c.lowWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
@@ -182,21 +183,22 @@ func (c *diskCache) diskUsageLow() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < minUsage
+	return int(usedPercent) < gcStopPct
 }
 
-// Return if the disk usage is high.
-// Disk usage is high if disk used is > cacheMaxDiskUsagePct
+// Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
+// gc starts if high water mark reached.
 func (c *diskCache) diskUsageHigh() bool {
+	gcTriggerPct := c.quotaPct * c.highWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
 		logger.LogIf(ctx, err)
-		return true
+		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) > c.quotaPct
+	return int(usedPercent) >= gcTriggerPct
 }
 
 // Returns if size space can be allocated without exceeding
@@ -213,12 +215,42 @@ func (c *diskCache) diskAvailable(size int64) bool {
 	return int(usedPercent) < c.quotaPct
 }
 
+// toClear returns how many bytes should be cleared to reach the low watermark quota.
+// returns 0 if below quota.
+func (c *diskCache) toClear() uint64 {
+	di, err := disk.GetInfo(c.dir)
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		return 0
+	}
+	return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark))
+}
+
 // Purge cache entries that were not accessed.
-func (c *diskCache) purge() {
+func (c *diskCache) purge(ctx context.Context, doneCh <-chan struct{}) {
+	if c.diskUsageLow() {
+		return
+	}
+	toFree := c.toClear()
+	if toFree == 0 {
+		return
+	}
+	// expiry for cleaning up old cache.json files that
+	// need to be cleaned up.
+	expiry := UTCNow().Add(-cacheExpiryDays)
+	// defaulting max hits count to 100
+	scorer, err := newFileScorer(int64(toFree), time.Now().Unix(), 100)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+
 	// this function returns FileInfo for cached range files and cache data file.
 	fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
 		fm := make(map[string]os.FileInfo)
-		fname := pathJoin(pathPrefix, cacheDataFile)
+		fname := pathJoin(pathPrefix, dataFile)
 		if fi, err := os.Stat(fname); err == nil {
 			fm[fname] = fi
 		}
@@ -231,63 +263,73 @@ func (c *diskCache) purge() {
 		}
 		return fm
 	}
-	ctx := context.Background()
-	for {
-		olderThan := c.expiry * 24
-		for !c.diskUsageLow() {
-			// delete unaccessed objects older than expiry duration
-			expiry := UTCNow().Add(time.Hour * time.Duration(-1*olderThan))
-			olderThan /= 2
-			if olderThan < 1 {
-				break
-			}
-			deletedCount := 0
+	objDirs, err := ioutil.ReadDir(c.dir)
+	if err != nil {
+		log.Fatal(err)
+	}
 
-			objDirs, err := ioutil.ReadDir(c.dir)
-			if err != nil {
-				log.Fatal(err)
-			}
+	for _, obj := range objDirs {
+		if obj.Name() == minioMetaBucket {
+			continue
+		}
 
-			for _, obj := range objDirs {
-				if obj.Name() == minioMetaBucket {
-					continue
-				}
-				meta, _, _, err := c.statCachedMeta(context.Background(), pathJoin(c.dir, obj.Name()))
-				if err != nil {
-					// delete any partially filled cache entry left behind.
-					removeAll(pathJoin(c.dir, obj.Name()))
-					continue
-				}
-				// stat all cached file ranges and cacheDataFile.
-				fis := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
-				objInfo := meta.ToObjectInfo("", "")
-				cc := cacheControlOpts(objInfo)
-
-				for fname, fi := range fis {
-					if atime.Get(fi).Before(expiry) ||
-						cc.isStale(objInfo.ModTime) {
-						if err = removeAll(fname); err != nil {
-							logger.LogIf(ctx, err)
-						}
-						deletedCount++
-						// break early if sufficient disk space reclaimed.
-						if !c.diskUsageLow() {
-							break
-						}
+		cacheDir := pathJoin(c.dir, obj.Name())
+		meta, _, numHits, err := c.statCachedMeta(ctx, cacheDir)
+		if err != nil {
+			// delete any partially filled cache entry left behind.
+			removeAll(cacheDir)
+			continue
+		}
+		// stat all cached file ranges and cacheDataFile.
+		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
+		objInfo := meta.ToObjectInfo("", "")
+		cc := cacheControlOpts(objInfo)
+		for fname, fi := range cachedFiles {
+			if cc != nil {
+				if cc.isStale(objInfo.ModTime) {
+					if err = removeAll(fname); err != nil {
+						logger.LogIf(ctx, err)
+					}
+					scorer.adjustSaveBytes(-fi.Size())
+					// break early if sufficient disk space reclaimed.
+					if c.diskUsageLow() {
+						return
 					}
 				}
+				continue
 			}
-			if deletedCount == 0 {
-				break
-			}
+			scorer.addFile(fname, atime.Get(fi), fi.Size(), numHits)
 		}
-		for {
-			<-c.purgeChan
-			if c.diskUsageHigh() {
-				break
-			}
+		// clean up stale cache.json files for objects that never got cached but access count was maintained in cache.json
+		fi, err := os.Stat(pathJoin(cacheDir, cacheMetaJSONFile))
+		if err != nil || (fi.ModTime().Before(expiry) && len(cachedFiles) == 0) {
+			removeAll(cacheDir)
+			scorer.adjustSaveBytes(-fi.Size())
+			continue
+		}
+		if c.diskUsageLow() {
+			return
 		}
 	}
+	for _, path := range scorer.fileNames() {
+		removeAll(path)
+		slashIdx := strings.LastIndex(path, SlashSeparator)
+		pathPrefix := path[0:slashIdx]
+		fname := path[slashIdx+1:]
+		if fname == cacheDataFile {
+			removeAll(pathPrefix)
+		}
+	}
+}
+
+func (c *diskCache) incGCCounter() {
+	c.gcCounter.Add(uint64(1))
+}
+func (c *diskCache) resetGCCounter() {
+	c.gcCounter.Store(uint64(0))
+}
+func (c *diskCache) gcCount() uint64 {
+	return c.gcCounter.Load()
 }
 
 // sets cache drive status
@@ -378,6 +420,9 @@ func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HT
 	if !ok {
 		return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
 	}
+	if _, err = os.Stat(pathJoin(cacheObjPath, rngFile)); err != nil {
+		return oi, rngInfo, numHits, ObjectNotFound{Bucket: bucket, Object: object}
+	}
 	rngInfo = RangeInfo{Range: rng, File: rngFile, Size: int64(actualRngSize)}
 
 	err = decryptCacheObjectETag(&oi)
@@ -568,10 +613,8 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
 	if c.diskUsageHigh() {
-		select {
-		case c.purgeChan <- struct{}{}:
-		default:
-		}
+		c.incGCCounter()
+		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
@@ -622,11 +665,11 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	if IsErr(err, baseErrs...) {
 		c.setOnline(false)
 	}
-
 	if err != nil {
 		removeAll(cachePath)
 		return err
 	}
+
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
 		return IncompleteBody{}

cmd/disk-cache-utils.go

@@ -17,8 +17,12 @@
 package cmd
 
 import (
+	"container/list"
 	"encoding/hex"
+	"errors"
+	"fmt"
 	"io"
+	"math"
 	"os"
 	"path"
 	"strconv"
@@ -57,13 +61,8 @@ type cacheControl struct {
 	noCache      bool
 }
 
-func (c cacheControl) isEmpty() bool {
-	return c == cacheControl{}
-
-}
-
-func (c cacheControl) isStale(modTime time.Time) bool {
-	if c.isEmpty() {
+func (c *cacheControl) isStale(modTime time.Time) bool {
+	if c == nil {
 		return false
 	}
 	// response will never be stale if only-if-cached is set
@@ -100,7 +99,8 @@ func (c cacheControl) isStale(modTime time.Time) bool {
 }
 
 // returns struct with cache-control settings from user metadata.
-func cacheControlOpts(o ObjectInfo) (c cacheControl) {
+func cacheControlOpts(o ObjectInfo) *cacheControl {
+	c := cacheControl{}
 	m := o.UserDefined
 	if o.Expires != timeSentinel {
 		c.expiry = o.Expires
@@ -114,7 +114,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 
 	}
 	if headerVal == "" {
-		return
+		return nil
 	}
 	headerVal = strings.ToLower(headerVal)
 	headerVal = strings.TrimSpace(headerVal)
@@ -146,7 +146,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			p[0] == "max-stale" {
 			i, err := strconv.Atoi(p[1])
 			if err != nil {
-				return cacheControl{}
+				return nil
 			}
 			if p[0] == "max-age" {
 				c.maxAge = i
@@ -162,7 +162,7 @@ func cacheControlOpts(o ObjectInfo) (c cacheControl) {
 			}
 		}
 	}
-	return c
+	return &c
 }
 
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
@@ -283,3 +283,161 @@ func isMetadataSame(m1, m2 map[string]string) bool {
 	}
 	return true
 }
+
+type fileScorer struct {
+	saveBytes int64
+	now       int64
+	maxHits   int
+	// 1/size for consistent score.
+	sizeMult float64
+
+	// queue is a linked list of files we want to delete.
+	// The list is kept sorted according to score, highest at top, lowest at bottom.
+	queue       list.List
+	queuedBytes int64
+}
+
+type queuedFile struct {
+	name  string
+	size  int64
+	score float64
+}
+
+// newFileScorer allows to collect files to save a specific number of bytes.
+// Each file is assigned a score based on its age, size and number of hits.
+// A list of files is maintained
+func newFileScorer(saveBytes int64, now int64, maxHits int) (*fileScorer, error) {
+	if saveBytes <= 0 {
+		return nil, errors.New("newFileScorer: saveBytes <= 0")
+	}
+	if now < 0 {
+		return nil, errors.New("newFileScorer: now < 0")
+	}
+	if maxHits <= 0 {
+		return nil, errors.New("newFileScorer: maxHits <= 0")
+	}
+	f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
+	f.queue.Init()
+	return &f, nil
+}
+
+func (f *fileScorer) addFile(name string, lastAccess time.Time, size int64, hits int) {
+	// Calculate how much we want to delete this object.
+	file := queuedFile{
+		name: name,
+		size: size,
+	}
+	score := float64(f.now - lastAccess.Unix())
+	// Size as fraction of how much we want to save, 0->1.
+	szWeight := math.Max(0, (math.Min(1, float64(size)*f.sizeMult)))
+	// 0 at f.maxHits, 1 at 0.
+	hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
+	file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
+	// If we still haven't saved enough, just add the file
+	if f.queuedBytes < f.saveBytes {
+		f.insertFile(file)
+		f.trimQueue()
+		return
+	}
+	// If we score less than the worst, don't insert.
+	worstE := f.queue.Back()
+	if worstE != nil && file.score < worstE.Value.(queuedFile).score {
+		return
+	}
+	f.insertFile(file)
+	f.trimQueue()
+}
+
+// adjustSaveBytes allows to adjust the number of bytes to save.
+// This can be used to adjust the count on the fly.
+// Returns true if there still is a need to delete files (saveBytes >0),
+// false if no more bytes needs to be saved.
+func (f *fileScorer) adjustSaveBytes(n int64) bool {
+	f.saveBytes += n
+	if f.saveBytes <= 0 {
+		f.queue.Init()
+		f.saveBytes = 0
+		return false
+	}
+	if n < 0 {
+		f.trimQueue()
+	}
+	return true
+}
+
+// insertFile will insert a file into the list, sorted by its score.
+func (f *fileScorer) insertFile(file queuedFile) {
+	e := f.queue.Front()
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if v.score < file.score {
+			break
+		}
+		e = e.Next()
+	}
+	f.queuedBytes += file.size
+	// We reached the end.
+	if e == nil {
+		f.queue.PushBack(file)
+		return
+	}
+	f.queue.InsertBefore(file, e)
+}
+
+// trimQueue will trim the back of queue and still keep below wantSave.
+func (f *fileScorer) trimQueue() {
+	for {
+		e := f.queue.Back()
+		if e == nil {
+			return
+		}
+		v := e.Value.(queuedFile)
+		if f.queuedBytes-v.size < f.saveBytes {
+			return
+		}
+		f.queue.Remove(e)
+		f.queuedBytes -= v.size
+	}
+}
+
+// fileNames returns all queued file names.
+func (f *fileScorer) fileNames() []string {
+	res := make([]string, 0, f.queue.Len())
+	e := f.queue.Front()
+	for e != nil {
+		res = append(res, e.Value.(queuedFile).name)
+		e = e.Next()
+	}
+	return res
+}
+
+func (f *fileScorer) reset() {
+	f.queue.Init()
+	f.queuedBytes = 0
+}
+
+func (f *fileScorer) queueString() string {
+	var res strings.Builder
+	e := f.queue.Front()
+	i := 0
+	for e != nil {
+		v := e.Value.(queuedFile)
+		if i > 0 {
+			res.WriteByte('\n')
+		}
+		res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
+		i++
+		e = e.Next()
+	}
+	return res.String()
+}
+
+// bytesToClear() returns the number of bytes to clear to reach low watermark
+// w.r.t quota given disk total and free space, quota in % allocated to cache
+// and low watermark % w.r.t allowed quota.
+func bytesToClear(total, free int64, quotaPct, lowWatermark uint64) uint64 {
+	used := (total - free)
+	quotaAllowed := total * (int64)(quotaPct) / 100
+	lowWMUsage := (total * (int64)(lowWatermark*quotaPct) / (100 * 100))
+	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
+}

cmd/disk-cache-utils_test.go

@@ -29,16 +29,16 @@ func TestGetCacheControlOpts(t *testing.T) {
 	testCases := []struct {
 		cacheControlHeaderVal string
 		expiryHeaderVal       time.Time
-		expectedCacheControl  cacheControl
+		expectedCacheControl  *cacheControl
 		expectedErr           bool
 	}{
-		{"", timeSentinel, cacheControl{}, false},
-		{"max-age=2592000, public", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"max-age=2592000, no-store", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
-		{"must-revalidate, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", expiry, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
-		{"s-maxAge=2500, max-age=600s", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
+		{"", timeSentinel, nil, false},
+		{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
+		{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
+		{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
 	}
 
 	for _, testCase := range testCases {
@@ -49,7 +49,7 @@ func TestGetCacheControlOpts(t *testing.T) {
 				m["expires"] = testCase.expiryHeaderVal.String()
 			}
 			c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
-			if testCase.expectedErr && (c != cacheControl{}) {
+			if testCase.expectedErr && (c != nil) {
 				t.Errorf("expected err, got <nil>")
 			}
 			if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
@@ -83,3 +83,90 @@ func TestIsMetadataSame(t *testing.T) {
 		}
 	}
 }
+
+func TestNewFileScorer(t *testing.T) {
+	fs, err := newFileScorer(1000, time.Now().Unix(), 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fs.fileNames()) != 0 {
+		t.Fatal("non zero files??")
+	}
+	now := time.Now()
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	fs.addFile("older", now.Add(-time.Hour), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("less", now.Add(-time.Minute), 1000, 5)
+	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.reset()
+	fs.addFile("small", now.Add(-time.Minute), 200, 10)
+	fs.addFile("medium", now.Add(-time.Minute), 300, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	fs.addFile("large", now.Add(-time.Minute), 700, 10)
+	fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+
+	fs.reset()
+	fs.addFile("less", now.Add(-time.Minute), 500, 5)
+	fs.addFile("recent", now.Add(-time.Minute), 500, 10)
+	if !fs.adjustSaveBytes(-500) {
+		t.Fatal("we should still need more bytes, got false")
+	}
+	// We should only need 500 bytes now.
+	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+	if fs.adjustSaveBytes(-500) {
+		t.Fatal("we shouldn't need any more bytes, got true")
+	}
+	fs, err = newFileScorer(1000, time.Now().Unix(), 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
+	// sorting should be consistent after adjusting savebytes.
+	fs.adjustSaveBytes(-800)
+	fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
+	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
+		t.Fatal("unexpected file list", fs.queueString())
+	}
+}
+func TestBytesToClear(t *testing.T) {
+	testCases := []struct {
+		total        int64
+		free         int64
+		quotaPct     uint64
+		watermarkLow uint64
+		expected     uint64
+	}{
+		{1000, 800, 40, 90, 0},
+		{1000, 200, 40, 90, 400},
+		{1000, 400, 40, 90, 240},
+		{1000, 600, 40, 90, 40},
+		{1000, 600, 40, 70, 120},
+		{1000, 1000, 90, 70, 0},
+		{1000, 0, 90, 70, 370},
+	}
+	for i, tc := range testCases {
+		toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow)
+		if tc.expected != toClear {
+			t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
+		}
+	}
+}

cmd/disk-cache.go

@@ -38,7 +38,8 @@ import (
 )
 
 const (
-	cacheBlkSize = int64(1 * 1024 * 1024)
+	cacheBlkSize    = int64(1 * 1024 * 1024)
+	cacheGCInterval = time.Minute * 30
 )
 
 // CacheStorageInfo - represents total, free capacity of
@@ -174,7 +175,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
-	var cc cacheControl
+	var cc *cacheControl
 	var cacheObjSize int64
 	// fetch diskCache if object is currently cached or nearest available cache drive
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
@@ -191,8 +192,8 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			}
 		}
 		cc = cacheControlOpts(cacheReader.ObjInfo)
-		if (!cc.isEmpty() && !cc.isStale(cacheReader.ObjInfo.ModTime)) ||
-			cc.onlyIfCached {
+		if cc != nil && (!cc.isStale(cacheReader.ObjInfo.ModTime) ||
+			cc.onlyIfCached) {
 			// This is a cache hit, mark it so
 			bytesServed := cacheReader.ObjInfo.Size
 			if rs != nil {
@@ -205,7 +206,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			c.incHitsToMeta(ctx, dcache, bucket, object, cacheReader.ObjInfo.Size, cacheReader.ObjInfo.ETag)
 			return cacheReader, nil
 		}
-		if cc.noStore {
+		if cc != nil && cc.noStore {
 			c.cacheStats.incMiss()
 			bReader, err := c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 			bReader.ObjInfo.CacheLookupStatus = CacheHit
@@ -259,11 +260,8 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	c.cacheStats.incMiss()
 	// Since we got here, we are serving the request from backend,
 	// and also adding the object to the cache.
-	if !dcache.diskUsageLow() {
-		select {
-		case dcache.purgeChan <- struct{}{}:
-		default:
-		}
+	if dcache.diskUsageHigh() {
+		dcache.incGCCounter()
 	}
 
 	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
@@ -330,12 +328,12 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 	if err != nil {
 		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
-	var cc cacheControl
+	var cc *cacheControl
 	// if cache control setting is valid, avoid HEAD operation to backend
 	cachedObjInfo, _, cerr := dcache.Stat(ctx, bucket, object)
 	if cerr == nil {
 		cc = cacheControlOpts(cachedObjInfo)
-		if !cc.isStale(cachedObjInfo.ModTime) {
+		if cc == nil || (cc != nil && !cc.isStale(cachedObjInfo.ModTime)) {
 			// This is a cache hit, mark it so
 			c.cacheStats.incHit()
 			return cachedObjInfo, nil
@@ -392,7 +390,7 @@ func (c *cacheObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dst
 	// if currently cached, evict old entry and revert to backend.
 	if cachedObjInfo, _, cerr := dcache.Stat(ctx, srcBucket, srcObject); cerr == nil {
 		cc := cacheControlOpts(cachedObjInfo)
-		if !cc.isStale(cachedObjInfo.ModTime) {
+		if cc == nil || !cc.isStale(cachedObjInfo.ModTime) {
 			dcache.Delete(ctx, srcBucket, srcObject)
 		}
 	}
@@ -522,15 +520,10 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		if quota == 0 {
 			quota = config.Quota
 		}
-
-		cache, err := newDiskCache(dir, config.Expiry, quota, config.After)
+		cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
 		if err != nil {
 			return nil, false, err
 		}
-		// Start the purging go-routine for entries that have expired if no migration in progress
-		if !migrating {
-			go cache.purge()
-		}
 		caches = append(caches, cache)
 	}
 	return caches, migrating, nil
@@ -577,13 +570,12 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	}
 
 	errCnt := 0
-	for index, err := range g.Wait() {
+	for _, err := range g.Wait() {
 		if err != nil {
 			errCnt++
 			logger.LogIf(ctx, err)
 			continue
 		}
-		go c.cache[index].purge()
 	}
 
 	if errCnt > 0 {
@@ -697,5 +689,35 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 	if migrateSw {
 		go c.migrateCacheFromV1toV2(ctx)
 	}
+	go c.gc(ctx, GlobalServiceDoneCh)
 	return c, nil
 }
+
+func (c *cacheObjects) gc(ctx context.Context, doneCh <-chan struct{}) {
+	ticker := time.NewTicker(cacheGCInterval)
+
+	defer ticker.Stop()
+	for {
+		select {
+		case <-doneCh:
+			return
+		case <-ticker.C:
+			if c.migrating {
+				continue
+			}
+			var wg sync.WaitGroup
+			for _, dcache := range c.cache {
+				if dcache.gcCount() == 0 {
+					continue
+				}
+				wg.Add(1)
+				go func(d *diskCache) {
+					defer wg.Done()
+					d.resetGCCounter()
+					d.purge(ctx, doneCh)
+				}(dcache)
+			}
+			wg.Wait()
+		}
+	}
+}

cmd/disk-cache_test.go

@@ -27,15 +27,15 @@ import (
 )
 
 // Initialize cache objects.
-func initCacheObjects(disk string, cacheMaxUse, cacheAfter int) (*diskCache, error) {
-	return newDiskCache(disk, 80, cacheMaxUse, cacheAfter)
+func initCacheObjects(disk string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int) (*diskCache, error) {
+	return newDiskCache(disk, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, cacheMaxUse, cacheAfter int, t *testing.T) ([]*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh int, t *testing.T) ([]*diskCache, error) {
 	var cb []*diskCache
 	for _, d := range drives {
-		obj, err := initCacheObjects(d, cacheMaxUse, cacheAfter)
+		obj, err := initCacheObjects(d, cacheMaxUse, cacheAfter, cacheWatermarkLow, cacheWatermarkHigh)
 		if err != nil {
 			return nil, err
 		}
@@ -70,7 +70,7 @@ func TestGetCachedLoc(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, 100, 1, t)
+		d, err := initDiskCaches(fsDirs, 100, 1, 80, 90, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -109,7 +109,7 @@ func TestGetCacheMaxUse(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, 80, 1, t)
+		d, err := initDiskCaches(fsDirs, 80, 1, 80, 90, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -180,7 +180,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, 100, 0, t)
+	d, err := initDiskCaches(fsDirs, 100, 0, 80, 90, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -258,7 +258,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, 80, 0, t)
+	d, err := initDiskCaches(fsDirs, 90, 0, 90, 99, t)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/encryption-v1_test.go

@@ -280,15 +280,11 @@ func TestGetDecryptedRange_Issue50(t *testing.T) {
 		Parts: []ObjectPartInfo{
 			{
 				Number:     1,
-				Name:       "part.1",
-				ETag:       "etag1",
 				Size:       297580380,
 				ActualSize: 297435132,
 			},
 			{
 				Number:     2,
-				Name:       "part.2",
-				ETag:       "etag2",
 				Size:       297580380,
 				ActualSize: 297435132,
 			},

cmd/endpoint-ellipses.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -108,12 +108,24 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		return ss
 	}
 
+	setCounts := possibleSetCounts(commonSize)
+	if len(setCounts) == 0 {
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
+		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
+	}
+
 	if customSetDriveCount > 0 {
-		msg := fmt.Sprintf("Invalid set drive count, leads to non-uniform distribution for the given number of disks. Possible values for custom set count are %d", possibleSetCounts(setSize))
+		msg := fmt.Sprintf("Invalid set drive count. Acceptable values for %d number drives are %d", commonSize, setCounts)
 		if customSetDriveCount > setSize {
 			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
-		if setSize%customSetDriveCount != 0 {
+		var found bool
+		for _, ss := range setCounts {
+			if ss == customSetDriveCount {
+				found = true
+			}
+		}
+		if !found {
 			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
 		setSize = customSetDriveCount
@@ -121,7 +133,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		msg := fmt.Sprintf("Incorrect number of endpoints provided %s", args)
+		msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes)
 		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 
@@ -261,9 +273,6 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, -1, config.ErrInvalidErasureSetSize(err)
 		}
-		if !isValidSetSize(uint64(setDriveCount)) {
-			return nil, -1, -1, config.ErrInvalidErasureSetSize(nil)
-		}
 	}
 
 	if !ellipses.HasEllipses(args...) {

cmd/endpoint-ellipses_test.go

@@ -104,6 +104,36 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
 			8,
 			true,
 		},
+		{
+			[]string{"http://host{1...12}/data{1...12}"},
+			[]uint64{144},
+			[][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16}},
+			16,
+			true,
+		},
+		{
+			[]string{"http://host{0...5}/data{1...28}"},
+			[]uint64{168},
+			[][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}},
+			12,
+			true,
+		},
+		// Incorrect custom set drive count.
+		{
+			[]string{"http://host{0...5}/data{1...28}"},
+			[]uint64{168},
+			nil,
+			10,
+			false,
+		},
+		// Failure not divisible number of disks.
+		{
+			[]string{"http://host{1...11}/data{1...11}"},
+			[]uint64{121},
+			nil,
+			11,
+			false,
+		},
 		{
 			[]string{"data{1...60}"},
 			nil,

cmd/fastwalk.go

@@ -11,173 +11,12 @@ package cmd
 import (
 	"errors"
 	"os"
-	"path/filepath"
 	"strings"
-	"sync"
 )
 
 var errSkipFile = errors.New("fastwalk: skip this file")
 
-// Walk is a faster implementation of filepath.Walk.
-//
-// filepath.Walk's design necessarily calls os.Lstat on each file,
-// even if the caller needs less info.
-// Many tools need only the type of each file.
-// On some platforms, this information is provided directly by the readdir
-// system call, avoiding the need to stat each file individually.
-// fastwalk_unix.go contains a fork of the syscall routines.
-//
-// See golang.org/issue/16399
-//
-// Walk walks the file tree rooted at root, calling walkFn for
-// each file or directory in the tree, including root.
-//
-// If fastWalk returns filepath.SkipDir, the directory is skipped.
-//
-// Unlike filepath.Walk:
-//   * file stat calls must be done by the user.
-//     The only provided metadata is the file type, which does not include
-//     any permission bits.
-//   * multiple goroutines stat the filesystem concurrently. The provided
-//     walkFn must be safe for concurrent use.
-//   * fastWalk can follow symlinks if walkFn returns the TraverseLink
-//     sentinel error. It is the walkFn's responsibility to prevent
-//     fastWalk from going into symlink cycles.
-func fastWalk(root string, nworkers int, doneCh <-chan struct{}, walkFn func(path string, typ os.FileMode) error) error {
-
-	// Make sure to wait for all workers to finish, otherwise
-	// walkFn could still be called after returning. This Wait call
-	// runs after close(e.donec) below.
-	var wg sync.WaitGroup
-	defer wg.Wait()
-
-	w := &walker{
-		fn:       walkFn,
-		enqueuec: make(chan walkItem, nworkers), // buffered for performance
-		workc:    make(chan walkItem, nworkers), // buffered for performance
-		donec:    make(chan struct{}),
-
-		// buffered for correctness & not leaking goroutines:
-		resc: make(chan error, nworkers),
-	}
-	defer close(w.donec)
-
-	for i := 0; i < nworkers; i++ {
-		wg.Add(1)
-		go w.doWork(&wg)
-	}
-
-	todo := []walkItem{{dir: root}}
-	out := 0
-	for {
-		workc := w.workc
-		var workItem walkItem
-		if len(todo) == 0 {
-			workc = nil
-		} else {
-			workItem = todo[len(todo)-1]
-		}
-		select {
-		case <-doneCh:
-			return nil
-		case workc <- workItem:
-			todo = todo[:len(todo)-1]
-			out++
-		case it := <-w.enqueuec:
-			todo = append(todo, it)
-		case err := <-w.resc:
-			out--
-			if err != nil {
-				return err
-			}
-			if out == 0 && len(todo) == 0 {
-				// It's safe to quit here, as long as the buffered
-				// enqueue channel isn't also readable, which might
-				// happen if the worker sends both another unit of
-				// work and its result before the other select was
-				// scheduled and both w.resc and w.enqueuec were
-				// readable.
-				select {
-				case it := <-w.enqueuec:
-					todo = append(todo, it)
-				default:
-					return nil
-				}
-			}
-		}
-	}
-}
-
-// doWork reads directories as instructed (via workc) and runs the
-// user's callback function.
-func (w *walker) doWork(wg *sync.WaitGroup) {
-	defer wg.Done()
-	for {
-		select {
-		case <-w.donec:
-			return
-		case it := <-w.workc:
-			select {
-			case <-w.donec:
-				return
-			case w.resc <- w.walk(it.dir, !it.callbackDone):
-			}
-		}
-	}
-}
-
-type walker struct {
-	fn func(path string, typ os.FileMode) error
-
-	donec    chan struct{} // closed on fastWalk's return
-	workc    chan walkItem // to workers
-	enqueuec chan walkItem // from workers
-	resc     chan error    // from workers
-}
-
-type walkItem struct {
-	dir          string
-	callbackDone bool // callback already called; don't do it again
-}
-
-func (w *walker) enqueue(it walkItem) {
-	select {
-	case w.enqueuec <- it:
-	case <-w.donec:
-	}
-}
-
-var stringsBuilderPool = sync.Pool{
-	New: func() interface{} {
-		return &strings.Builder{}
-	},
-}
-
-func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
-	builder := stringsBuilderPool.Get().(*strings.Builder)
-	defer func() {
-		builder.Reset()
-		stringsBuilderPool.Put(builder)
-	}()
-
-	builder.WriteString(dirName)
-	if !strings.HasSuffix(dirName, SlashSeparator) {
-		builder.WriteString(SlashSeparator)
-	}
-	builder.WriteString(baseName)
-	if typ == os.ModeDir {
-		w.enqueue(walkItem{dir: builder.String()})
-		return nil
-	}
-
-	err := w.fn(builder.String(), typ)
-	if err == filepath.SkipDir || err == errSkipFile {
-		return nil
-	}
-	return err
-}
-
-func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error {
+func readDirFn(dirName string, fn func(entName string, typ os.FileMode) error) error {
 	fis, err := readDir(dirName)
 	if err != nil {
 		return err
@@ -188,23 +27,9 @@ func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode)
 			mode |= os.ModeDir
 		}
 
-		if err = fn(dirName, fi, mode); err != nil {
+		if err = fn(fi, mode); err != nil {
 			return err
 		}
 	}
 	return nil
 }
-
-func (w *walker) walk(root string, runUserCallback bool) error {
-	if runUserCallback {
-		err := w.fn(root, os.ModeDir)
-		if err == filepath.SkipDir || err == errSkipFile {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-	}
-
-	return readDirFn(root, w.onDirEnt)
-}

cmd/format-xl.go

@@ -426,7 +426,7 @@ func checkFormatXLValue(formatXL *formatXLV3) error {
 }
 
 // Check all format values.
-func checkFormatXLValues(formats []*formatXLV3) error {
+func checkFormatXLValues(formats []*formatXLV3, drivesPerSet int) error {
 	for i, formatXL := range formats {
 		if formatXL == nil {
 			continue
@@ -438,6 +438,9 @@ func checkFormatXLValues(formats []*formatXLV3) error {
 			return fmt.Errorf("%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)",
 				humanize.Ordinal(i+1), len(formats), humanize.Ordinal(i+1), len(formatXL.XL.Sets)*len(formatXL.XL.Sets[0]))
 		}
+		if len(formatXL.XL.Sets[0]) != drivesPerSet {
+			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatXL.XL.Sets[0]), drivesPerSet)
+		}
 	}
 	return nil
 }
@@ -825,7 +828,7 @@ func ecDrivesNoConfig(drivesPerSet int) int {
 // Make XL backend meta volumes.
 func makeFormatXLMetaVolumes(disk StorageAPI) error {
 	// Attempt to create MinIO internal buckets.
-	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, minioMetaBackgroundOpsBucket)
+	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
 }
 
 var initMetaVolIgnoredErrs = append(baseIgnoredErrs, errVolumeExists)

cmd/fs-v1-helpers.go

@@ -429,7 +429,7 @@ func fsDeleteFile(ctx context.Context, basePath, deletePath string) error {
 		return err
 	}
 
-	if err := deleteFile(basePath, deletePath); err != nil {
+	if err := deleteFile(basePath, deletePath, false); err != nil {
 		if err != errFileNotFound {
 			logger.LogIf(ctx, err)
 		}

cmd/fs-v1-multipart.go

@@ -575,7 +575,6 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 		fsMeta.Parts[i] = ObjectPartInfo{
 			Number:     part.PartNumber,
-			ETag:       part.ETag,
 			Size:       fi.Size(),
 			ActualSize: actualSize,
 		}
@@ -758,7 +757,7 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
-func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/fs-v1-multipart_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 )
@@ -34,33 +35,38 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	obj := initFSObjects(disk, t)
 	fs := obj.(*FSObjects)
 
-	// Close the go-routine, we are going to
-	// manually start it and test in this test case.
-	GlobalServiceDoneCh <- struct{}{}
-
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{})
+	// Create a context we can cancel.
+	ctx, cancel := context.WithCancel(context.Background())
+	obj.MakeBucketWithLocation(ctx, bucketName, "")
+
+	uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
-	go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, GlobalServiceDoneCh)
+	var cleanupWg sync.WaitGroup
+	cleanupWg.Add(1)
+	go func() {
+		defer cleanupWg.Done()
+		fs.cleanupStaleMultipartUploads(context.Background(), time.Millisecond, 0, ctx.Done())
+	}()
 
-	// Wait for 40ms such that - we have given enough time for
-	// cleanup routine to kick in.
-	time.Sleep(40 * time.Millisecond)
-
-	// Close the routine we do not need it anymore.
-	GlobalServiceDoneCh <- struct{}{}
+	// Wait for 100ms such that - we have given enough time for
+	// cleanup routine to kick in. Flaky on slow systems...
+	time.Sleep(100 * time.Millisecond)
+	cancel()
+	cleanupWg.Wait()
 
 	// Check if upload id was already purged.
 	if err = obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
+	} else {
+		t.Error("Item was not cleaned up.")
 	}
 }
 

cmd/fs-v1.go

@@ -37,12 +37,10 @@ import (
 	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/object/tagging"
 	"github.com/minio/minio/pkg/bucket/policy"
-
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
@@ -112,7 +110,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 		return err
 	}
 
-	if err := os.MkdirAll(pathJoin(fsPath, minioMetaBackgroundOpsBucket), 0777); err != nil {
+	if err := os.MkdirAll(pathJoin(fsPath, dataUsageBucket), 0777); err != nil {
 		return err
 	}
 
@@ -188,9 +186,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
 	// lockers are explicitly 'nil' for FS mode since there are only local lockers
-	return fs.nsMutex.NewNSLock(ctx, nil, bucket, object)
+	return fs.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
 }
 
 // Shutdown - should be called when process shuts down.
@@ -235,9 +233,22 @@ func (fs *FSObjects) waitForLowActiveIO() {
 }
 
 // CrawlAndGetDataUsage returns data usage stats of the current FS deployment
-func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-	dataUsageInfo := updateUsage(fs.fsPath, endCh, fs.waitForLowActiveIO, func(item Item) (int64, error) {
-		// Get file size, symlinks which cannot bex
+func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+	// Load bucket totals
+	var oldCache dataUsageCache
+	err := oldCache.load(ctx, fs, dataUsageCacheName)
+	if err != nil {
+		return err
+	}
+	if oldCache.Info.Name == "" {
+		oldCache.Info.Name = dataUsageRoot
+	}
+	buckets, err := fs.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+	cache, err := updateUsage(ctx, fs.fsPath, oldCache, fs.waitForLowActiveIO, func(item Item) (int64, error) {
+		// Get file size, symlinks which cannot be
 		// followed are automatically filtered by fastwalk.
 		fi, err := os.Stat(item.Path)
 		if err != nil {
@@ -246,10 +257,13 @@ func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan stru
 		return fi.Size(), nil
 	})
 
-	dataUsageInfo.LastUpdate = UTCNow()
-	atomic.StoreUint64(&fs.totalUsed, dataUsageInfo.ObjectsTotalSize)
+	// Even if there was an error, the new cache may have better info.
+	if cache.Info.LastUpdate.After(oldCache.Info.LastUpdate) {
+		logger.LogIf(ctx, cache.save(ctx, fs, dataUsageCacheName))
+		updates <- cache.dui(dataUsageRoot, buckets)
+	}
 
-	return dataUsageInfo
+	return err
 }
 
 /// Bucket operations
@@ -490,7 +504,6 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 // GetObjectNInfo - returns object info and a reader for object
 // content.
 func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-
 	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
 		return nil, err
 	}
@@ -1059,15 +1072,18 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // is a leaf or non-leaf entry.
 func (fs *FSObjects) listDirFactory() ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
-			return
+			return false, nil
+		}
+		if len(entries) == 0 {
+			return true, nil
 		}
 		sort.Strings(entries)
-		return filterMatchingPrefix(entries, prefixEntry)
+		return false, filterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -1234,7 +1250,7 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
@@ -1247,8 +1263,17 @@ func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remo
 	return madmin.HealResultItem{}, NotImplemented{}
 }
 
+// Walk a bucket, optionally prefix recursively, until we have returned
+// all the content to objectInfo channel, it is callers responsibility
+// to allocate a receive channel for ObjectInfo, upon any unhandled
+// error walker returns error. Optionally if context.Done() is received
+// then Walk() stops the walker.
+func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfo, fs.getObjectInfo)
+}
+
 // HealObjects - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) (e error) {
+func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) (e error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }

cmd/fs-v1_test.go

@@ -356,8 +356,6 @@ func TestFSListBuckets(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	GlobalServiceDoneCh <- struct{}{}
-
 	// Create a bucket with invalid name
 	if err := os.MkdirAll(pathJoin(fs.fsPath, "vo^"), 0777); err != nil {
 		t.Fatal("Unexpected error: ", err)
@@ -392,7 +390,7 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false, madmin.HealDeepScan)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", madmin.HealOpts{})
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
@@ -404,7 +402,7 @@ func TestFSHealObjects(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	err := obj.HealObjects(context.Background(), "bucket", "prefix", nil)
+	err := obj.HealObjects(context.Background(), "bucket", "prefix", madmin.HealOpts{}, nil)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -309,8 +309,5 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
 	}
 
-	// Set uptime time after object layer has initialized.
-	globalBootTime = UTCNow()
-
 	handleSignals()
 }

cmd/gateway-unsupported.go

@@ -37,8 +37,8 @@ type GatewayLocker struct {
 }
 
 // NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, object)
+func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
 }
 
 // NewGatewayLayerWithLocker - initialize gateway with locker.
@@ -50,13 +50,13 @@ func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
 type GatewayUnsupported struct{}
 
 // CrawlAndGetDataUsage - crawl is not implemented for gateway
-func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
-	return DataUsageInfo{}
+	return NotImplemented{}
 }
 
 // NewNSLock is a dummy stub for gateway.
-func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
 	return nil
 }
@@ -167,7 +167,7 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 }
 
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 
@@ -176,8 +176,13 @@ func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, c
 	return result, NotImplemented{}
 }
 
+// Walk - Not implemented stub
+func (a GatewayUnsupported) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	return NotImplemented{}
+}
+
 // HealObjects - Not implemented stub
-func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) (e error) {
+func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) (e error) {
 	return NotImplemented{}
 }
 

cmd/gateway/azure/gateway-azure.go

@@ -39,7 +39,6 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
-	"github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bucket/policy"
@@ -94,8 +93,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
      {{.Prompt}} {{.HelpName}}
 `
 
@@ -1194,7 +1195,7 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	objMetadata["md5sum"] = cmd.ComputeCompleteMultipartMD5(uploadedParts)
+	objMetadata["md5sum"] = minio.ComputeCompleteMultipartMD5(uploadedParts)
 
 	_, err = objBlob.CommitBlockList(ctx, allBlocks, objProperties, objMetadata, azblob.BlobAccessConditions{})
 	if err != nil {

cmd/gateway/b2/gateway-b2.go

@@ -69,8 +69,11 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}applicationKey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+
      {{.Prompt}} {{.HelpName}}
 `
 	minio.RegisterGatewayCommand(cli.Command{

cmd/gateway/gcs/gateway-gcs.go

@@ -124,8 +124,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
      {{.Prompt}} {{.HelpName}} mygcsprojectid
 `
 

cmd/gateway/hdfs/gateway-hdfs.go

@@ -75,8 +75,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90 
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
      {{.Prompt}} {{.HelpName}} hdfs://namenode:8200
 `
 
@@ -324,7 +326,7 @@ func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketIn
 
 func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
 		f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, prefixDir))
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -339,6 +341,9 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 			logger.LogIf(context.Background(), err)
 			return
 		}
+		if len(fis) == 0 {
+			return true, nil
+		}
 		for _, fi := range fis {
 			if fi.IsDir() {
 				entries = append(entries, fi.Name()+hdfsSeparator)
@@ -346,7 +351,7 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 				entries = append(entries, fi.Name())
 			}
 		}
-		return minio.FilterMatchingPrefix(entries, prefixEntry)
+		return false, minio.FilterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.

cmd/gateway/nas/gateway-nas.go

@@ -52,8 +52,11 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+
      {{.Prompt}} {{.HelpName}} /shared/nasvol
 `
 

cmd/gateway/oss/gateway-oss.go

@@ -71,8 +71,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
      {{.Prompt}} {{.HelpName}}
 `
 

cmd/gateway/s3/gateway-s3-sse.go

@@ -506,7 +506,6 @@ func (l *s3EncObjects) PutObjectPart(ctx context.Context, bucket string, object
 		Number: partID,
 		ETag:   pi.ETag,
 		Size:   pi.Size,
-		Name:   strconv.Itoa(partID),
 	}
 	gwMeta.ETag = data.MD5CurrentHexString() // encrypted ETag
 	gwMeta.Stat.Size = pi.Size
@@ -680,7 +679,7 @@ func getGWContentPath(object string) string {
 }
 
 // Clean-up the stale incomplete encrypted multipart uploads. Should be run in a Go routine.
-func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 

cmd/gateway/s3/gateway-s3.go

@@ -66,8 +66,10 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
      {{.Prompt}} {{.HelpName}}
 `
 

cmd/generic-handlers.go

@@ -353,7 +353,7 @@ func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
 // supported amz date formats.
 func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
 	for _, amzDateHeader := range amzDateHeaders {
-		amzDateStr := req.Header.Get(http.CanonicalHeaderKey(amzDateHeader))
+		amzDateStr := req.Header.Get(amzDateHeader)
 		if amzDateStr != "" {
 			return parseAmzDate(amzDateStr)
 		}
@@ -538,9 +538,8 @@ func setHTTPStatsHandler(h http.Handler) http.Handler {
 func (h httpStatsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	isS3Request := !strings.HasPrefix(r.URL.Path, minioReservedBucketPath)
 	// record s3 connection stats.
-	recordRequest := &recordTrafficRequest{ReadCloser: r.Body, isS3Request: isS3Request}
-	r.Body = recordRequest
-	recordResponse := &recordTrafficResponse{w, isS3Request}
+	r.Body = &recordTrafficRequest{ReadCloser: r.Body, isS3Request: isS3Request}
+	recordResponse := &recordTrafficResponse{ResponseWriter: w, isS3Request: isS3Request}
 	// Execute the request
 	h.handler.ServeHTTP(recordResponse, r)
 }

cmd/global-heal.go

@@ -29,7 +29,6 @@ import (
 const (
 	bgHealingUUID = "0000-0000-0000-0000"
 	leaderTick    = time.Hour
-	healTick      = time.Hour
 	healInterval  = 30 * 24 * time.Hour
 )
 
@@ -40,7 +39,7 @@ var leaderLockTimeout = newDynamicTimeout(time.Minute, time.Minute)
 func newBgHealSequence(numDisks int) *healSequence {
 
 	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
@@ -75,6 +74,7 @@ func getLocalBackgroundHealStatus() madmin.BgHealState {
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.scannedItemsCount,
 		LastHealActivity:  bgSeq.lastHealActivity,
+		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
 	}
 }
 
@@ -112,6 +112,19 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
 	return nil
 }
 
+// Returns the duration to the next background healing round
+func durationToNextHealRound(lastHeal time.Time) time.Duration {
+	if lastHeal.IsZero() {
+		lastHeal = globalBootTime
+	}
+
+	d := lastHeal.Add(healInterval).Sub(UTCNow())
+	if d < 0 {
+		return time.Second
+	}
+	return d
+}
+
 // Healing leader will take the charge of healing all erasure sets
 func execLeaderTasks(z *xlZones) {
 	ctx := context.Background()
@@ -132,10 +145,7 @@ func execLeaderTasks(z *xlZones) {
 
 	lastScanTime := time.Now() // So that we don't heal immediately, but after one month.
 	for {
-		if time.Since(lastScanTime) < healInterval {
-			time.Sleep(healTick)
-			continue
-		}
+		time.Sleep(durationToNextHealRound(lastScanTime))
 		for _, zone := range z.zones {
 			// Heal set by set
 			for i, set := range zone.sets {

cmd/globals.go

@@ -144,8 +144,11 @@ var (
 
 	globalNotificationSys  *NotificationSys
 	globalConfigTargetList *event.TargetList
-	globalPolicySys        *PolicySys
-	globalIAMSys           *IAMSys
+	// globalEnvTargetList has list of targets configured via env.
+	globalEnvTargetList *event.TargetList
+
+	globalPolicySys *PolicySys
+	globalIAMSys    *IAMSys
 
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
@@ -185,11 +188,14 @@ var (
 	// Global HTTP request statisitics
 	globalHTTPStats = newHTTPStats()
 
-	// Time when object layer was initialized on start up.
-	globalBootTime time.Time
+	// Time when the server is started
+	globalBootTime = UTCNow()
 
 	globalActiveCred auth.Credentials
 
+	// Hold the old server credentials passed by the environment
+	globalOldCred auth.Credentials
+
 	// Indicates if config is to be encrypted
 	globalConfigEncrypted bool
 

cmd/handler-utils.go

@@ -362,14 +362,16 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 
 		isS3Request := !strings.HasPrefix(r.URL.Path, minioReservedBucketPath)
-		apiStatsWriter := &recordAPIStats{w, UTCNow(), false, 0, isS3Request}
 
 		// Time start before the call is about to start.
 		tBefore := UTCNow()
 
+		apiStatsWriter := &recordAPIStats{ResponseWriter: w, TTFB: tBefore, isS3Request: isS3Request}
+
 		if isS3Request {
 			globalHTTPStats.currentS3Requests.Inc(api)
 		}
+
 		// Execute the request
 		f.ServeHTTP(apiStatsWriter, r)
 

cmd/http-stats.go

@@ -101,28 +101,24 @@ type HTTPAPIStats struct {
 
 // Inc increments the api stats counter.
 func (stats *HTTPAPIStats) Inc(api string) {
-	stats.Lock()
-	defer stats.Unlock()
 	if stats == nil {
 		return
 	}
+	stats.Lock()
+	defer stats.Unlock()
 	if stats.apiStats == nil {
 		stats.apiStats = make(map[string]int)
 	}
-	if _, ok := stats.apiStats[api]; ok {
-		stats.apiStats[api]++
-		return
-	}
-	stats.apiStats[api] = 1
+	stats.apiStats[api]++
 }
 
 // Dec increments the api stats counter.
 func (stats *HTTPAPIStats) Dec(api string) {
-	stats.Lock()
-	defer stats.Unlock()
 	if stats == nil {
 		return
 	}
+	stats.Lock()
+	defer stats.Unlock()
 	if val, ok := stats.apiStats[api]; ok && val > 0 {
 		stats.apiStats[api]--
 	}

cmd/http-tracer.go

@@ -83,14 +83,15 @@ func getOpName(name string) (op string) {
 	op = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
 	op = strings.TrimSuffix(op, "Handler-fm")
 	op = strings.Replace(op, "objectAPIHandlers", "s3", 1)
-	op = strings.Replace(op, "webAPIHandlers", "s3", 1)
+	op = strings.Replace(op, "webAPIHandlers", "webui", 1)
 	op = strings.Replace(op, "adminAPIHandlers", "admin", 1)
 	op = strings.Replace(op, "(*storageRESTServer)", "internal", 1)
 	op = strings.Replace(op, "(*peerRESTServer)", "internal", 1)
 	op = strings.Replace(op, "(*lockRESTServer)", "internal", 1)
-	op = strings.Replace(op, "stsAPIHandlers", "sts", 1)
+	op = strings.Replace(op, "(*stsAPIHandlers)", "sts", 1)
 	op = strings.Replace(op, "LivenessCheckHandler", "healthcheck", 1)
 	op = strings.Replace(op, "ReadinessCheckHandler", "healthcheck", 1)
+	op = strings.Replace(op, "-fm", "", 1)
 	return op
 }
 

cmd/http-traffic-recorder.go

@@ -41,23 +41,13 @@ func (r *recordTrafficRequest) Read(p []byte) (n int, err error) {
 // Records the outgoing bytes through the responseWriter.
 type recordTrafficResponse struct {
 	// wrapper for underlying http.ResponseWriter.
-	writer      http.ResponseWriter
+	http.ResponseWriter
 	isS3Request bool
 }
 
-// Calls the underlying WriteHeader.
-func (r *recordTrafficResponse) WriteHeader(i int) {
-	r.writer.WriteHeader(i)
-}
-
-// Calls the underlying Header.
-func (r *recordTrafficResponse) Header() http.Header {
-	return r.writer.Header()
-}
-
 // Records the output bytes
 func (r *recordTrafficResponse) Write(p []byte) (n int, err error) {
-	n, err = r.writer.Write(p)
+	n, err = r.ResponseWriter.Write(p)
 	globalConnStats.incOutputBytes(n)
 	// Check if it is s3 request
 	if r.isS3Request {
@@ -68,13 +58,12 @@ func (r *recordTrafficResponse) Write(p []byte) (n int, err error) {
 
 // Calls the underlying Flush.
 func (r *recordTrafficResponse) Flush() {
-	r.writer.(http.Flusher).Flush()
+	r.ResponseWriter.(http.Flusher).Flush()
 }
 
 // Records the outgoing bytes through the responseWriter.
 type recordAPIStats struct {
-	// wrapper for underlying http.ResponseWriter.
-	writer         http.ResponseWriter
+	http.ResponseWriter
 	TTFB           time.Time // TimeToFirstByte.
 	firstByteRead  bool
 	respStatusCode int
@@ -84,12 +73,7 @@ type recordAPIStats struct {
 // Calls the underlying WriteHeader.
 func (r *recordAPIStats) WriteHeader(i int) {
 	r.respStatusCode = i
-	r.writer.WriteHeader(i)
-}
-
-// Calls the underlying Header.
-func (r *recordAPIStats) Header() http.Header {
-	return r.writer.Header()
+	r.ResponseWriter.WriteHeader(i)
 }
 
 // Records the TTFB on the first byte write.
@@ -98,10 +82,10 @@ func (r *recordAPIStats) Write(p []byte) (n int, err error) {
 		r.TTFB = UTCNow()
 		r.firstByteRead = true
 	}
-	return r.writer.Write(p)
+	return r.ResponseWriter.Write(p)
 }
 
 // Calls the underlying Flush.
 func (r *recordAPIStats) Flush() {
-	r.writer.(http.Flusher).Flush()
+	r.ResponseWriter.(http.Flusher).Flush()
 }

cmd/http/headers.go

@@ -77,6 +77,9 @@ const (
 	AmzObjectLockLegalHold        = "X-Amz-Object-Lock-Legal-Hold"
 	AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention"
 
+	// Multipart parts count
+	AmzMpPartsCount = "x-amz-mp-parts-count"
+
 	// Dummy putBucketACL
 	AmzACL = "x-amz-acl"
 

cmd/iam-etcd-store.go

@@ -28,6 +28,7 @@ import (
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/mvcc/mvccpb"
+	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -174,7 +175,11 @@ func (ies *IAMEtcdStore) migrateUsersConfigToV1(isSTS bool) error {
 
 			// 2. copy policy to new loc.
 			mp := newMappedPolicy(policyName)
-			path := getMappedPolicyPath(user, isSTS, false)
+			userType := regularUser
+			if isSTS {
+				userType = stsUser
+			}
+			path := getMappedPolicyPath(user, userType, false)
 			if err := ies.saveIAMConfig(mp, path); err != nil {
 				return err
 			}
@@ -300,9 +305,9 @@ func (ies *IAMEtcdStore) loadPolicyDocs(m map[string]iampolicy.Policy) error {
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadUser(user string, isSTS bool, m map[string]auth.Credentials) error {
+func (ies *IAMEtcdStore) loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error {
 	var u UserIdentity
-	err := ies.loadIAMConfig(&u, getUserIdentityPath(user, isSTS))
+	err := ies.loadIAMConfig(&u, getUserIdentityPath(user, userType))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchUser
@@ -313,11 +318,31 @@ func (ies *IAMEtcdStore) loadUser(user string, isSTS bool, m map[string]auth.Cre
 	if u.Credentials.IsExpired() {
 		// Delete expired identity.
 		ctx := ies.getContext()
-		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, isSTS))
-		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, isSTS, false))
+		deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
+		deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
 		return nil
 	}
 
+	// If this is a service account, rotate the session key if we are changing the server creds
+	if globalOldCred.IsValid() && u.Credentials.IsServiceAccount() {
+		if !globalOldCred.Equal(globalActiveCred) {
+			m := jwtgo.MapClaims{}
+			stsTokenCallback := func(t *jwtgo.Token) (interface{}, error) {
+				return []byte(globalOldCred.SecretKey), nil
+			}
+			if _, err := jwtgo.ParseWithClaims(u.Credentials.SessionToken, m, stsTokenCallback); err == nil {
+				jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.MapClaims(m))
+				if token, err := jwt.SignedString([]byte(globalActiveCred.SecretKey)); err == nil {
+					u.Credentials.SessionToken = token
+					err := ies.saveIAMConfig(&u, getUserIdentityPath(user, userType))
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
@@ -326,10 +351,15 @@ func (ies *IAMEtcdStore) loadUser(user string, isSTS bool, m map[string]auth.Cre
 
 }
 
-func (ies *IAMEtcdStore) loadUsers(isSTS bool, m map[string]auth.Credentials) error {
-	basePrefix := iamConfigUsersPrefix
-	if isSTS {
+func (ies *IAMEtcdStore) loadUsers(userType IAMUserType, m map[string]auth.Credentials) error {
+	var basePrefix string
+	switch userType {
+	case srvAccUser:
+		basePrefix = iamConfigServiceAccountsPrefix
+	case stsUser:
 		basePrefix = iamConfigSTSPrefix
+	default:
+		basePrefix = iamConfigUsersPrefix
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
@@ -345,7 +375,7 @@ func (ies *IAMEtcdStore) loadUsers(isSTS bool, m map[string]auth.Credentials) er
 
 	// Reload config for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadUser(user, isSTS, m); err != nil {
+		if err = ies.loadUser(user, userType, m); err != nil {
 			return err
 		}
 	}
@@ -388,9 +418,9 @@ func (ies *IAMEtcdStore) loadGroups(m map[string]GroupInfo) error {
 
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicy(name string, isSTS, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicy(name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
 	var p MappedPolicy
-	err := ies.loadIAMConfig(&p, getMappedPolicyPath(name, isSTS, isGroup))
+	err := ies.loadIAMConfig(&p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
@@ -402,19 +432,23 @@ func (ies *IAMEtcdStore) loadMappedPolicy(name string, isSTS, isGroup bool, m ma
 
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicies(isSTS, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	defer cancel()
 	ies.setContext(ctx)
 	defer ies.clearContext()
 	var basePrefix string
-	switch {
-	case isSTS:
-		basePrefix = iamConfigPolicyDBSTSUsersPrefix
-	case isGroup:
+	if isGroup {
 		basePrefix = iamConfigPolicyDBGroupsPrefix
-	default:
-		basePrefix = iamConfigPolicyDBUsersPrefix
+	} else {
+		switch userType {
+		case srvAccUser:
+			basePrefix = iamConfigPolicyDBServiceAccountsPrefix
+		case stsUser:
+			basePrefix = iamConfigPolicyDBSTSUsersPrefix
+		default:
+			basePrefix = iamConfigPolicyDBUsersPrefix
+		}
 	}
 	r, err := ies.client.Get(ctx, basePrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
@@ -425,7 +459,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(isSTS, isGroup bool, m map[string]Ma
 
 	// Reload config and policies for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadMappedPolicy(user, isSTS, isGroup, m); err != nil {
+		if err = ies.loadMappedPolicy(user, userType, isGroup, m); err != nil {
 			return err
 		}
 	}
@@ -452,29 +486,32 @@ func (ies *IAMEtcdStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 	}
 
 	// load STS temp users
-	if err := ies.loadUsers(true, iamUsersMap); err != nil {
+	if err := ies.loadUsers(stsUser, iamUsersMap); err != nil {
 		return err
 	}
 
 	if isMinIOUsersSys {
 		// load long term users
-		if err := ies.loadUsers(false, iamUsersMap); err != nil {
+		if err := ies.loadUsers(regularUser, iamUsersMap); err != nil {
+			return err
+		}
+		if err := ies.loadUsers(srvAccUser, iamUsersMap); err != nil {
 			return err
 		}
 		if err := ies.loadGroups(iamGroupsMap); err != nil {
 			return err
 		}
-		if err := ies.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
+		if err := ies.loadMappedPolicies(regularUser, false, iamUserPolicyMap); err != nil {
 			return err
 		}
 	}
 
 	// load STS policy mappings into the same map
-	if err := ies.loadMappedPolicies(true, false, iamUserPolicyMap); err != nil {
+	if err := ies.loadMappedPolicies(stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 	// load policies mapped to groups
-	if err := ies.loadMappedPolicies(false, true, iamGroupPolicyMap); err != nil {
+	if err := ies.loadMappedPolicies(regularUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
@@ -498,12 +535,12 @@ func (ies *IAMEtcdStore) savePolicyDoc(policyName string, p iampolicy.Policy) er
 	return ies.saveIAMConfig(&p, getPolicyDocPath(policyName))
 }
 
-func (ies *IAMEtcdStore) saveMappedPolicy(name string, isSTS, isGroup bool, mp MappedPolicy) error {
-	return ies.saveIAMConfig(mp, getMappedPolicyPath(name, isSTS, isGroup))
+func (ies *IAMEtcdStore) saveMappedPolicy(name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error {
+	return ies.saveIAMConfig(mp, getMappedPolicyPath(name, userType, isGroup))
 }
 
-func (ies *IAMEtcdStore) saveUserIdentity(name string, isSTS bool, u UserIdentity) error {
-	return ies.saveIAMConfig(u, getUserIdentityPath(name, isSTS))
+func (ies *IAMEtcdStore) saveUserIdentity(name string, userType IAMUserType, u UserIdentity) error {
+	return ies.saveIAMConfig(u, getUserIdentityPath(name, userType))
 }
 
 func (ies *IAMEtcdStore) saveGroupInfo(name string, gi GroupInfo) error {
@@ -518,16 +555,16 @@ func (ies *IAMEtcdStore) deletePolicyDoc(name string) error {
 	return err
 }
 
-func (ies *IAMEtcdStore) deleteMappedPolicy(name string, isSTS, isGroup bool) error {
-	err := ies.deleteIAMConfig(getMappedPolicyPath(name, isSTS, isGroup))
+func (ies *IAMEtcdStore) deleteMappedPolicy(name string, userType IAMUserType, isGroup bool) error {
+	err := ies.deleteIAMConfig(getMappedPolicyPath(name, userType, isGroup))
 	if err == errConfigNotFound {
 		err = errNoSuchPolicy
 	}
 	return err
 }
 
-func (ies *IAMEtcdStore) deleteUserIdentity(name string, isSTS bool) error {
-	err := ies.deleteIAMConfig(getUserIdentityPath(name, isSTS))
+func (ies *IAMEtcdStore) deleteUserIdentity(name string, userType IAMUserType) error {
+	err := ies.deleteIAMConfig(getUserIdentityPath(name, userType))
 	if err == errConfigNotFound {
 		err = errNoSuchUser
 	}
@@ -599,11 +636,11 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 		case usersPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigUsersPrefix))
-			ies.loadUser(accessKey, false, sys.iamUsersMap)
+			ies.loadUser(accessKey, regularUser, sys.iamUsersMap)
 		case stsPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigSTSPrefix))
-			ies.loadUser(accessKey, true, sys.iamUsersMap)
+			ies.loadUser(accessKey, stsUser, sys.iamUsersMap)
 		case groupsPrefix:
 			group := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigGroupsPrefix))
@@ -619,17 +656,17 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBUsersPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(user, false, false, sys.iamUserPolicyMap)
+			ies.loadMappedPolicy(user, regularUser, false, sys.iamUserPolicyMap)
 		case policyDBSTSUsersPrefix:
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBSTSUsersPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(user, true, false, sys.iamUserPolicyMap)
+			ies.loadMappedPolicy(user, stsUser, false, sys.iamUserPolicyMap)
 		case policyDBGroupsPrefix:
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBGroupsPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(user, false, true, sys.iamGroupPolicyMap)
+			ies.loadMappedPolicy(user, regularUser, true, sys.iamGroupPolicyMap)
 		}
 	case eventDelete:
 		switch {

cmd/iam-object-store.go

@@ -25,6 +25,8 @@ import (
 	"sync"
 	"time"
 
+	jwtgo "github.com/dgrijalva/jwt-go"
+
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -116,7 +118,11 @@ func (iamOS *IAMObjectStore) migrateUsersConfigToV1(isSTS bool) error {
 
 			// 2. copy policy file to new location.
 			mp := newMappedPolicy(policyName)
-			if err := iamOS.saveMappedPolicy(user, isSTS, false, mp); err != nil {
+			userType := regularUser
+			if isSTS {
+				userType = stsUser
+			}
+			if err := iamOS.saveMappedPolicy(user, userType, false, mp); err != nil {
 				return err
 			}
 
@@ -289,14 +295,14 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(m map[string]iampolicy.Policy) error
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUser(user string, isSTS bool, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error {
 	objectAPI := iamOS.getObjectAPI()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
 	var u UserIdentity
-	err := iamOS.loadIAMConfig(&u, getUserIdentityPath(user, isSTS))
+	err := iamOS.loadIAMConfig(&u, getUserIdentityPath(user, userType))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchUser
@@ -306,11 +312,31 @@ func (iamOS *IAMObjectStore) loadUser(user string, isSTS bool, m map[string]auth
 
 	if u.Credentials.IsExpired() {
 		// Delete expired identity - ignoring errors here.
-		iamOS.deleteIAMConfig(getUserIdentityPath(user, isSTS))
-		iamOS.deleteIAMConfig(getMappedPolicyPath(user, isSTS, false))
+		iamOS.deleteIAMConfig(getUserIdentityPath(user, userType))
+		iamOS.deleteIAMConfig(getMappedPolicyPath(user, userType, false))
 		return nil
 	}
 
+	// If this is a service account, rotate the session key if needed
+	if globalOldCred.IsValid() && u.Credentials.IsServiceAccount() {
+		if !globalOldCred.Equal(globalActiveCred) {
+			m := jwtgo.MapClaims{}
+			stsTokenCallback := func(t *jwtgo.Token) (interface{}, error) {
+				return []byte(globalOldCred.SecretKey), nil
+			}
+			if _, err := jwtgo.ParseWithClaims(u.Credentials.SessionToken, m, stsTokenCallback); err == nil {
+				jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.MapClaims(m))
+				if token, err := jwt.SignedString([]byte(globalActiveCred.SecretKey)); err == nil {
+					u.Credentials.SessionToken = token
+					err := iamOS.saveIAMConfig(&u, getUserIdentityPath(user, userType))
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
@@ -318,7 +344,7 @@ func (iamOS *IAMObjectStore) loadUser(user string, isSTS bool, m map[string]auth
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUsers(isSTS bool, m map[string]auth.Credentials) error {
+func (iamOS *IAMObjectStore) loadUsers(userType IAMUserType, m map[string]auth.Credentials) error {
 	objectAPI := iamOS.getObjectAPI()
 	if objectAPI == nil {
 		return errServerNotInitialized
@@ -326,17 +352,24 @@ func (iamOS *IAMObjectStore) loadUsers(isSTS bool, m map[string]auth.Credentials
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	basePrefix := iamConfigUsersPrefix
-	if isSTS {
+
+	var basePrefix string
+	switch userType {
+	case srvAccUser:
+		basePrefix = iamConfigServiceAccountsPrefix
+	case stsUser:
 		basePrefix = iamConfigSTSPrefix
+	default:
+		basePrefix = iamConfigUsersPrefix
 	}
+
 	for item := range listIAMConfigItems(objectAPI, basePrefix, true, doneCh) {
 		if item.Err != nil {
 			return item.Err
 		}
 
 		userName := item.Item
-		err := iamOS.loadUser(userName, isSTS, m)
+		err := iamOS.loadUser(userName, userType, m)
 		if err != nil {
 			return err
 		}
@@ -384,7 +417,7 @@ func (iamOS *IAMObjectStore) loadGroups(m map[string]GroupInfo) error {
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicy(name string, isSTS, isGroup bool,
+func (iamOS *IAMObjectStore) loadMappedPolicy(name string, userType IAMUserType, isGroup bool,
 	m map[string]MappedPolicy) error {
 
 	objectAPI := iamOS.getObjectAPI()
@@ -393,7 +426,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicy(name string, isSTS, isGroup bool,
 	}
 
 	var p MappedPolicy
-	err := iamOS.loadIAMConfig(&p, getMappedPolicyPath(name, isSTS, isGroup))
+	err := iamOS.loadIAMConfig(&p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
@@ -404,7 +437,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicy(name string, isSTS, isGroup bool,
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicies(isSTS, isGroup bool, m map[string]MappedPolicy) error {
+func (iamOS *IAMObjectStore) loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
 	objectAPI := iamOS.getObjectAPI()
 	if objectAPI == nil {
 		return errServerNotInitialized
@@ -413,13 +446,17 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(isSTS, isGroup bool, m map[strin
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 	var basePath string
-	switch {
-	case isSTS:
-		basePath = iamConfigPolicyDBSTSUsersPrefix
-	case isGroup:
+	if isGroup {
 		basePath = iamConfigPolicyDBGroupsPrefix
-	default:
-		basePath = iamConfigPolicyDBUsersPrefix
+	} else {
+		switch userType {
+		case srvAccUser:
+			basePath = iamConfigPolicyDBServiceAccountsPrefix
+		case stsUser:
+			basePath = iamConfigPolicyDBSTSUsersPrefix
+		default:
+			basePath = iamConfigPolicyDBUsersPrefix
+		}
 	}
 	for item := range listIAMConfigItems(objectAPI, basePath, false, doneCh) {
 		if item.Err != nil {
@@ -428,7 +465,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(isSTS, isGroup bool, m map[strin
 
 		policyFile := item.Item
 		userOrGroupName := strings.TrimSuffix(policyFile, ".json")
-		err := iamOS.loadMappedPolicy(userOrGroupName, isSTS, isGroup, m)
+		err := iamOS.loadMappedPolicy(userOrGroupName, userType, isGroup, m)
 		if err != nil {
 			return err
 		}
@@ -466,27 +503,30 @@ func (iamOS *IAMObjectStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 		return err
 	}
 	// load STS temp users
-	if err := iamOS.loadUsers(true, iamUsersMap); err != nil {
+	if err := iamOS.loadUsers(stsUser, iamUsersMap); err != nil {
 		return err
 	}
 	if isMinIOUsersSys {
-		if err := iamOS.loadUsers(false, iamUsersMap); err != nil {
+		if err := iamOS.loadUsers(regularUser, iamUsersMap); err != nil {
+			return err
+		}
+		if err := iamOS.loadUsers(srvAccUser, iamUsersMap); err != nil {
 			return err
 		}
 		if err := iamOS.loadGroups(iamGroupsMap); err != nil {
 			return err
 		}
 
-		if err := iamOS.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
+		if err := iamOS.loadMappedPolicies(regularUser, false, iamUserPolicyMap); err != nil {
 			return err
 		}
 	}
 	// load STS policy mappings
-	if err := iamOS.loadMappedPolicies(true, false, iamUserPolicyMap); err != nil {
+	if err := iamOS.loadMappedPolicies(stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 	// load policies mapped to groups
-	if err := iamOS.loadMappedPolicies(false, true, iamGroupPolicyMap); err != nil {
+	if err := iamOS.loadMappedPolicies(regularUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
@@ -510,12 +550,12 @@ func (iamOS *IAMObjectStore) savePolicyDoc(policyName string, p iampolicy.Policy
 	return iamOS.saveIAMConfig(&p, getPolicyDocPath(policyName))
 }
 
-func (iamOS *IAMObjectStore) saveMappedPolicy(name string, isSTS, isGroup bool, mp MappedPolicy) error {
-	return iamOS.saveIAMConfig(mp, getMappedPolicyPath(name, isSTS, isGroup))
+func (iamOS *IAMObjectStore) saveMappedPolicy(name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error {
+	return iamOS.saveIAMConfig(mp, getMappedPolicyPath(name, userType, isGroup))
 }
 
-func (iamOS *IAMObjectStore) saveUserIdentity(name string, isSTS bool, u UserIdentity) error {
-	return iamOS.saveIAMConfig(u, getUserIdentityPath(name, isSTS))
+func (iamOS *IAMObjectStore) saveUserIdentity(name string, userType IAMUserType, u UserIdentity) error {
+	return iamOS.saveIAMConfig(u, getUserIdentityPath(name, userType))
 }
 
 func (iamOS *IAMObjectStore) saveGroupInfo(name string, gi GroupInfo) error {
@@ -530,16 +570,16 @@ func (iamOS *IAMObjectStore) deletePolicyDoc(name string) error {
 	return err
 }
 
-func (iamOS *IAMObjectStore) deleteMappedPolicy(name string, isSTS, isGroup bool) error {
-	err := iamOS.deleteIAMConfig(getMappedPolicyPath(name, isSTS, isGroup))
+func (iamOS *IAMObjectStore) deleteMappedPolicy(name string, userType IAMUserType, isGroup bool) error {
+	err := iamOS.deleteIAMConfig(getMappedPolicyPath(name, userType, isGroup))
 	if err == errConfigNotFound {
 		err = errNoSuchPolicy
 	}
 	return err
 }
 
-func (iamOS *IAMObjectStore) deleteUserIdentity(name string, isSTS bool) error {
-	err := iamOS.deleteIAMConfig(getUserIdentityPath(name, isSTS))
+func (iamOS *IAMObjectStore) deleteUserIdentity(name string, userType IAMUserType) error {
+	err := iamOS.deleteIAMConfig(getUserIdentityPath(name, userType))
 	if err == errConfigNotFound {
 		err = errNoSuchUser
 	}
@@ -624,14 +664,15 @@ func listIAMConfigItems(objectAPI ObjectLayer, pathPrefix string, dirs bool,
 }
 
 func (iamOS *IAMObjectStore) watch(sys *IAMSys) {
+	ctx := GlobalContext
 	watchDisk := func() {
 		for {
 			select {
-			case <-GlobalServiceDoneCh:
+			case <-ctx.Done():
 				return
 			case <-time.NewTimer(globalRefreshIAMInterval).C:
 				err := iamOS.loadAll(sys, nil)
-				logger.LogIf(context.Background(), err)
+				logger.LogIf(ctx, err)
 			}
 		}
 	}

cmd/iam.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"sync"
 
 	"github.com/minio/minio-go/v6/pkg/set"
@@ -52,6 +54,9 @@ const (
 	// IAM users directory.
 	iamConfigUsersPrefix = iamConfigPrefix + "/users/"
 
+	// IAM service accounts directory.
+	iamConfigServiceAccountsPrefix = iamConfigPrefix + "/service-accounts/"
+
 	// IAM groups directory.
 	iamConfigGroupsPrefix = iamConfigPrefix + "/groups/"
 
@@ -62,10 +67,11 @@ const (
 	iamConfigSTSPrefix = iamConfigPrefix + "/sts/"
 
 	// IAM Policy DB prefixes.
-	iamConfigPolicyDBPrefix         = iamConfigPrefix + "/policydb/"
-	iamConfigPolicyDBUsersPrefix    = iamConfigPolicyDBPrefix + "users/"
-	iamConfigPolicyDBSTSUsersPrefix = iamConfigPolicyDBPrefix + "sts-users/"
-	iamConfigPolicyDBGroupsPrefix   = iamConfigPolicyDBPrefix + "groups/"
+	iamConfigPolicyDBPrefix                = iamConfigPrefix + "/policydb/"
+	iamConfigPolicyDBUsersPrefix           = iamConfigPolicyDBPrefix + "users/"
+	iamConfigPolicyDBSTSUsersPrefix        = iamConfigPolicyDBPrefix + "sts-users/"
+	iamConfigPolicyDBServiceAccountsPrefix = iamConfigPolicyDBPrefix + "service-accounts/"
+	iamConfigPolicyDBGroupsPrefix          = iamConfigPolicyDBPrefix + "groups/"
 
 	// IAM identity file which captures identity credentials.
 	iamIdentityFile = "identity.json"
@@ -99,10 +105,15 @@ func getIAMFormatFilePath() string {
 	return iamConfigPrefix + SlashSeparator + iamFormatFile
 }
 
-func getUserIdentityPath(user string, isSTS bool) string {
-	basePath := iamConfigUsersPrefix
-	if isSTS {
+func getUserIdentityPath(user string, userType IAMUserType) string {
+	var basePath string
+	switch userType {
+	case srvAccUser:
+		basePath = iamConfigServiceAccountsPrefix
+	case stsUser:
 		basePath = iamConfigSTSPrefix
+	default:
+		basePath = iamConfigUsersPrefix
 	}
 	return pathJoin(basePath, user, iamIdentityFile)
 }
@@ -115,12 +126,15 @@ func getPolicyDocPath(name string) string {
 	return pathJoin(iamConfigPoliciesPrefix, name, iamPolicyFile)
 }
 
-func getMappedPolicyPath(name string, isSTS, isGroup bool) string {
-	switch {
-	case isSTS:
-		return pathJoin(iamConfigPolicyDBSTSUsersPrefix, name+".json")
-	case isGroup:
+func getMappedPolicyPath(name string, userType IAMUserType, isGroup bool) string {
+	if isGroup {
 		return pathJoin(iamConfigPolicyDBGroupsPrefix, name+".json")
+	}
+	switch userType {
+	case srvAccUser:
+		return pathJoin(iamConfigPolicyDBServiceAccountsPrefix, name+".json")
+	case stsUser:
+		return pathJoin(iamConfigPolicyDBSTSUsersPrefix, name+".json")
 	default:
 		return pathJoin(iamConfigPolicyDBUsersPrefix, name+".json")
 	}
@@ -180,6 +194,15 @@ type IAMSys struct {
 	store IAMStorageAPI
 }
 
+// IAMUserType represents a user type inside MinIO server
+type IAMUserType int
+
+const (
+	regularUser IAMUserType = iota
+	stsUser
+	srvAccUser
+)
+
 // IAMStorageAPI defines an interface for the IAM persistence layer
 type IAMStorageAPI interface {
 	migrateBackendFormat(ObjectLayer) error
@@ -187,14 +210,14 @@ type IAMStorageAPI interface {
 	loadPolicyDoc(policy string, m map[string]iampolicy.Policy) error
 	loadPolicyDocs(m map[string]iampolicy.Policy) error
 
-	loadUser(user string, isSTS bool, m map[string]auth.Credentials) error
-	loadUsers(isSTS bool, m map[string]auth.Credentials) error
+	loadUser(user string, userType IAMUserType, m map[string]auth.Credentials) error
+	loadUsers(userType IAMUserType, m map[string]auth.Credentials) error
 
 	loadGroup(group string, m map[string]GroupInfo) error
 	loadGroups(m map[string]GroupInfo) error
 
-	loadMappedPolicy(name string, isSTS, isGroup bool, m map[string]MappedPolicy) error
-	loadMappedPolicies(isSTS, isGroup bool, m map[string]MappedPolicy) error
+	loadMappedPolicy(name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
+	loadMappedPolicies(userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error
 
 	loadAll(*IAMSys, ObjectLayer) error
 
@@ -203,13 +226,13 @@ type IAMStorageAPI interface {
 	deleteIAMConfig(path string) error
 
 	savePolicyDoc(policyName string, p iampolicy.Policy) error
-	saveMappedPolicy(name string, isSTS, isGroup bool, mp MappedPolicy) error
-	saveUserIdentity(name string, isSTS bool, u UserIdentity) error
+	saveMappedPolicy(name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error
+	saveUserIdentity(name string, userType IAMUserType, u UserIdentity) error
 	saveGroupInfo(group string, gi GroupInfo) error
 
 	deletePolicyDoc(policyName string) error
-	deleteMappedPolicy(name string, isSTS, isGroup bool) error
-	deleteUserIdentity(name string, isSTS bool) error
+	deleteMappedPolicy(name string, userType IAMUserType, isGroup bool) error
+	deleteUserIdentity(name string, userType IAMUserType) error
 	deleteGroupInfo(name string) error
 
 	watch(*IAMSys)
@@ -290,9 +313,9 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 	if globalEtcdClient == nil {
 		var err error
 		if isGroup {
-			err = sys.store.loadMappedPolicy(userOrGroup, false, isGroup, sys.iamGroupPolicyMap)
+			err = sys.store.loadMappedPolicy(userOrGroup, regularUser, isGroup, sys.iamGroupPolicyMap)
 		} else {
-			err = sys.store.loadMappedPolicy(userOrGroup, false, isGroup, sys.iamUserPolicyMap)
+			err = sys.store.loadMappedPolicy(userOrGroup, regularUser, isGroup, sys.iamUserPolicyMap)
 		}
 
 		// Ignore policy not mapped error
@@ -305,7 +328,7 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 }
 
 // LoadUser - reloads a specific user from backend disks or etcd.
-func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, isSTS bool) error {
+func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUserType) error {
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -314,11 +337,11 @@ func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, isSTS bool) er
 	}
 
 	if globalEtcdClient == nil {
-		err := sys.store.loadUser(accessKey, isSTS, sys.iamUsersMap)
+		err := sys.store.loadUser(accessKey, userType, sys.iamUsersMap)
 		if err != nil {
 			return err
 		}
-		err = sys.store.loadMappedPolicy(accessKey, isSTS, false, sys.iamUserPolicyMap)
+		err = sys.store.loadMappedPolicy(accessKey, userType, false, sys.iamUserPolicyMap)
 		// Ignore policy not mapped error
 		if err != nil && err != errConfigNotFound {
 			return err
@@ -337,14 +360,6 @@ func (sys *IAMSys) Load() error {
 
 // Perform IAM configuration migration.
 func (sys *IAMSys) doIAMConfigMigration(objAPI ObjectLayer) error {
-	// Take IAM configuration migration lock
-	lockPath := iamConfigPrefix + "/migration.lock"
-	objLock := objAPI.NewNSLock(context.Background(), minioMetaBucket, lockPath)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
-
 	return sys.store.migrateBackendFormat(objAPI)
 }
 
@@ -368,8 +383,12 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	}
 
 	sys.store.watch(sys)
+	err := sys.store.loadAll(sys, objAPI)
 
-	return sys.store.loadAll(sys, objAPI)
+	// Invalidate the old cred after finishing IAM initialization
+	globalOldCred = auth.Credentials{}
+
+	return err
 }
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
@@ -401,7 +420,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 
 	// Delete user-policy mappings that will no longer apply
 	var usersToDel []string
-	var isUserSTS []bool
+	var usersType []IAMUserType
 	for u, mp := range sys.iamUserPolicyMap {
 		if mp.Policy == policyName {
 			usersToDel = append(usersToDel, u)
@@ -411,12 +430,15 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 				return errNoSuchUser
 			}
 			// User is from STS if the creds are temporary
-			isSTS := cr.IsTemp()
-			isUserSTS = append(isUserSTS, isSTS)
+			if cr.IsTemp() {
+				usersType = append(usersType, stsUser)
+			} else {
+				usersType = append(usersType, regularUser)
+			}
 		}
 	}
 	for i, u := range usersToDel {
-		sys.policyDBSet(u, "", isUserSTS[i], false)
+		sys.policyDBSet(u, "", usersType[i], false)
 	}
 
 	// Delete group-policy mappings that will no longer apply
@@ -427,7 +449,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 		}
 	}
 	for _, g := range groupsToDel {
-		sys.policyDBSet(g, "", false, true)
+		sys.policyDBSet(g, "", regularUser, true)
 	}
 
 	return err
@@ -531,8 +553,8 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
-	sys.store.deleteMappedPolicy(accessKey, false, false)
-	err := sys.store.deleteUserIdentity(accessKey, false)
+	sys.store.deleteMappedPolicy(accessKey, regularUser, false)
+	err := sys.store.deleteUserIdentity(accessKey, regularUser)
 	switch err.(type) {
 	case ObjectNotFound:
 		// ignore if user is already deleted.
@@ -542,6 +564,15 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	delete(sys.iamUsersMap, accessKey)
 	delete(sys.iamUserPolicyMap, accessKey)
 
+	for _, u := range sys.iamUsersMap {
+		if u.IsServiceAccount() {
+			if u.ParentUser == accessKey {
+				_ = sys.store.deleteUserIdentity(u.AccessKey, srvAccUser)
+				delete(sys.iamUsersMap, u.AccessKey)
+			}
+		}
+	}
+
 	return err
 }
 
@@ -573,7 +604,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 		}
 
 		mp := newMappedPolicy(policyName)
-		if err := sys.store.saveMappedPolicy(accessKey, true, false, mp); err != nil {
+		if err := sys.store.saveMappedPolicy(accessKey, stsUser, false, mp); err != nil {
 			return err
 		}
 
@@ -585,7 +616,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 	}
 
 	u := newUserIdentity(cred)
-	if err := sys.store.saveUserIdentity(accessKey, true, u); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, stsUser, u); err != nil {
 		return err
 	}
 
@@ -610,7 +641,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 	}
 
 	for k, v := range sys.iamUsersMap {
-		if !v.IsTemp() {
+		if !v.IsTemp() && !v.IsServiceAccount() {
 			users[k] = madmin.UserInfo{
 				PolicyName: sys.iamUserPolicyMap[k].Policy,
 				Status: func() madmin.AccountStatus {
@@ -644,6 +675,28 @@ func (sys *IAMSys) IsTempUser(name string) (bool, error) {
 	return creds.IsTemp(), nil
 }
 
+// IsServiceAccount - returns if given key is a service account
+func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil {
+		return false, "", errServerNotInitialized
+	}
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	creds, found := sys.iamUsersMap[name]
+	if !found {
+		return false, "", errNoSuchUser
+	}
+
+	if creds.IsServiceAccount() {
+		return true, creds.ParentUser, nil
+	}
+
+	return false, "", nil
+}
+
 // GetUserInfo - get info on a user.
 func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
@@ -725,7 +778,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		return errServerNotInitialized
 	}
 
-	if err := sys.store.saveUserIdentity(accessKey, false, uinfo); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, regularUser, uinfo); err != nil {
 		return err
 	}
 
@@ -733,6 +786,104 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 	return nil
 }
 
+// NewServiceAccount - create a new service account
+func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser, sessionPolicy string) (auth.Credentials, error) {
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	if len(sessionPolicy) > 16*1024 {
+		return auth.Credentials{}, fmt.Errorf("Session policy should not exceed 16*1024 characters")
+	}
+
+	policy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicy)))
+	if err != nil {
+		return auth.Credentials{}, err
+	}
+
+	// Version in policy must not be empty
+	if policy.Version == "" {
+		return auth.Credentials{}, fmt.Errorf("Invalid session policy version")
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	if sys.usersSysType != MinIOUsersSysType {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	if sys.store == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	if parentUser == globalActiveCred.AccessKey {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	cr, ok := sys.iamUsersMap[parentUser]
+	if !ok {
+		return auth.Credentials{}, errNoSuchUser
+	}
+
+	if cr.IsTemp() {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	m := make(map[string]interface{})
+	m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicy))
+	m[parentClaim] = parentUser
+	m[iamPolicyClaimName()] = "embedded-policy"
+
+	secret := globalActiveCred.SecretKey
+	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
+	if err != nil {
+		return auth.Credentials{}, err
+	}
+
+	cred.ParentUser = parentUser
+	u := newUserIdentity(cred)
+
+	if err := sys.store.saveUserIdentity(u.Credentials.AccessKey, srvAccUser, u); err != nil {
+		return auth.Credentials{}, err
+	}
+
+	sys.iamUsersMap[u.Credentials.AccessKey] = u.Credentials
+
+	return cred, nil
+}
+
+// GetServiceAccount - returns the credentials of the given service account
+func (sys *IAMSys) GetServiceAccount(ctx context.Context, serviceAccountAccessKey string) (auth.Credentials, error) {
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	if sys.usersSysType != MinIOUsersSysType {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	if sys.store == nil {
+		return auth.Credentials{}, errServerNotInitialized
+	}
+
+	cr, ok := sys.iamUsersMap[serviceAccountAccessKey]
+	if !ok {
+		return auth.Credentials{}, errNoSuchUser
+	}
+
+	if !cr.IsServiceAccount() {
+		return auth.Credentials{}, errIAMActionNotAllowed
+	}
+
+	return cr, nil
+}
+
 // SetUser - set user credentials and policy.
 func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 	objectAPI := newObjectLayerWithoutSafeModeFn()
@@ -762,7 +913,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 		return errIAMActionNotAllowed
 	}
 
-	if err := sys.store.saveUserIdentity(accessKey, false, u); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, regularUser, u); err != nil {
 		return err
 	}
 
@@ -770,7 +921,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 
 	// Set policy if specified.
 	if uinfo.PolicyName != "" {
-		return sys.policyDBSet(accessKey, uinfo.PolicyName, false, false)
+		return sys.policyDBSet(accessKey, uinfo.PolicyName, regularUser, false)
 	}
 	return nil
 }
@@ -800,7 +951,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 	cred.SecretKey = secretKey
 	u := newUserIdentity(cred)
-	if err := sys.store.saveUserIdentity(accessKey, false, u); err != nil {
+	if err := sys.store.saveUserIdentity(accessKey, regularUser, u); err != nil {
 		return err
 	}
 
@@ -931,7 +1082,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 		// Remove the group from storage. First delete the
 		// mapped policy.
-		err := sys.store.deleteMappedPolicy(group, false, true)
+		err := sys.store.deleteMappedPolicy(group, regularUser, true)
 		// No-mapped-policy case is ignored.
 		if err != nil && err != errConfigNotFound {
 			return err
@@ -1076,12 +1227,12 @@ func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
 
 	// isSTS is always false when called via PolicyDBSet as policy
 	// is never set by an external API call for STS users.
-	return sys.policyDBSet(name, policy, false, isGroup)
+	return sys.policyDBSet(name, policy, regularUser, isGroup)
 }
 
 // policyDBSet - sets a policy for user in the policy db. Assumes that caller
 // has sys.Lock(). If policy == "", then policy mapping is removed.
-func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
+func (sys *IAMSys) policyDBSet(name, policy string, userType IAMUserType, isGroup bool) error {
 	if sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1107,7 +1258,7 @@ func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
 
 	// Handle policy mapping removal
 	if policy == "" {
-		if err := sys.store.deleteMappedPolicy(name, isSTS, isGroup); err != nil {
+		if err := sys.store.deleteMappedPolicy(name, userType, isGroup); err != nil {
 			return err
 		}
 		if !isGroup {
@@ -1120,7 +1271,7 @@ func (sys *IAMSys) policyDBSet(name, policy string, isSTS, isGroup bool) error {
 
 	// Handle policy mapping set/update
 	mp := newMappedPolicy(policy)
-	if err := sys.store.saveMappedPolicy(name, isSTS, isGroup, mp); err != nil {
+	if err := sys.store.saveMappedPolicy(name, userType, isGroup, mp); err != nil {
 		return err
 	}
 	if !isGroup {
@@ -1302,6 +1453,93 @@ func (sys *IAMSys) policyDBGet(name string, isGroup bool) ([]string, error) {
 	return result, nil
 }
 
+// IsAllowedServiceAccount - checks if the given service account is allowed to perform
+// actions. The permission of the parent user is checked first
+func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) bool {
+	// Now check if we have a subject claim
+	p, ok := args.Claims[parentClaim]
+	if ok {
+		parentInClaim, ok := p.(string)
+		if !ok {
+			// Reject malformed/malicious requests.
+			return false
+		}
+		// The parent claim in the session token should be equal
+		// to the parent detected in the backend
+		if parentInClaim != parent {
+			return false
+		}
+	}
+
+	// Check if the parent is allowed to perform this action, reject if not
+	parentUserPolicies, err := sys.PolicyDBGet(parent, false)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return false
+	}
+
+	if len(parentUserPolicies) == 0 {
+		return false
+	}
+
+	var availablePolicies []iampolicy.Policy
+
+	// Policies were found, evaluate all of them.
+	sys.RLock()
+	for _, pname := range parentUserPolicies {
+		p, found := sys.iamPolicyDocsMap[pname]
+		if found {
+			availablePolicies = append(availablePolicies, p)
+		}
+	}
+	sys.RUnlock()
+
+	if len(availablePolicies) == 0 {
+		return false
+	}
+
+	combinedPolicy := availablePolicies[0]
+	for i := 1; i < len(availablePolicies); i++ {
+		combinedPolicy.Statements = append(combinedPolicy.Statements,
+			availablePolicies[i].Statements...)
+	}
+
+	serviceAcc := args.AccountName
+	args.AccountName = parent
+	if !combinedPolicy.IsAllowed(args) {
+		return false
+	}
+	args.AccountName = serviceAcc
+
+	// Now check if we have a sessionPolicy.
+	spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
+	if ok {
+		spolicyStr, ok := spolicy.(string)
+		if !ok {
+			// Sub policy if set, should be a string reject
+			// malformed/malicious requests.
+			return false
+		}
+
+		// Check if policy is parseable.
+		subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+		if err != nil {
+			// Log any error in input session policy config.
+			logger.LogIf(context.Background(), err)
+			return false
+		}
+
+		// Policy without Version string value reject it.
+		if subPolicy.Version == "" {
+			return false
+		}
+
+		return subPolicy.IsAllowed(args)
+	}
+
+	return false
+}
+
 // IsAllowedSTS is meant for STS based temporary credentials,
 // which implements claims validation and verification other than
 // applying policies.
@@ -1452,6 +1690,17 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 		return sys.IsAllowedSTS(args)
 	}
 
+	// If the credential is for a service account, perform related check
+	ok, parentUser, err := sys.IsServiceAccount(args.AccountName)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return false
+	}
+	if ok {
+		return sys.IsAllowedServiceAccount(args, parentUser)
+	}
+
+	// Continue with the assumption of a regular user
 	policies, err := sys.PolicyDBGet(args.AccountName, false)
 	if err != nil {
 		logger.LogIf(context.Background(), err)

cmd/jwt.go

@@ -86,7 +86,6 @@ func authenticateNode(accessKey, secretKey, audience string) (string, error) {
 	claims.SetExpiry(UTCNow().Add(defaultInterNodeJWTExpiry))
 	claims.SetAccessKey(accessKey)
 	claims.SetAudience(audience)
-	claims.SetIssuer(ReleaseTag)
 
 	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
 	return jwt.SignedString([]byte(secretKey))

cmd/jwt/parser_test.go

@@ -27,7 +27,6 @@ import (
 	"time"
 
 	"github.com/dgrijalva/jwt-go"
-	jwtgo "github.com/dgrijalva/jwt-go"
 )
 
 var (
@@ -49,7 +48,7 @@ var jwtTestData = []struct {
 		"",
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -61,7 +60,7 @@ var jwtTestData = []struct {
 		"", // autogen
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 				"exp": float64(time.Now().Unix() - 100),
 			},
@@ -74,7 +73,7 @@ var jwtTestData = []struct {
 		"", // autogen
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 				"nbf": float64(time.Now().Unix() + 100),
 			},
@@ -87,7 +86,7 @@ var jwtTestData = []struct {
 		"", // autogen
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 				"nbf": float64(time.Now().Unix() + 100),
 				"exp": float64(time.Now().Unix() - 100),
@@ -101,7 +100,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.EhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		defaultKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -113,7 +112,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		nil,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -125,7 +124,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		emptyKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -137,7 +136,7 @@ var jwtTestData = []struct {
 		"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg",
 		errorKeyFunc,
 		&MapClaims{
-			MapClaims: jwtgo.MapClaims{
+			MapClaims: jwt.MapClaims{
 				"foo": "bar",
 			},
 		},
@@ -149,7 +148,7 @@ var jwtTestData = []struct {
 		"",
 		defaultKeyFunc,
 		&StandardClaims{
-			StandardClaims: jwtgo.StandardClaims{
+			StandardClaims: jwt.StandardClaims{
 				ExpiresAt: time.Now().Add(time.Second * 10).Unix(),
 			},
 		},
@@ -160,7 +159,7 @@ var jwtTestData = []struct {
 
 func mapClaimsToken(claims *MapClaims) string {
 	claims.SetAccessKey("test")
-	j := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
+	j := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
 	tk, _ := j.SignedString([]byte("HelloSecret"))
 	return tk
 }
@@ -168,7 +167,7 @@ func mapClaimsToken(claims *MapClaims) string {
 func standardClaimsToken(claims *StandardClaims) string {
 	claims.AccessKey = "test"
 	claims.Subject = "test"
-	j := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
+	j := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
 	tk, _ := j.SignedString([]byte("HelloSecret"))
 	return tk
 }

cmd/local-locker.go

@@ -49,12 +49,42 @@ func (l *localLocker) String() string {
 	return l.endpoint.String()
 }
 
+func (l *localLocker) canTakeUnlock(resources ...string) bool {
+	var lkCnt int
+	for _, resource := range resources {
+		isWriteLockTaken := isWriteLock(l.lockMap[resource])
+		if isWriteLockTaken {
+			lkCnt++
+		}
+	}
+	return lkCnt == len(resources)
+}
+
+func (l *localLocker) canTakeLock(resources ...string) bool {
+	var noLkCnt int
+	for _, resource := range resources {
+		_, lockTaken := l.lockMap[resource]
+		if !lockTaken {
+			noLkCnt++
+		}
+	}
+	return noLkCnt == len(resources)
+}
+
 func (l *localLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
-	_, isLockTaken := l.lockMap[args.Resource]
-	if !isLockTaken { // No locks held on the given name, so claim write lock
-		l.lockMap[args.Resource] = []lockRequesterInfo{
+
+	if !l.canTakeLock(args.Resources...) {
+		// Not all locks can be taken on resources,
+		// reject it completely.
+		return false, nil
+	}
+
+	// No locks held on the all resources, so claim write
+	// lock on all resources at once.
+	for _, resource := range args.Resources {
+		l.lockMap[resource] = []lockRequesterInfo{
 			{
 				Writer:        true,
 				Source:        args.Source,
@@ -64,24 +94,22 @@ func (l *localLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
 			},
 		}
 	}
-	// return reply=true if lock was granted.
-	return !isLockTaken, nil
+	return true, nil
 }
 
 func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
-	var lri []lockRequesterInfo
-	if lri, reply = l.lockMap[args.Resource]; !reply {
-		// No lock is held on the given name
-		return reply, fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.Resource)
+
+	if !l.canTakeUnlock(args.Resources...) {
+		// Unless it is a write lock reject it.
+		return reply, fmt.Errorf("Unlock attempted on a read locked entity: %s", args.Resources)
 	}
-	if reply = isWriteLock(lri); !reply {
-		// Unless it is a write lock
-		return reply, fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Resource, len(lri))
-	}
-	if !l.removeEntry(args.Resource, args.UID, &lri) {
-		return false, fmt.Errorf("Unlock unable to find corresponding lock for uid: %s", args.UID)
+	for _, resource := range args.Resources {
+		lri := l.lockMap[resource]
+		if !l.removeEntry(resource, args.UID, &lri) {
+			return false, fmt.Errorf("Unlock unable to find corresponding lock for uid: %s on resource %s", args.UID, resource)
+		}
 	}
 	return true, nil
 
@@ -120,14 +148,15 @@ func (l *localLocker) RLock(args dsync.LockArgs) (reply bool, err error) {
 		Timestamp:     UTCNow(),
 		TimeLastCheck: UTCNow(),
 	}
-	if lri, ok := l.lockMap[args.Resource]; ok {
+	resource := args.Resources[0]
+	if lri, ok := l.lockMap[resource]; ok {
 		if reply = !isWriteLock(lri); reply {
 			// Unless there is a write lock
-			l.lockMap[args.Resource] = append(l.lockMap[args.Resource], lrInfo)
+			l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
 		}
 	} else {
 		// No locks held on the given name, so claim (first) read lock
-		l.lockMap[args.Resource] = []lockRequesterInfo{lrInfo}
+		l.lockMap[resource] = []lockRequesterInfo{lrInfo}
 		reply = true
 	}
 	return reply, nil
@@ -137,15 +166,17 @@ func (l *localLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	var lri []lockRequesterInfo
-	if lri, reply = l.lockMap[args.Resource]; !reply {
+
+	resource := args.Resources[0]
+	if lri, reply = l.lockMap[resource]; !reply {
 		// No lock is held on the given name
-		return reply, fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.Resource)
+		return reply, fmt.Errorf("RUnlock attempted on an unlocked entity: %s", resource)
 	}
 	if reply = !isWriteLock(lri); !reply {
 		// A write-lock is held, cannot release a read lock
-		return reply, fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Resource)
+		return reply, fmt.Errorf("RUnlock attempted on a write locked entity: %s", resource)
 	}
-	if !l.removeEntry(args.Resource, args.UID, &lri) {
+	if !l.removeEntry(resource, args.UID, &lri) {
 		return false, fmt.Errorf("RUnlock unable to find corresponding read lock for uid: %s", args.UID)
 	}
 	return reply, nil
@@ -176,11 +207,13 @@ func (l *localLocker) Expired(args dsync.LockArgs) (expired bool, err error) {
 	defer l.mutex.Unlock()
 
 	// Lock found, proceed to verify if belongs to given uid.
-	if lri, ok := l.lockMap[args.Resource]; ok {
-		// Check whether uid is still active
-		for _, entry := range lri {
-			if entry.UID == args.UID {
-				return false, nil
+	for _, resource := range args.Resources {
+		if lri, ok := l.lockMap[resource]; ok {
+			// Check whether uid is still active
+			for _, entry := range lri {
+				if entry.UID == args.UID {
+					return false, nil
+				}
 			}
 		}
 	}

cmd/lock-rest-client.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"crypto/tls"
 	"errors"
@@ -104,9 +105,12 @@ func (client *lockRESTClient) restCall(call string, args dsync.LockArgs) (reply
 	values := url.Values{}
 	values.Set(lockRESTUID, args.UID)
 	values.Set(lockRESTSource, args.Source)
-	values.Set(lockRESTResource, args.Resource)
-
-	respBody, err := client.call(call, values, nil, -1)
+	var buffer bytes.Buffer
+	for _, resource := range args.Resources {
+		buffer.WriteString(resource)
+		buffer.WriteString("\n")
+	}
+	respBody, err := client.call(call, values, &buffer, -1)
 	defer http.DrainBody(respBody)
 	switch err {
 	case nil:
@@ -143,12 +147,6 @@ func (client *lockRESTClient) Expired(args dsync.LockArgs) (expired bool, err er
 	return client.restCall(lockRESTMethodExpired, args)
 }
 
-func closeLockers(lockers []dsync.NetLocker) {
-	for _, locker := range lockers {
-		locker.Close()
-	}
-}
-
 func newLockAPI(endpoint Endpoint) dsync.NetLocker {
 	if endpoint.IsLocal {
 		return globalLockServers[endpoint]

cmd/lock-rest-server-common.go

@@ -21,8 +21,8 @@ import (
 )
 
 const (
-	lockRESTVersion       = "v2"
-	lockRESTVersionPrefix = SlashSeparator + "v2"
+	lockRESTVersion       = "v3"
+	lockRESTVersionPrefix = SlashSeparator + lockRESTVersion
 	lockRESTPrefix        = minioReservedBucketPath + "/lock"
 )
 
@@ -38,8 +38,6 @@ const (
 	// Source contains the line number, function and file name of the code
 	// on the client node that requested the lock.
 	lockRESTSource = "source"
-	// Resource contains a entity to be locked/unlocked.
-	lockRESTResource = "resource"
 )
 
 var (

cmd/lock-rest-server.go

@@ -17,11 +17,13 @@
 package cmd
 
 import (
+	"bufio"
 	"context"
 	"errors"
 	"math/rand"
 	"net/http"
 	"path"
+	"sort"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -55,12 +57,25 @@ func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
-func getLockArgs(r *http.Request) dsync.LockArgs {
-	return dsync.LockArgs{
-		UID:      r.URL.Query().Get(lockRESTUID),
-		Source:   r.URL.Query().Get(lockRESTSource),
-		Resource: r.URL.Query().Get(lockRESTResource),
+func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
+	args = dsync.LockArgs{
+		UID:    r.URL.Query().Get(lockRESTUID),
+		Source: r.URL.Query().Get(lockRESTSource),
 	}
+
+	var resources []string
+	bio := bufio.NewScanner(r.Body)
+	for bio.Scan() {
+		resources = append(resources, bio.Text())
+	}
+
+	if err := bio.Err(); err != nil {
+		return args, err
+	}
+
+	sort.Strings(resources)
+	args.Resources = resources
+	return args, nil
 }
 
 // LockHandler - Acquires a lock.
@@ -70,7 +85,13 @@ func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	success, err := l.ll.Lock(getLockArgs(r))
+	args, err := getLockArgs(r)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	success, err := l.ll.Lock(args)
 	if err == nil && !success {
 		err = errLockConflict
 	}
@@ -87,7 +108,13 @@ func (l *lockRESTServer) UnlockHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	_, err := l.ll.Unlock(getLockArgs(r))
+	args, err := getLockArgs(r)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	_, err = l.ll.Unlock(args)
 	// Ignore the Unlock() "reply" return value because if err == nil, "reply" is always true
 	// Consequently, if err != nil, reply is always false
 	if err != nil {
@@ -103,7 +130,13 @@ func (l *lockRESTServer) RLockHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	success, err := l.ll.RLock(getLockArgs(r))
+	args, err := getLockArgs(r)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	success, err := l.ll.RLock(args)
 	if err == nil && !success {
 		err = errLockConflict
 	}
@@ -120,10 +153,15 @@ func (l *lockRESTServer) RUnlockHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	args, err := getLockArgs(r)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
 	// Ignore the RUnlock() "reply" return value because if err == nil, "reply" is always true.
 	// Consequently, if err != nil, reply is always false
-	_, err := l.ll.RUnlock(getLockArgs(r))
-	if err != nil {
+	if _, err = l.ll.RUnlock(args); err != nil {
 		l.writeErrorResponse(w, err)
 		return
 	}
@@ -136,17 +174,24 @@ func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	lockArgs := getLockArgs(r)
+	args, err := getLockArgs(r)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
 
 	l.ll.mutex.Lock()
 	defer l.ll.mutex.Unlock()
+
 	// Lock found, proceed to verify if belongs to given uid.
-	if lri, ok := l.ll.lockMap[lockArgs.Resource]; ok {
-		// Check whether uid is still active
-		for _, entry := range lri {
-			if entry.UID == lockArgs.UID {
-				l.writeErrorResponse(w, errLockNotExpired)
-				return
+	for _, resource := range args.Resources {
+		if lri, ok := l.ll.lockMap[resource]; ok {
+			// Check whether uid is still active
+			for _, entry := range lri {
+				if entry.UID == args.UID {
+					l.writeErrorResponse(w, errLockNotExpired)
+					return
+				}
 			}
 		}
 	}
@@ -180,8 +225,6 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 	return nlripMap
 }
 
-var lockMaintenanceTimeout = newDynamicTimeout(60*time.Second, time.Second)
-
 // lockMaintenance loops over locks that have been active for some time and checks back
 // with the original server whether it is still alive or not
 //
@@ -190,55 +233,59 @@ var lockMaintenanceTimeout = newDynamicTimeout(60*time.Second, time.Second)
 // - some network error (and server is up normally)
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
-func lockMaintenance(ctx context.Context, interval time.Duration, objAPI ObjectLayer) error {
-	// Lock to avoid concurrent lock maintenance loops
-	maintenanceLock := objAPI.NewNSLock(ctx, "system", "lock-maintenance-ops")
-	if err := maintenanceLock.GetLock(lockMaintenanceTimeout); err != nil {
-		return err
-	}
-	defer maintenanceLock.Unlock()
-
+func lockMaintenance(ctx context.Context, interval time.Duration) error {
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
 	for lendpoint, nlrips := range getLongLivedLocks(interval) {
+		nlripsMap := make(map[string]int, len(nlrips))
 		for _, nlrip := range nlrips {
-			for _, ep := range globalEndpoints {
-				for _, endpoint := range ep.Endpoints {
-					if endpoint.String() == lendpoint.String() {
-						continue
-					}
-
-					c := newLockAPI(endpoint)
-					if !c.IsOnline() {
-						continue
-					}
-
-					// Call back to original server verify whether the lock is
-					// still active (based on name & uid)
-					expired, err := c.Expired(dsync.LockArgs{
-						UID:      nlrip.lri.UID,
-						Resource: nlrip.name,
-					})
-
-					if err != nil {
-						c.Close()
-						continue
-					}
-
-					// For successful response, verify if lock was indeed active or stale.
-					if expired {
-						// The lock is no longer active at server that originated
-						// the lock, attempt to remove the lock.
-						globalLockServers[lendpoint].mutex.Lock()
-						// Purge the stale entry if it exists.
-						globalLockServers[lendpoint].removeEntryIfExists(nlrip)
-						globalLockServers[lendpoint].mutex.Unlock()
-					}
-
-					// Close the connection regardless of the call response.
-					c.Close()
+			// Locks are only held on first zone, make sure that
+			// we only look for ownership of locks from endpoints
+			// on first zone.
+			for _, endpoint := range globalEndpoints[0].Endpoints {
+				c := newLockAPI(endpoint)
+				if !c.IsOnline() {
+					nlripsMap[nlrip.name]++
+					continue
 				}
+
+				// Call back to original server verify whether the lock is
+				// still active (based on name & uid)
+				expired, err := c.Expired(dsync.LockArgs{
+					UID:       nlrip.lri.UID,
+					Resources: []string{nlrip.name},
+				})
+				if err != nil {
+					nlripsMap[nlrip.name]++
+					c.Close()
+					continue
+				}
+
+				if !expired {
+					nlripsMap[nlrip.name]++
+				}
+
+				// Close the connection regardless of the call response.
+				c.Close()
 			}
+
+			// Read locks we assume quorum for be N/2 success
+			quorum := globalXLSetDriveCount / 2
+			if nlrip.lri.Writer {
+				// For write locks we need N/2+1 success
+				quorum = globalXLSetDriveCount/2 + 1
+			}
+
+			// less than the quorum, we have locks expired.
+			if nlripsMap[nlrip.name] < quorum {
+				// The lock is no longer active at server that originated
+				// the lock, attempt to remove the lock.
+				globalLockServers[lendpoint].mutex.Lock()
+				// Purge the stale entry if it exists.
+				globalLockServers[lendpoint].removeEntryIfExists(nlrip)
+				globalLockServers[lendpoint].mutex.Unlock()
+			}
+
 		}
 	}
 
@@ -247,12 +294,13 @@ func lockMaintenance(ctx context.Context, interval time.Duration, objAPI ObjectL
 
 // Start lock maintenance from all lock servers.
 func startLockMaintenance() {
-	var objAPI ObjectLayer
 	var ctx = context.Background()
 
 	// Wait until the object API is ready
+	// no need to start the lock maintenance
+	// if ObjectAPI is not initialized.
 	for {
-		objAPI = newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue
@@ -276,7 +324,7 @@ func startLockMaintenance() {
 			// "synchronous checks" between servers
 			duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
 			time.Sleep(duration)
-			if err := lockMaintenance(ctx, lockValidityCheckInterval, objAPI); err != nil {
+			if err := lockMaintenance(ctx, lockValidityCheckInterval); err != nil {
 				// Sleep right after an error.
 				duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
 				time.Sleep(duration)
@@ -287,7 +335,7 @@ func startLockMaintenance() {
 
 // registerLockRESTHandlers - register lock rest router.
 func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
-	queries := restQueries(lockRESTUID, lockRESTSource, lockRESTResource)
+	queries := restQueries(lockRESTUID, lockRESTSource)
 	for _, ep := range endpointZones {
 		for _, endpoint := range ep.Endpoints {
 			if !endpoint.IsLocal {

cmd/logger/target/http/http.go

@@ -21,7 +21,6 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
-	gohttp "net/http"
 	"strings"
 
 	xhttp "github.com/minio/minio/cmd/http"
@@ -41,7 +40,7 @@ type Target struct {
 	// User-Agent to be set on each log request sent to the `endpoint`
 	userAgent string
 	logKind   string
-	client    gohttp.Client
+	client    http.Client
 }
 
 func (h *Target) startHTTPLogger() {
@@ -54,7 +53,7 @@ func (h *Target) startHTTPLogger() {
 				continue
 			}
 
-			req, err := gohttp.NewRequest(http.MethodPost, h.endpoint, bytes.NewBuffer(logJSON))
+			req, err := http.NewRequest(http.MethodPost, h.endpoint, bytes.NewBuffer(logJSON))
 			if err != nil {
 				continue
 			}
@@ -78,12 +77,12 @@ func (h *Target) startHTTPLogger() {
 
 // New initializes a new logger target which
 // sends log over http to the specified endpoint
-func New(endpoint, userAgent, logKind string, transport *gohttp.Transport) *Target {
+func New(endpoint, userAgent, logKind string, transport *http.Transport) *Target {
 	h := Target{
 		endpoint:  endpoint,
 		userAgent: userAgent,
 		logKind:   strings.ToUpper(logKind),
-		client: gohttp.Client{
+		client: http.Client{
 			Transport: transport,
 		},
 		logCh: make(chan interface{}, 10000),

cmd/main.go

@@ -74,7 +74,7 @@ FLAGS:
   {{range .VisibleFlags}}{{.}}
   {{end}}{{end}}
 VERSION:
-{{.Version}}
+  {{.Version}}
 `
 
 func newApp(name string) *cli.App {

cmd/namespace-lock.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	pathutil "path"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 
@@ -51,16 +52,10 @@ func newNSLock(isDistXL bool) *nsLockMap {
 	if isDistXL {
 		return &nsMutex
 	}
-	nsMutex.lockMap = make(map[nsParam]*nsLock)
+	nsMutex.lockMap = make(map[string]*nsLock)
 	return &nsMutex
 }
 
-// nsParam - carries name space resource.
-type nsParam struct {
-	volume string
-	path   string
-}
-
 // nsLock - provides primitives for locking critical namespace regions.
 type nsLock struct {
 	*lsync.LRWMutex
@@ -72,23 +67,24 @@ type nsLock struct {
 type nsLockMap struct {
 	// Indicates if namespace is part of a distributed setup.
 	isDistXL     bool
-	lockMap      map[nsParam]*nsLock
+	lockMap      map[string]*nsLock
 	lockMapMutex sync.RWMutex
 }
 
 // Lock the namespace resource.
-func (n *nsLockMap) lock(ctx context.Context, volume, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) {
+func (n *nsLockMap) lock(ctx context.Context, volume string, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) {
 	var nsLk *nsLock
 
+	resource := pathJoin(volume, path)
+
 	n.lockMapMutex.Lock()
-	param := nsParam{volume, path}
-	nsLk, found := n.lockMap[param]
+	nsLk, found := n.lockMap[resource]
 	if !found {
-		n.lockMap[param] = &nsLock{
+		nsLk = &nsLock{
 			LRWMutex: lsync.NewLRWMutex(ctx),
 			ref:      1,
 		}
-		nsLk = n.lockMap[param]
+		n.lockMap[resource] = nsLk
 	} else {
 		// Update ref count here to avoid multiple races.
 		nsLk.ref++
@@ -109,7 +105,7 @@ func (n *nsLockMap) lock(ctx context.Context, volume, path string, lockSource, o
 		nsLk.ref--
 		if nsLk.ref == 0 {
 			// Remove from the map if there are no more references.
-			delete(n.lockMap, param)
+			delete(n.lockMap, resource)
 		}
 		n.lockMapMutex.Unlock()
 	}
@@ -117,10 +113,10 @@ func (n *nsLockMap) lock(ctx context.Context, volume, path string, lockSource, o
 }
 
 // Unlock the namespace resource.
-func (n *nsLockMap) unlock(volume, path string, readLock bool) {
-	param := nsParam{volume, path}
+func (n *nsLockMap) unlock(volume string, path string, readLock bool) {
+	resource := pathJoin(volume, path)
 	n.lockMapMutex.RLock()
-	nsLk, found := n.lockMap[param]
+	nsLk, found := n.lockMap[resource]
 	n.lockMapMutex.RUnlock()
 	if !found {
 		return
@@ -137,45 +133,16 @@ func (n *nsLockMap) unlock(volume, path string, readLock bool) {
 		nsLk.ref--
 		if nsLk.ref == 0 {
 			// Remove from the map if there are no more references.
-			delete(n.lockMap, param)
+			delete(n.lockMap, resource)
 		}
 	}
 	n.lockMapMutex.Unlock()
 }
 
-// Lock - locks the given resource for writes, using a previously
-// allocated name space lock or initializing a new one.
-func (n *nsLockMap) Lock(volume, path, opsID string, timeout time.Duration) (locked bool) {
-	readLock := false // This is a write lock.
-
-	lockSource := getSource() // Useful for debugging
-	return n.lock(context.Background(), volume, path, lockSource, opsID, readLock, timeout)
-}
-
-// Unlock - unlocks any previously acquired write locks.
-func (n *nsLockMap) Unlock(volume, path, opsID string) {
-	readLock := false
-	n.unlock(volume, path, readLock)
-}
-
-// RLock - locks any previously acquired read locks.
-func (n *nsLockMap) RLock(volume, path, opsID string, timeout time.Duration) (locked bool) {
-	readLock := true
-
-	lockSource := getSource() // Useful for debugging
-	return n.lock(context.Background(), volume, path, lockSource, opsID, readLock, timeout)
-}
-
-// RUnlock - unlocks any previously acquired read locks.
-func (n *nsLockMap) RUnlock(volume, path, opsID string) {
-	readLock := true
-	n.unlock(volume, path, readLock)
-}
-
 // dsync's distributed lock instance.
 type distLockInstance struct {
-	rwMutex             *dsync.DRWMutex
-	volume, path, opsID string
+	rwMutex *dsync.DRWMutex
+	opsID   string
 }
 
 // Lock - block until write lock is taken or timeout has occurred.
@@ -185,7 +152,7 @@ func (di *distLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error)
 
 	if !di.rwMutex.GetLock(di.opsID, lockSource, timeout.Timeout()) {
 		timeout.LogFailure()
-		return OperationTimedOut{Path: di.path}
+		return OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
 	return nil
@@ -202,7 +169,7 @@ func (di *distLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr error
 	start := UTCNow()
 	if !di.rwMutex.GetRLock(di.opsID, lockSource, timeout.Timeout()) {
 		timeout.LogFailure()
-		return OperationTimedOut{Path: di.path}
+		return OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
 	return nil
@@ -215,22 +182,26 @@ func (di *distLockInstance) RUnlock() {
 
 // localLockInstance - frontend/top-level interface for namespace locks.
 type localLockInstance struct {
-	ctx                 context.Context
-	ns                  *nsLockMap
-	volume, path, opsID string
+	ctx    context.Context
+	ns     *nsLockMap
+	volume string
+	paths  []string
+	opsID  string
 }
 
 // NewNSLock - returns a lock instance for a given volume and
 // path. The returned lockInstance object encapsulates the nsLockMap,
 // volume, path and operation ID.
-func (n *nsLockMap) NewNSLock(ctx context.Context, lockersFn func() []dsync.NetLocker, volume, path string) RWLocker {
+func (n *nsLockMap) NewNSLock(ctx context.Context, lockersFn func() []dsync.NetLocker, volume string, paths ...string) RWLocker {
 	opsID := mustGetUUID()
 	if n.isDistXL {
-		return &distLockInstance{dsync.NewDRWMutex(ctx, pathJoin(volume, path), &dsync.Dsync{
+		drwmutex := dsync.NewDRWMutex(ctx, &dsync.Dsync{
 			GetLockersFn: lockersFn,
-		}), volume, path, opsID}
+		}, pathsJoinPrefix(volume, paths...)...)
+		return &distLockInstance{drwmutex, opsID}
 	}
-	return &localLockInstance{ctx, n, volume, path, opsID}
+	sort.Strings(paths)
+	return &localLockInstance{ctx, n, volume, paths, opsID}
 }
 
 // Lock - block until write lock is taken or timeout has occurred.
@@ -238,9 +209,16 @@ func (li *localLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error
 	lockSource := getSource()
 	start := UTCNow()
 	readLock := false
-	if !li.ns.lock(li.ctx, li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
-		timeout.LogFailure()
-		return OperationTimedOut{Path: li.path}
+	var success []int
+	for i, path := range li.paths {
+		if !li.ns.lock(li.ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+			timeout.LogFailure()
+			for _, sint := range success {
+				li.ns.unlock(li.volume, li.paths[sint], readLock)
+			}
+			return OperationTimedOut{}
+		}
+		success = append(success, i)
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
 	return
@@ -249,7 +227,9 @@ func (li *localLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error
 // Unlock - block until write lock is released.
 func (li *localLockInstance) Unlock() {
 	readLock := false
-	li.ns.unlock(li.volume, li.path, readLock)
+	for _, path := range li.paths {
+		li.ns.unlock(li.volume, path, readLock)
+	}
 }
 
 // RLock - block until read lock is taken or timeout has occurred.
@@ -257,9 +237,16 @@ func (li *localLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr erro
 	lockSource := getSource()
 	start := UTCNow()
 	readLock := true
-	if !li.ns.lock(li.ctx, li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
-		timeout.LogFailure()
-		return OperationTimedOut{Path: li.path}
+	var success []int
+	for i, path := range li.paths {
+		if !li.ns.lock(li.ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+			timeout.LogFailure()
+			for _, sint := range success {
+				li.ns.unlock(li.volume, li.paths[sint], readLock)
+			}
+			return OperationTimedOut{}
+		}
+		success = append(success, i)
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
 	return
@@ -268,7 +255,9 @@ func (li *localLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr erro
 // RUnlock - block until read lock is released.
 func (li *localLockInstance) RUnlock() {
 	readLock := true
-	li.ns.unlock(li.volume, li.path, readLock)
+	for _, path := range li.paths {
+		li.ns.unlock(li.volume, path, readLock)
+	}
 }
 
 func getSource() string {

cmd/namespace-lock_test.go

@@ -18,184 +18,20 @@ package cmd
 
 import (
 	"testing"
-	"time"
 )
 
 // WARNING:
 //
-// Expected source line number is hard coded, 32, in the
+// Expected source line number is hard coded, 31, in the
 // following test. Adding new code before this test or changing its
 // position will cause the line number to change and the test to FAIL
 // Tests getSource().
 func TestGetSource(t *testing.T) {
 	currentSource := func() string { return getSource() }
 	gotSource := currentSource()
-	// Hard coded line number, 32, in the "expectedSource" value
-	expectedSource := "[namespace-lock_test.go:32:TestGetSource()]"
+	// Hard coded line number, 31, in the "expectedSource" value
+	expectedSource := "[namespace-lock_test.go:31:TestGetSource()]"
 	if gotSource != expectedSource {
 		t.Errorf("expected : %s, got : %s", expectedSource, gotSource)
 	}
 }
-
-// Tests functionality provided by namespace lock.
-func TestNamespaceLockTest(t *testing.T) {
-	isDistXL := false
-	nsMutex := newNSLock(isDistXL)
-
-	// List of test cases.
-	testCases := []struct {
-		lk               func(s1, s2, s3 string, t time.Duration) bool
-		unlk             func(s1, s2, s3 string)
-		rlk              func(s1, s2, s3 string, t time.Duration) bool
-		runlk            func(s1, s2, s3 string)
-		lockedRefCount   uint
-		unlockedRefCount uint
-		shouldPass       bool
-	}{
-		{
-			lk:               nsMutex.Lock,
-			unlk:             nsMutex.Unlock,
-			lockedRefCount:   1,
-			unlockedRefCount: 0,
-			shouldPass:       true,
-		},
-		{
-			rlk:              nsMutex.RLock,
-			runlk:            nsMutex.RUnlock,
-			lockedRefCount:   4,
-			unlockedRefCount: 2,
-			shouldPass:       true,
-		},
-		{
-			rlk:              nsMutex.RLock,
-			runlk:            nsMutex.RUnlock,
-			lockedRefCount:   1,
-			unlockedRefCount: 0,
-			shouldPass:       true,
-		},
-	}
-
-	// Run all test cases.
-
-	// Write lock tests.
-	testCase := testCases[0]
-	if !testCase.lk("a", "b", "c", 60*time.Second) { // lock once.
-		t.Fatalf("Failed to acquire lock")
-	}
-	nsLk, ok := nsMutex.lockMap[nsParam{"a", "b"}]
-	if !ok && testCase.shouldPass {
-		t.Errorf("Lock in map missing.")
-	}
-	// Validate locked ref count.
-	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
-		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref)
-	}
-	testCase.unlk("a", "b", "c") // unlock once.
-	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
-		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.unlockedRefCount, nsLk.ref)
-	}
-	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
-	if ok && !testCase.shouldPass {
-		t.Errorf("Lock map found after unlock.")
-	}
-
-	// Read lock tests.
-	testCase = testCases[1]
-	if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock once.
-		t.Fatalf("Failed to acquire first read lock")
-	}
-	if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock second time.
-		t.Fatalf("Failed to acquire second read lock")
-	}
-	if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock third time.
-		t.Fatalf("Failed to acquire third read lock")
-	}
-	if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock fourth time.
-		t.Fatalf("Failed to acquire fourth read lock")
-	}
-	nsLk, ok = nsMutex.lockMap[nsParam{"a", "b"}]
-	if !ok && testCase.shouldPass {
-		t.Errorf("Lock in map missing.")
-	}
-	// Validate locked ref count.
-	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
-		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.lockedRefCount, nsLk.ref)
-	}
-
-	testCase.runlk("a", "b", "c") // unlock once.
-	testCase.runlk("a", "b", "c") // unlock second time.
-	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
-		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 2, testCase.unlockedRefCount, nsLk.ref)
-	}
-	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
-	if !ok && testCase.shouldPass {
-		t.Errorf("Lock map not found.")
-	}
-
-	// Read lock 0 ref count.
-	testCase = testCases[2]
-	if !testCase.rlk("a", "c", "d", 60*time.Second) { // lock once.
-		t.Fatalf("Failed to acquire read lock")
-	}
-
-	nsLk, ok = nsMutex.lockMap[nsParam{"a", "c"}]
-	if !ok && testCase.shouldPass {
-		t.Errorf("Lock in map missing.")
-	}
-	// Validate locked ref count.
-	if testCase.lockedRefCount != nsLk.ref && testCase.shouldPass {
-		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.lockedRefCount, nsLk.ref)
-	}
-	testCase.runlk("a", "c", "d") // unlock once.
-	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
-		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.unlockedRefCount, nsLk.ref)
-	}
-	_, ok = nsMutex.lockMap[nsParam{"a", "c"}]
-	if ok && !testCase.shouldPass {
-		t.Errorf("Lock map not found.")
-	}
-}
-
-func TestNamespaceLockTimedOut(t *testing.T) {
-	isDistXL := false
-	nsMutex := newNSLock(isDistXL)
-	// Get write lock
-	if !nsMutex.Lock("my-bucket", "my-object", "abc", 60*time.Second) {
-		t.Fatalf("Failed to acquire lock")
-	}
-
-	// Second attempt for write lock on same resource should time out
-	locked := nsMutex.Lock("my-bucket", "my-object", "def", 1*time.Second)
-	if locked {
-		t.Fatalf("Should not have acquired lock")
-	}
-
-	// Read lock on same resource should also time out
-	locked = nsMutex.RLock("my-bucket", "my-object", "def", 1*time.Second)
-	if locked {
-		t.Fatalf("Should not have acquired read lock while write lock is active")
-	}
-
-	// Release write lock
-	nsMutex.Unlock("my-bucket", "my-object", "abc")
-
-	// Get read lock
-	if !nsMutex.RLock("my-bucket", "my-object", "ghi", 60*time.Second) {
-		t.Fatalf("Failed to acquire read lock")
-	}
-
-	// Write lock on same resource should time out
-	locked = nsMutex.Lock("my-bucket", "my-object", "klm", 1*time.Second)
-	if locked {
-		t.Fatalf("Should not have acquired lock")
-	}
-
-	// 2nd read lock should be just fine
-	if !nsMutex.RLock("my-bucket", "my-object", "nop", 60*time.Second) {
-		t.Fatalf("Failed to acquire second read lock")
-	}
-
-	// Release both read locks
-	nsMutex.RUnlock("my-bucket", "my-object", "ghi")
-	nsMutex.RUnlock("my-bucket", "my-object", "nop")
-}

cmd/naughty-disk_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"io"
 	"sync"
 )
@@ -80,8 +81,8 @@ func (d *naughtyDisk) calcError() (err error) {
 func (d *naughtyDisk) SetDiskID(id string) {
 }
 
-func (d *naughtyDisk) CrawlAndGetDataUsage(endCh <-chan struct{}) (info DataUsageInfo, err error) {
-	return d.disk.CrawlAndGetDataUsage(endCh)
+func (d *naughtyDisk) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (info dataUsageCache, err error) {
+	return d.disk.CrawlAndGetDataUsage(ctx, cache)
 }
 
 func (d *naughtyDisk) DiskInfo() (info DiskInfo, err error) {
@@ -125,7 +126,7 @@ func (d *naughtyDisk) DeleteVol(volume string) (err error) {
 	return d.disk.DeleteVol(volume)
 }
 
-func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, leafFile string, readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (chan FileInfo, error) {
+func (d *naughtyDisk) Walk(volume, path, marker string, recursive bool, leafFile string, readMetadataFn readMetadataFunc, endWalkCh <-chan struct{}) (chan FileInfo, error) {
 	if err := d.calcError(); err != nil {
 		return nil, err
 	}
@@ -196,6 +197,13 @@ func (d *naughtyDisk) DeleteFileBulk(volume string, paths []string) ([]error, er
 	return errs, nil
 }
 
+func (d *naughtyDisk) DeletePrefixes(volume string, paths []string) ([]error, error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.DeletePrefixes(volume, paths)
+}
+
 func (d *naughtyDisk) WriteAll(volume string, path string, reader io.Reader) (err error) {
 	if err := d.calcError(); err != nil {
 		return err

cmd/notification.go

@@ -326,7 +326,7 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		for typ, data := range data {
 			// Send profiling data to zip as file
 			header, zerr := zip.FileInfoHeader(dummyFileInfo{
-				name:    fmt.Sprintf("profiling-%s-%s.pprof", client.host.String(), typ),
+				name:    fmt.Sprintf("profile-%s-%s", client.host.String(), typ),
 				size:    int64(len(data)),
 				mode:    0600,
 				modTime: UTCNow(),
@@ -375,7 +375,7 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 	// Send profiling data to zip as file
 	for typ, data := range data {
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profiling-%s-%s.pprof", thisAddr, typ),
+			name:    fmt.Sprintf("profile-%s-%s", thisAddr, typ),
 			size:    int64(len(data)),
 			mode:    0600,
 			modTime: UTCNow(),
@@ -688,53 +688,6 @@ func (sys *NotificationSys) load(buckets []BucketInfo, objAPI ObjectLayer) error
 	return nil
 }
 
-func (sys *NotificationSys) initBucketObjectLockConfig(objAPI ObjectLayer) error {
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return err
-	}
-	for _, bucket := range buckets {
-		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
-		configFile := path.Join(bucketConfigPrefix, bucket.Name, bucketObjectLockEnabledConfigFile)
-		bucketObjLockData, err := readConfig(ctx, objAPI, configFile)
-
-		if err != nil {
-			if err == errConfigNotFound {
-				continue
-			}
-			return err
-		}
-
-		if string(bucketObjLockData) != bucketObjectLockEnabledConfig {
-			// this should never happen
-			logger.LogIf(ctx, objectlock.ErrMalformedBucketObjectConfig)
-			continue
-		}
-
-		configFile = path.Join(bucketConfigPrefix, bucket.Name, objectLockConfig)
-		configData, err := readConfig(ctx, objAPI, configFile)
-
-		if err != nil {
-			if err == errConfigNotFound {
-				globalBucketObjectLockConfig.Set(bucket.Name, objectlock.Retention{})
-				continue
-			}
-			return err
-		}
-
-		config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
-		if err != nil {
-			return err
-		}
-		retention := objectlock.Retention{}
-		if config.Rule != nil {
-			retention = config.ToRetention()
-		}
-		globalBucketObjectLockConfig.Set(bucket.Name, retention)
-	}
-	return nil
-}
-
 // Init - initializes notification system from notification.xml and listener.json of all buckets.
 func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
 	if objAPI == nil {
@@ -754,11 +707,7 @@ func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error
 		}
 	}
 
-	if err := sys.load(buckets, objAPI); err != nil {
-		return err
-	}
-
-	return sys.initBucketObjectLockConfig(objAPI)
+	return sys.load(buckets, objAPI)
 }
 
 // AddRulesMap - adds rules map for bucket name.
@@ -808,8 +757,14 @@ func (sys *NotificationSys) ConfiguredTargetIDs() []event.TargetID {
 			}
 		}
 	}
-
-	return targetIDs
+	// Filter out targets configured via env
+	var tIDs []event.TargetID
+	for _, targetID := range targetIDs {
+		if !globalEnvTargetList.Exists(targetID) {
+			tIDs = append(tIDs, targetID)
+		}
+	}
+	return tIDs
 }
 
 // RemoveNotification - removes all notification configuration for bucket name.

cmd/object-api-common.go

@@ -147,78 +147,6 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 	return err
 }
 
-// Cleanup objects in bulk and recursively: each object will have a list of sub-files to delete in the backend
-func cleanupObjectsBulk(storage StorageAPI, volume string, objsPaths []string, errs []error) ([]error, error) {
-	// The list of files in disk to delete
-	var filesToDelete []string
-	// Map files to delete to the passed objsPaths
-	var filesToDeleteObjsIndexes []int
-
-	// Traverse and return the list of sub entries
-	var traverse func(string) ([]string, error)
-	traverse = func(entryPath string) ([]string, error) {
-		var output = make([]string, 0)
-		if !HasSuffix(entryPath, SlashSeparator) {
-			output = append(output, entryPath)
-			return output, nil
-		}
-		entries, err := storage.ListDir(volume, entryPath, -1, "")
-		if err != nil {
-			if err == errFileNotFound {
-				return nil, nil
-			}
-			return nil, err
-		}
-
-		for _, entry := range entries {
-			subEntries, err := traverse(pathJoin(entryPath, entry))
-			if err != nil {
-				return nil, err
-			}
-			output = append(output, subEntries...)
-		}
-		return output, nil
-	}
-
-	// Find and collect the list of files to remove associated
-	// to the passed objects paths
-	for idx, objPath := range objsPaths {
-		if errs[idx] != nil {
-			continue
-		}
-		output, err := traverse(retainSlash(pathJoin(objPath)))
-		if err != nil {
-			errs[idx] = err
-			continue
-		} else {
-			errs[idx] = nil
-		}
-		filesToDelete = append(filesToDelete, output...)
-		for i := 0; i < len(output); i++ {
-			filesToDeleteObjsIndexes = append(filesToDeleteObjsIndexes, idx)
-		}
-	}
-
-	// Reverse the list so remove can succeed
-	reverseStringSlice(filesToDelete)
-
-	dErrs, err := storage.DeleteFileBulk(volume, filesToDelete)
-	if err != nil {
-		return nil, err
-	}
-
-	// Map files deletion errors to the correspondent objects
-	for i := range dErrs {
-		if dErrs[i] != nil {
-			if errs[filesToDeleteObjsIndexes[i]] != nil {
-				errs[filesToDeleteObjsIndexes[i]] = dErrs[i]
-			}
-		}
-	}
-
-	return errs, nil
-}
-
 // Removes notification.xml for a given bucket, only used during DeleteBucket.
 func removeNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
 	// Verify bucket is valid.
@@ -313,12 +241,67 @@ func listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter
 	return result, nil
 }
 
+// Walk a bucket, optionally prefix recursively, until we have returned
+// all the content to objectInfo channel, it is callers responsibility
+// to allocate a receive channel for ObjectInfo, upon any unhandled
+// error walker returns error. Optionally if context.Done() is received
+// then Walk() stops the walker.
+func fsWalk(ctx context.Context, obj ObjectLayer, bucket, prefix string, listDir ListDirFunc, results chan<- ObjectInfo, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) error {
+	if err := checkListObjsArgs(ctx, bucket, prefix, "", obj); err != nil {
+		// Upon error close the channel.
+		close(results)
+		return err
+	}
+
+	walkResultCh := startTreeWalk(ctx, bucket, prefix, "", true, listDir, ctx.Done())
+
+	go func() {
+		defer close(results)
+
+		for {
+			walkResult, ok := <-walkResultCh
+			if !ok {
+				break
+			}
+
+			var objInfo ObjectInfo
+			var err error
+			if HasSuffix(walkResult.entry, SlashSeparator) {
+				for _, getObjectInfoDir := range getObjectInfoDirs {
+					objInfo, err = getObjectInfoDir(ctx, bucket, walkResult.entry)
+					if err == nil {
+						break
+					}
+					if err == errFileNotFound {
+						err = nil
+						objInfo = ObjectInfo{
+							Bucket: bucket,
+							Name:   walkResult.entry,
+							IsDir:  true,
+						}
+					}
+				}
+			} else {
+				objInfo, err = getObjInfo(ctx, bucket, walkResult.entry)
+			}
+			if err != nil {
+				continue
+			}
+			results <- objInfo
+			if walkResult.end {
+				break
+			}
+		}
+	}()
+	return nil
+}
+
 func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
 	if delimiter != SlashSeparator && delimiter != "" {
 		return listObjectsNonSlash(ctx, bucket, prefix, marker, delimiter, maxKeys, tpool, listDir, getObjInfo, getObjectInfoDirs...)
 	}
 
-	if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, obj); err != nil {
+	if err := checkListObjsArgs(ctx, bucket, prefix, marker, obj); err != nil {
 		return loi, err
 	}
 

cmd/object-api-datatypes.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"io"
+	"math"
 	"time"
 
 	"github.com/minio/minio/pkg/hash"
@@ -77,6 +78,11 @@ type objectHistogramInterval struct {
 	start, end int64
 }
 
+const (
+	// dataUsageBucketLen must be length of ObjectsHistogramIntervals
+	dataUsageBucketLen = 7
+)
+
 // ObjectsHistogramIntervals is the list of all intervals
 // of object sizes to be included in objects histogram.
 var ObjectsHistogramIntervals = []objectHistogramInterval{
@@ -86,20 +92,25 @@ var ObjectsHistogramIntervals = []objectHistogramInterval{
 	{"BETWEEN_10_MB_AND_64_MB", 1024 * 1024 * 10, 1024*1024*64 - 1},
 	{"BETWEEN_64_MB_AND_128_MB", 1024 * 1024 * 64, 1024*1024*128 - 1},
 	{"BETWEEN_128_MB_AND_512_MB", 1024 * 1024 * 128, 1024*1024*512 - 1},
-	{"GREATER_THAN_512_MB", 1024 * 1024 * 512, -1},
+	{"GREATER_THAN_512_MB", 1024 * 1024 * 512, math.MaxInt64},
 }
 
 // DataUsageInfo represents data usage stats of the underlying Object API
 type DataUsageInfo struct {
-	// The timestamp of when the data usage info is generated
+	// LastUpdate is the timestamp of when the data usage info was last updated.
+	// This does not indicate a full scan.
 	LastUpdate time.Time `json:"lastUpdate"`
 
 	ObjectsCount uint64 `json:"objectsCount"`
 	// Objects total size
-	ObjectsTotalSize      uint64            `json:"objectsTotalSize"`
+	ObjectsTotalSize uint64 `json:"objectsTotalSize"`
+
+	// ObjectsSizesHistogram contains information on objects across all buckets.
+	// See ObjectsHistogramIntervals.
 	ObjectsSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
 
-	BucketsCount uint64            `json:"bucketsCount"`
+	BucketsCount uint64 `json:"bucketsCount"`
+	// BucketsSizes is "bucket name" -> size.
 	BucketsSizes map[string]uint64 `json:"bucketsSizes"`
 }
 

cmd/object-api-errors.go

@@ -274,7 +274,7 @@ func (e BucketSSEConfigNotFound) Error() string {
 // BucketNameInvalid - bucketname provided is invalid.
 type BucketNameInvalid GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e BucketNameInvalid) Error() string {
 	return "Bucket name invalid: " + e.Bucket
 }
@@ -290,17 +290,17 @@ type ObjectNameTooLong GenericError
 // ObjectNamePrefixAsSlash - object name has a slash as prefix.
 type ObjectNamePrefixAsSlash GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e ObjectNameInvalid) Error() string {
 	return "Object name invalid: " + e.Bucket + "#" + e.Object
 }
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e ObjectNameTooLong) Error() string {
 	return "Object name too long: " + e.Bucket + "#" + e.Object
 }
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e ObjectNamePrefixAsSlash) Error() string {
 	return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
 }
@@ -308,7 +308,7 @@ func (e ObjectNamePrefixAsSlash) Error() string {
 // AllAccessDisabled All access to this object has been disabled
 type AllAccessDisabled GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e AllAccessDisabled) Error() string {
 	return "All access to this object has been disabled"
 }
@@ -316,7 +316,7 @@ func (e AllAccessDisabled) Error() string {
 // IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header.
 type IncompleteBody GenericError
 
-// Return string an error formatted as the given text.
+// Error returns string an error formatted as the given text.
 func (e IncompleteBody) Error() string {
 	return e.Bucket + "#" + e.Object + "has incomplete body"
 }
@@ -348,11 +348,10 @@ func (e ObjectTooSmall) Error() string {
 
 // OperationTimedOut - a timeout occurred.
 type OperationTimedOut struct {
-	Path string
 }
 
 func (e OperationTimedOut) Error() string {
-	return "Operation timed out: " + e.Path
+	return "Operation timed out"
 }
 
 /// Multipart related errors.

cmd/object-api-input-checks.go

@@ -54,7 +54,7 @@ func checkBucketAndObjectNames(ctx context.Context, bucket, object string) error
 }
 
 // Checks for all ListObjects arguments validity.
-func checkListObjsArgs(ctx context.Context, bucket, prefix, marker, delimiter string, obj ObjectLayer) error {
+func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj ObjectLayer) error {
 	// Verify if bucket exists before validating object name.
 	// This is done on purpose since the order of errors is
 	// important here bucket does not exist error should
@@ -90,7 +90,7 @@ func checkListObjsArgs(ctx context.Context, bucket, prefix, marker, delimiter st
 
 // Checks for all ListMultipartUploads arguments validity.
 func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, obj ObjectLayer) error {
-	if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker, delimiter, obj); err != nil {
+	if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker, obj); err != nil {
 		return err
 	}
 	if uploadIDMarker != "" {

cmd/object-api-interface.go

@@ -55,11 +55,11 @@ const (
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
 	// Locking operations on object.
-	NewNSLock(ctx context.Context, bucket string, object string) RWLocker
+	NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker
 
 	// Storage operations.
 	Shutdown(context.Context) error
-	CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo
+	CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error
 	StorageInfo(ctx context.Context, local bool) StorageInfo // local queries only local disks
 
 	// Bucket operations.
@@ -69,6 +69,7 @@ type ObjectLayer interface {
 	DeleteBucket(ctx context.Context, bucket string) error
 	ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
+	Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error
 
 	// Object operations.
 
@@ -100,8 +101,8 @@ type ObjectLayer interface {
 	ReloadFormat(ctx context.Context, dryRun bool) error
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)
 	HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error)
-	HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (madmin.HealResultItem, error)
-	HealObjects(ctx context.Context, bucket, prefix string, fn healObjectFn) error
+	HealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error)
+	HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn healObjectFn) error
 
 	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
 

cmd/object-api-listobjects_test.go

@@ -45,6 +45,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		// Will not store any objects in this bucket,
 		// Its to test ListObjects on an empty bucket.
 		"empty-bucket",
+		// Listing the case where the marker > last object.
+		"test-bucket-single-object",
 	}
 	for _, bucket := range testBuckets {
 		err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
@@ -72,6 +74,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		{testBuckets[1], "obj1", "obj1", nil},
 		{testBuckets[1], "obj2", "obj2", nil},
 		{testBuckets[1], "temporary/0/", "", nil},
+		{testBuckets[3], "A/B", "contentstring", nil},
 	}
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
@@ -445,6 +448,11 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 				{Name: "temporary/0/"},
 			},
 		},
+		// ListObjectsResult-34 Listing with marker > last object should return empty
+		{
+			IsTruncated: false,
+			Objects:     []ObjectInfo{},
+		},
 	}
 
 	testCases := []struct {
@@ -559,6 +567,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		{"test-bucket-empty-dir", "", "", SlashSeparator, 10, resultCases[32], nil, true},
 		// Test listing a directory which contains an empty directory (64)
 		{"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true},
+		// Test listing with marker > last object such that response should be empty (65)
+		{"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true},
 	}
 
 	for i, testCase := range testCases {

cmd/object-api-utils.go

@@ -51,10 +51,6 @@ import (
 const (
 	// MinIO meta bucket.
 	minioMetaBucket = ".minio.sys"
-	// Background ops meta prefix
-	backgroundOpsMetaPrefix = "background-ops"
-	// MinIO Stats meta prefix.
-	minioMetaBackgroundOpsBucket = minioMetaBucket + SlashSeparator + backgroundOpsMetaPrefix
 	// Multipart meta prefix.
 	mpartMetaPrefix = "multipart"
 	// MinIO Multipart meta prefix.
@@ -77,7 +73,7 @@ func isMinioMetaBucketName(bucket string) bool {
 	return bucket == minioMetaBucket ||
 		bucket == minioMetaMultipartBucket ||
 		bucket == minioMetaTmpBucket ||
-		bucket == minioMetaBackgroundOpsBucket
+		bucket == dataUsageBucket
 }
 
 // IsValidBucketName verifies that a bucket name is in accordance with
@@ -198,6 +194,16 @@ func retainSlash(s string) string {
 	return strings.TrimSuffix(s, SlashSeparator) + SlashSeparator
 }
 
+// pathsJoinPrefix - like pathJoin retains trailing SlashSeparator
+// for all elements, prepends them with 'prefix' respectively.
+func pathsJoinPrefix(prefix string, elem ...string) (paths []string) {
+	paths = make([]string, len(elem))
+	for i, e := range elem {
+		paths[i] = pathJoin(prefix, e)
+	}
+	return paths
+}
+
 // pathJoin - like path.Join() but retains trailing SlashSeparator of the last element
 func pathJoin(elem ...string) string {
 	trailingSlash := ""

cmd/object-handlers.go

@@ -24,7 +24,6 @@ import (
 	"encoding/hex"
 	"encoding/xml"
 	"io"
-	goioutil "io/ioutil"
 	"net/http"
 	"net/url"
 	"sort"
@@ -1215,7 +1214,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if tags := r.Header.Get(http.CanonicalHeaderKey(xhttp.AmzObjectTagging)); tags != "" {
+	if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
 		metadata[xhttp.AmzObjectTagging], err = extractTags(ctx, tags)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -1897,6 +1896,9 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 
+	// To detect if the client has disconnected.
+	r.Body = &detectDisconnect{r.Body, r.Context().Done()}
+
 	// X-Amz-Copy-Source shouldn't be set for this call.
 	if _, ok := r.Header[xhttp.AmzCopySource]; ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
@@ -2331,12 +2333,19 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	}
 
+	// Content-Length is required and should be non-zero
+	if r.ContentLength <= 0 {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	// Reject retention or governance headers if set, CompleteMultipartUpload spec
 	// does not use these headers, and should not be passed down to checkPutObjectLockAllowed
 	if objectlock.IsObjectLockRequested(r.Header) || objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	// Enforce object lock governance in case a competing upload finalized first.
 	retPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
@@ -2353,14 +2362,9 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	}
 
-	completeMultipartBytes, err := goioutil.ReadAll(r.Body)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
 	complMultipartUpload := &CompleteMultipartUpload{}
-	if err = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+	if err = xmlDecoder(r.Body, complMultipartUpload, r.ContentLength); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	if len(complMultipartUpload.Parts) == 0 {
@@ -2595,7 +2599,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 	}
 
 	// Check permissions to perform this legal hold operation
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction); s3Err != ErrNone {
+	if s3Err := checkRequestAuthType(ctx, r, policy.PutObjectLegalHoldAction, bucket, object); s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -2604,11 +2608,8 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
-	// Get Content-Md5 sent by client and verify if valid
-	md5Bytes, err := checkValidMD5(r.Header)
-	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
+	if !hasContentMD5(r.Header) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -2623,18 +2624,6 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// verify Content-MD5 sum of request body if this header set
-	if len(md5Bytes) > 0 {
-		data, err := xml.Marshal(legalHold)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		if hex.EncodeToString(md5Bytes) != getMD5Hash(data) {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfo = api.CacheAPI().GetObjectInfo
@@ -2765,7 +2754,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 	// Check permissions to perform this governance operation
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction); s3Err != ErrNone {
+	if s3Err := checkRequestAuthType(ctx, r, policy.PutObjectRetentionAction, bucket, object); s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -2774,11 +2763,8 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
-	// Get Content-Md5 sent by client and verify if valid
-	md5Bytes, err := checkValidMD5(r.Header)
-	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
+	if !hasContentMD5(r.Header) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -2805,18 +2791,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	// verify Content-MD5 sum of request body if this header set
-	if len(md5Bytes) > 0 {
-		data, err := xml.Marshal(objRetention)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		if hex.EncodeToString(md5Bytes) != getMD5Hash(data) {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+
 	objInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objRetention.Mode)
 	objInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = objRetention.RetainUntilDate.UTC().Format(time.RFC3339)
 	objInfo.metadataOnly = true
@@ -3006,5 +2981,5 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessNoContent(w)
 }

cmd/object-lock.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"net/http"
+	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
@@ -244,3 +246,44 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 	}
 	return mode, retainDate, legalHold, ErrNone
 }
+
+func initBucketObjectLockConfig(buckets []BucketInfo, objAPI ObjectLayer) error {
+	for _, bucket := range buckets {
+		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
+		configFile := path.Join(bucketConfigPrefix, bucket.Name, bucketObjectLockEnabledConfigFile)
+		bucketObjLockData, err := readConfig(ctx, objAPI, configFile)
+		if err != nil {
+			if err == errConfigNotFound {
+				continue
+			}
+			return err
+		}
+
+		if string(bucketObjLockData) != bucketObjectLockEnabledConfig {
+			// this should never happen
+			logger.LogIf(ctx, objectlock.ErrMalformedBucketObjectConfig)
+			continue
+		}
+
+		configFile = path.Join(bucketConfigPrefix, bucket.Name, objectLockConfig)
+		configData, err := readConfig(ctx, objAPI, configFile)
+		if err != nil {
+			if err == errConfigNotFound {
+				globalBucketObjectLockConfig.Set(bucket.Name, objectlock.Retention{})
+				continue
+			}
+			return err
+		}
+
+		config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
+		if err != nil {
+			return err
+		}
+		retention := objectlock.Retention{}
+		if config.Rule != nil {
+			retention = config.ToRetention()
+		}
+		globalBucketObjectLockConfig.Set(bucket.Name, retention)
+	}
+	return nil
+}

cmd/peer-rest-server.go

@@ -325,7 +325,12 @@ func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if err = globalIAMSys.LoadUser(objAPI, accessKey, temp); err != nil {
+	var userType = regularUser
+	if temp {
+		userType = stsUser
+	}
+
+	if err = globalIAMSys.LoadUser(objAPI, accessKey, userType); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}

cmd/posix-diskid-check.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"io"
 )
 
@@ -38,8 +39,8 @@ func (p *posixDiskIDCheck) IsOnline() bool {
 	return storedDiskID == p.diskID
 }
 
-func (p *posixDiskIDCheck) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
-	return p.storage.CrawlAndGetDataUsage(endCh)
+func (p *posixDiskIDCheck) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
+	return p.storage.CrawlAndGetDataUsage(ctx, cache)
 }
 
 func (p *posixDiskIDCheck) Hostname() string {
@@ -109,7 +110,7 @@ func (p *posixDiskIDCheck) DeleteVol(volume string) (err error) {
 	return p.storage.DeleteVol(volume)
 }
 
-func (p *posixDiskIDCheck) Walk(volume, dirPath string, marker string, recursive bool, leafFile string, readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (chan FileInfo, error) {
+func (p *posixDiskIDCheck) Walk(volume, dirPath string, marker string, recursive bool, leafFile string, readMetadataFn readMetadataFunc, endWalkCh <-chan struct{}) (chan FileInfo, error) {
 	if p.isDiskStale() {
 		return nil, errDiskNotFound
 	}
@@ -179,6 +180,13 @@ func (p *posixDiskIDCheck) DeleteFileBulk(volume string, paths []string) (errs [
 	return p.storage.DeleteFileBulk(volume, paths)
 }
 
+func (p *posixDiskIDCheck) DeletePrefixes(volume string, paths []string) (errs []error, err error) {
+	if p.isDiskStale() {
+		return nil, errDiskNotFound
+	}
+	return p.storage.DeletePrefixes(volume, paths)
+}
+
 func (p *posixDiskIDCheck) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	if p.isDiskStale() {
 		return errDiskNotFound

cmd/posix.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -44,7 +44,6 @@ import (
 	"github.com/minio/minio/pkg/disk"
 	xioutil "github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/mountinfo"
-	"github.com/ncw/directio"
 )
 
 const (
@@ -213,7 +212,7 @@ func newPosix(path string) (*posix, error) {
 		diskPath: path,
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := directio.AlignedBlock(readBlockSize)
+				b := disk.AlignedBlock(readBlockSize)
 				return &b
 			},
 		},
@@ -339,8 +338,8 @@ func (s *posix) waitForLowActiveIO() {
 	}
 }
 
-func (s *posix) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
-	dataUsageInfo := updateUsage(s.diskPath, endCh, s.waitForLowActiveIO, func(item Item) (int64, error) {
+func (s *posix) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCache) (dataUsageCache, error) {
+	dataUsageInfo, err := updateUsage(ctx, s.diskPath, cache, s.waitForLowActiveIO, func(item Item) (int64, error) {
 		// Look for `xl.json' at the leaf.
 		if !strings.HasSuffix(item.Path, SlashSeparator+xlMetaJSONFile) {
 			// if no xl.json found, skip the file.
@@ -354,14 +353,20 @@ func (s *posix) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, erro
 
 		meta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaBuf)
 		if err != nil {
-			return 0, errSkipFile
+			return 0, nil
 		}
 
 		return meta.Stat.Size, nil
 	})
-
-	dataUsageInfo.LastUpdate = UTCNow()
-	atomic.StoreUint64(&s.totalUsed, dataUsageInfo.ObjectsTotalSize)
+	if err != nil {
+		return dataUsageInfo, err
+	}
+	dataUsageInfo.Info.LastUpdate = time.Now()
+	total := dataUsageInfo.sizeRecursive(dataUsageInfo.Info.Name)
+	if total == nil {
+		total = &dataUsageEntry{}
+	}
+	atomic.StoreUint64(&s.totalUsed, uint64(total.Size))
 	return dataUsageInfo, nil
 }
 
@@ -643,7 +648,7 @@ func (s *posix) DeleteVol(volume string) (err error) {
 // Walk - is a sorted walker which returns file entries in lexically
 // sorted order, additionally along with metadata about each of those entries.
 func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile string,
-	readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (ch chan FileInfo, err error) {
+	readMetadataFn readMetadataFunc, endWalkCh <-chan struct{}) (ch chan FileInfo, err error) {
 
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
@@ -670,13 +675,16 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
 	ch = make(chan FileInfo)
 	go func() {
 		defer close(ch)
-		listDir := func(volume, dirPath, dirEntry string) (entries []string) {
+		listDir := func(volume, dirPath, dirEntry string) (emptyDir bool, entries []string) {
 			entries, err := s.ListDir(volume, dirPath, -1, leafFile)
 			if err != nil {
 				return
 			}
+			if len(entries) == 0 {
+				return true, nil
+			}
 			sort.Strings(entries)
-			return filterMatchingPrefix(entries, dirEntry)
+			return false, filterMatchingPrefix(entries, dirEntry)
 		}
 
 		walkResultCh := startTreeWalk(context.Background(), volume, dirPath, marker, recursive, listDir, endWalkCh)
@@ -1265,16 +1273,28 @@ func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 	}, nil
 }
 
-// deleteFile deletes a file path if its empty. If it's successfully deleted,
-// it will recursively move up the tree, deleting empty parent directories
-// until it finds one with files in it. Returns nil for a non-empty directory.
-func deleteFile(basePath, deletePath string) error {
-	if basePath == deletePath {
+// deleteFile deletes a file or a directory if its empty unless recursive
+// is set to true. If the target is successfully deleted, it will recursively
+// move up the tree, deleting empty parent directories until it finds one
+// with files in it. Returns nil for a non-empty directory even when
+// recursive is set to false.
+func deleteFile(basePath, deletePath string, recursive bool) error {
+	if basePath == "" || deletePath == "" {
+		return nil
+	}
+	basePath = filepath.Clean(basePath)
+	deletePath = filepath.Clean(deletePath)
+	if !strings.HasPrefix(deletePath, basePath) || deletePath == basePath {
 		return nil
 	}
 
-	// Attempt to remove path.
-	if err := os.Remove((deletePath)); err != nil {
+	var err error
+	if recursive {
+		err = os.RemoveAll(deletePath)
+	} else {
+		err = os.Remove(deletePath)
+	}
+	if err != nil {
 		switch {
 		case isSysErrNotEmpty(err):
 			// Ignore errors if the directory is not empty. The server relies on
@@ -1297,12 +1317,58 @@ func deleteFile(basePath, deletePath string) error {
 	deletePath = strings.TrimSuffix(deletePath, SlashSeparator)
 	deletePath = slashpath.Dir(deletePath)
 
-	// Delete parent directory. Errors for parent directories shouldn't trickle down.
-	deleteFile(basePath, deletePath)
+	// Delete parent directory obviously not recursively. Errors for
+	// parent directories shouldn't trickle down.
+	deleteFile(basePath, deletePath, false)
 
 	return nil
 }
 
+// DeletePrefixes forcibly deletes all the contents of a set of specified paths.
+// Parent directories are automatically removed if they become empty. err can
+// bil nil while errs can contain some errors for corresponding objects. No error
+// is set if a specified prefix path does not exist.
+func (s *posix) DeletePrefixes(volume string, paths []string) (errs []error, err error) {
+	atomic.AddInt32(&s.activeIOCount, 1)
+	defer func() {
+		atomic.AddInt32(&s.activeIOCount, -1)
+	}()
+
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return nil, err
+	}
+
+	// Stat a volume entry.
+	_, err = os.Stat(volumeDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, errVolumeNotFound
+		} else if os.IsPermission(err) {
+			return nil, errVolumeAccessDenied
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		}
+		return nil, err
+	}
+
+	errs = make([]error, len(paths))
+	// Following code is needed so that we retain SlashSeparator
+	// suffix if any in path argument.
+	for idx, path := range paths {
+		filePath := pathJoin(volumeDir, path)
+		errs[idx] = checkPathLength(filePath)
+		if errs[idx] != nil {
+			continue
+		}
+		// Delete file or a directory recursively, delete parent
+		// directory as well if its empty.
+		errs[idx] = deleteFile(volumeDir, filePath, true)
+	}
+
+	return
+}
+
 // DeleteFile - delete a file at path.
 func (s *posix) DeleteFile(volume, path string) (err error) {
 	atomic.AddInt32(&s.activeIOCount, 1)
@@ -1316,7 +1382,7 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 	}
 
 	// Stat a volume entry.
-	_, err = os.Stat((volumeDir))
+	_, err = os.Stat(volumeDir)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
@@ -1331,18 +1397,49 @@ func (s *posix) DeleteFile(volume, path string) (err error) {
 	// Following code is needed so that we retain SlashSeparator suffix if any in
 	// path argument.
 	filePath := pathJoin(volumeDir, path)
-	if err = checkPathLength((filePath)); err != nil {
+	if err = checkPathLength(filePath); err != nil {
 		return err
 	}
 
 	// Delete file and delete parent directory as well if its empty.
-	return deleteFile(volumeDir, filePath)
+	return deleteFile(volumeDir, filePath, false)
 }
 
 func (s *posix) DeleteFileBulk(volume string, paths []string) (errs []error, err error) {
+	atomic.AddInt32(&s.activeIOCount, 1)
+	defer func() {
+		atomic.AddInt32(&s.activeIOCount, -1)
+	}()
+
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return nil, err
+	}
+
+	// Stat a volume entry.
+	_, err = os.Stat(volumeDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, errVolumeNotFound
+		} else if os.IsPermission(err) {
+			return nil, errVolumeAccessDenied
+		} else if isSysErrIO(err) {
+			return nil, errFaultyDisk
+		}
+		return nil, err
+	}
+
 	errs = make([]error, len(paths))
+	// Following code is needed so that we retain SlashSeparator
+	// suffix if any in path argument.
 	for idx, path := range paths {
-		errs[idx] = s.DeleteFile(volume, path)
+		filePath := pathJoin(volumeDir, path)
+		errs[idx] = checkPathLength(filePath)
+		if errs[idx] != nil {
+			continue
+		}
+		// Delete file and delete parent directory as well if its empty.
+		errs[idx] = deleteFile(volumeDir, filePath, false)
 	}
 	return
 }
@@ -1429,7 +1526,7 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 
 	// Remove parent dir of the source file if empty
 	if parentDir := slashpath.Dir(srcFilePath); isDirEmpty(parentDir) {
-		deleteFile(srcVolumeDir, parentDir)
+		deleteFile(srcVolumeDir, parentDir, false)
 	}
 
 	return nil

cmd/prepare-storage.go

@@ -244,6 +244,10 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		if _, ok := formatCriticalErrors[sErr]; ok {
 			return nil, fmt.Errorf("Disk %s: %w", endpoints[i], sErr)
 		}
+		// not critical error but still print the error, nonetheless, which is perhaps unhandled
+		if sErr != errUnformattedDisk && sErr != errDiskNotFound && retryCount >= 5 {
+			logger.Info("Unable to read 'format.json' from %s: %v\n", endpoints[i], sErr)
+		}
 	}
 
 	// Pre-emptively check if one of the formatted disks
@@ -251,7 +255,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 	// most part unless one of the formats is not consistent
 	// with expected XL format. For example if a user is
 	// trying to pool FS backend into an XL set.
-	if err := checkFormatXLValues(formatConfigs); err != nil {
+	if err := checkFormatXLValues(formatConfigs, drivesPerSet); err != nil {
 		return nil, err
 	}
 

cmd/server-main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -263,7 +264,12 @@ func initAllSubsystems(buckets []BucketInfo, newObject ObjectLayer) (err error)
 
 	// Initialize policy system.
 	if err = globalPolicySys.Init(buckets, newObject); err != nil {
-		return fmt.Errorf("Unable to initialize policy system; %w", err)
+		return fmt.Errorf("Unable to initialize policy system: %w", err)
+	}
+
+	// Initialize bucket object lock.
+	if err = initBucketObjectLockConfig(buckets, newObject); err != nil {
+		return fmt.Errorf("Unable to initialize object lock system: %w", err)
 	}
 
 	// Initialize lifecycle system.
@@ -275,6 +281,7 @@ func initAllSubsystems(buckets []BucketInfo, newObject ObjectLayer) (err error)
 	if err = globalBucketSSEConfigSys.Init(buckets, newObject); err != nil {
 		return fmt.Errorf("Unable to initialize bucket encryption subsystem: %w", err)
 	}
+
 	return nil
 }
 
@@ -355,6 +362,9 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	httpServer := xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{handler}, getCert)
+	httpServer.BaseContext = func(listener net.Listener) context.Context {
+		return GlobalContext
+	}
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start()
 	}()
@@ -435,9 +445,6 @@ func serverMain(ctx *cli.Context) {
 		logger.StartupMessage(color.RedBold(msg))
 	}
 
-	// Set uptime time after object layer has initialized.
-	globalBootTime = UTCNow()
-
 	handleSignals()
 }