fix ObjectInfo returned by CopyObject (#11377)

erasure CopyObject was returning old metadata

Dockerfile.release

@@ -5,8 +5,8 @@ ARG TARGETARCH
 LABEL name="MinIO" \
       vendor="MinIO Inc <dev@min.io>" \
       maintainer="MinIO Inc <dev@min.io>" \
-      version="RELEASE.2021-01-08T21-18-21Z" \
-      release="RELEASE.2021-01-08T21-18-21Z" \
+      version="RELEASE.2021-01-16T02-19-44Z" \
+      release="RELEASE.2021-01-16T02-19-44Z" \
       summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
       description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
 

Makefile

@@ -38,7 +38,7 @@ fmt:
 lint:
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
-	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=10m --config ./.golangci.yml
 
 ruleguard:
 	@echo "Running $@ check"
@@ -71,10 +71,15 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
-hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
+hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))
+	TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)"
 hotfix: install
 
+docker-hotfix: hotfix checks
+	@echo "Building minio docker image '$(TAG)'"
+	@docker build -t $(TAG) . -f Dockerfile.dev
+
 docker: checks
 	@echo "Building minio docker image '$(TAG)'"
 	@GOOS=linux GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null

cmd/acl-handlers.go

@@ -61,7 +61,7 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketACL")
 
-	defer logger.AuditLog(w, r, "PutBucketACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -125,7 +125,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
-	defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -176,7 +176,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectACL")
 
-	defer logger.AuditLog(w, r, "PutObjectACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -240,7 +240,7 @@ func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
-	defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/admin-bucket-handlers.go

@@ -41,7 +41,7 @@ const (
 func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketQuotaConfig")
 
-	defer logger.AuditLog(w, r, "PutBucketQuotaConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -81,7 +81,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketQuotaConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketQuotaConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -116,7 +116,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetBucketTarget")
 
-	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	update := r.URL.Query().Get("update") == "true"
@@ -156,7 +156,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
-
 	sameTarget, _ := isLocalHost(target.URL().Hostname(), target.URL().Port(), globalMinioPort)
 	if sameTarget && bucket == target.TargetBucket {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
@@ -205,7 +204,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBucketTargets")
 
-	defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arnType := vars["type"]
@@ -244,7 +243,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveBucketTarget")
 
-	defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	arn := vars["arn"]

cmd/admin-handlers-config-kv.go

@@ -62,7 +62,7 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteConfigKV")
 
-	defer logger.AuditLog(w, r, "DeleteConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -104,7 +104,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigKV")
 
-	defer logger.AuditLog(w, r, "SetConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -130,13 +130,14 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
 	dynamic, err := cfg.ReadConfig(bytes.NewReader(kvBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -158,15 +159,14 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
-	// Apply dynamic values.
-	if err := applyDynamicConfig(GlobalContext, cfg); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	globalNotificationSys.SignalService(serviceReloadDynamic)
-
-	// If all values were dynamic, tell the client.
 	if dynamic {
+		// Apply dynamic values.
+		if err := applyDynamicConfig(GlobalContext, objectAPI, cfg); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		globalNotificationSys.SignalService(serviceReloadDynamic)
+		// If all values were dynamic, tell the client.
 		w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
 	}
 	writeSuccessResponseHeadersOnly(w)
@@ -176,7 +176,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
 
-	defer logger.AuditLog(w, r, "GetConfigKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -214,7 +214,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClearConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "ClearConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -251,7 +251,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RestoreConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "RestoreConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -282,7 +282,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -299,7 +299,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListConfigHistoryKV")
 
-	defer logger.AuditLog(w, r, "ListConfigHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -339,7 +339,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HelpConfigKV")
 
-	defer logger.AuditLog(w, r, "HelpHistoryKV", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -367,7 +367,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfig")
 
-	defer logger.AuditLog(w, r, "SetConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -394,7 +394,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = validateConfig(cfg, objectAPI.SetDriveCount()); err != nil {
+	if err = validateConfig(cfg, objectAPI.SetDriveCounts()); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -424,7 +424,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfig")
 
-	defer logger.AuditLog(w, r, "GetConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {

cmd/admin-handlers-users.go

@@ -56,7 +56,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
-	defer logger.AuditLog(w, r, "RemoveUser", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
@@ -94,7 +94,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
 
-	defer logger.AuditLog(w, r, "ListUsers", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
@@ -128,7 +128,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetUserInfo")
 
-	defer logger.AuditLog(w, r, "GetUserInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	name := vars["accessKey"]
@@ -184,7 +184,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "UpdateGroupMembers")
 
-	defer logger.AuditLog(w, r, "UpdateGroupMembers", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
@@ -229,7 +229,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetGroup")
 
-	defer logger.AuditLog(w, r, "GetGroup", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetGroupAdminAction)
 	if objectAPI == nil {
@@ -258,7 +258,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListGroups")
 
-	defer logger.AuditLog(w, r, "ListGroups", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
 	if objectAPI == nil {
@@ -284,7 +284,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetGroupStatus")
 
-	defer logger.AuditLog(w, r, "SetGroupStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
 	if objectAPI == nil {
@@ -321,7 +321,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
-	defer logger.AuditLog(w, r, "SetUserStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
@@ -356,7 +356,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddUser")
 
-	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	accessKey := path.Clean(vars["accessKey"])
@@ -426,7 +426,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
+	if err = globalIAMSys.CreateUser(accessKey, uinfo); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -444,7 +444,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddServiceAccount")
 
-	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -523,7 +523,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListServiceAccounts")
 
-	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -578,7 +578,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteServiceAccount")
 
-	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -637,7 +637,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AccountInfo")
 
-	defer logger.AuditLog(w, r, "AccountInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -751,7 +751,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicyV2")
 
-	defer logger.AuditLog(w, r, "InfoCannedPolicyV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
@@ -778,7 +778,7 @@ func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
-	defer logger.AuditLog(w, r, "InfoCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
@@ -802,7 +802,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPoliciesV2")
 
-	defer logger.AuditLog(w, r, "ListCannedPoliciesV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -836,7 +836,7 @@ func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
 
-	defer logger.AuditLog(w, r, "ListCannedPolicies", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -870,7 +870,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveCannedPolicy")
 
-	defer logger.AuditLog(w, r, "RemoveCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
 	if objectAPI == nil {
@@ -898,7 +898,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddCannedPolicy")
 
-	defer logger.AuditLog(w, r, "AddCannedPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
 	if objectAPI == nil {
@@ -950,7 +950,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
-	defer logger.AuditLog(w, r, "SetPolicyForUserOrGroup", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {

cmd/admin-handlers.go

@@ -42,6 +42,7 @@ import (
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bandwidth"
+	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -78,7 +79,7 @@ func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, mode string) (
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerUpdate")
 
-	defer logger.AuditLog(w, r, "ServerUpdate", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
 	if objectAPI == nil {
@@ -187,7 +188,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Service")
 
-	defer logger.AuditLog(w, r, "Service", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	action := vars["action"]
@@ -284,7 +285,7 @@ type ServerInfo struct {
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
 
-	defer logger.AuditLog(w, r, "StorageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
@@ -327,7 +328,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
 
-	defer logger.AuditLog(w, r, "DataUsageInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
@@ -403,11 +404,50 @@ type PeerLocks struct {
 	Locks map[string][]lockRequesterInfo
 }
 
+// ForceUnlockHandler force unlocks requested resource
+func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ForceUnlock")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+
+	var args dsync.LockArgs
+	lockersMap := make(map[string]dsync.NetLocker)
+	for _, path := range strings.Split(vars["paths"], ",") {
+		if path == "" {
+			continue
+		}
+		args.Resources = append(args.Resources, path)
+		lockers, _ := z.serverPools[0].getHashedSet(path).getLockers()
+		for _, locker := range lockers {
+			if locker != nil {
+				lockersMap[locker.String()] = locker
+			}
+		}
+	}
+
+	for _, locker := range lockersMap {
+		locker.ForceUnlock(ctx, args)
+	}
+}
+
 // TopLocksHandler Get list of locks in use
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
-	defer logger.AuditLog(w, r, "TopLocks", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
@@ -459,7 +499,7 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
-	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
@@ -564,7 +604,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
-	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
@@ -665,7 +705,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Heal")
 
-	defer logger.AuditLog(w, r, "Heal", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -876,7 +916,7 @@ func getAggregatedBackgroundHealState(ctx context.Context) (madmin.BgHealState,
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
-	defer logger.AuditLog(w, r, "HealBackgroundStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -1097,7 +1137,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
-	defer logger.AuditLog(w, r, "ConsoleLog", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
@@ -1168,7 +1208,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 // KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSCreateKey")
-	defer logger.AuditLog(w, r, "KMSCreateKey", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
 	if objectAPI == nil {
@@ -1191,7 +1231,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatus")
 
-	defer logger.AuditLog(w, r, "KMSKeyStatus", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
@@ -1264,7 +1304,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealthInfo")
 
-	defer logger.AuditLog(w, r, "HealthInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1442,7 +1482,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "BandwidthMonitor")
 
-	defer logger.AuditLog(w, r, "BandwidthMonitor", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
@@ -1495,7 +1535,7 @@ func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
 
-	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
@@ -1561,9 +1601,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 				Type:             madmin.ErasureType,
 				OnlineDisks:      onlineDisks.Sum(),
 				OfflineDisks:     offlineDisks.Sum(),
-				StandardSCData:   backendInfo.StandardSCData,
 				StandardSCParity: backendInfo.StandardSCParity,
-				RRSCData:         backendInfo.RRSCData,
 				RRSCParity:       backendInfo.RRSCParity,
 			}
 		} else {

cmd/admin-handlers_test.go

@@ -66,7 +66,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 	// Initialize boot time
 	globalBootTime = UTCNow()
 
-	globalEndpoints = mustGetZoneEndpoints(erasureDirs...)
+	globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
 
 	newAllSubsystems()
 
@@ -97,16 +97,9 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	if err != nil {
 		return nil, nil, err
 	}
-	endpoints := mustGetNewEndpoints(erasureDirs...)
-	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "")
-	if err != nil {
-		removeRoots(erasureDirs)
-		return nil, nil, err
-	}
-
+	endpoints := mustGetPoolEndpoints(erasureDirs...)
 	globalPolicySys = NewPolicySys()
-	objLayer := &erasureServerPools{serverPools: make([]*erasureSets, 1)}
-	objLayer.serverPools[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
+	objLayer, err := newErasureServerPools(ctx, endpoints)
 	if err != nil {
 		return nil, nil, err
 	}

cmd/admin-heal-ops.go

@@ -750,16 +750,19 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			if !ok {
 				return nil
 			}
+
 			var itemType madmin.HealItemType
-			switch {
-			case source.bucket == nopHeal:
+			switch source.bucket {
+			case nopHeal:
 				continue
-			case source.bucket == SlashSeparator:
+			case SlashSeparator:
 				itemType = madmin.HealItemMetadata
-			case source.bucket != "" && source.object == "":
-				itemType = madmin.HealItemBucket
 			default:
-				itemType = madmin.HealItemObject
+				if source.object == "" {
+					itemType = madmin.HealItemBucket
+				} else {
+					itemType = madmin.HealItemObject
+				}
 			}
 
 			if err := h.queueHealTask(source, itemType); err != nil {

cmd/admin-router.go

@@ -189,10 +189,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 		}
 
-		// -- Top APIs --
-		// Top locks
 		if globalIsDistErasure {
+			// Top locks
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+			// Force unlocks paths
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock").
+				Queries("paths", "{paths:.*}").HandlerFunc(httpTraceHdrs(adminAPI.ForceUnlockHandler))
 		}
 
 		// HTTP Trace

cmd/admin-server-info.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -45,7 +46,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := isServerResolvable(endpoint); err == nil {
+				if err := isServerResolvable(endpoint, time.Second); err == nil {
 					network[nodeName] = "online"
 				} else {
 					network[nodeName] = "offline"
@@ -69,3 +70,35 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		Disks:    storageInfo.Disks,
 	}
 }
+
+func getLocalDisks(endpointServerPools EndpointServerPools) []madmin.Disk {
+	var localEndpoints Endpoints
+	network := make(map[string]string)
+
+	for _, ep := range endpointServerPools {
+		for _, endpoint := range ep.Endpoints {
+			nodeName := endpoint.Host
+			if nodeName == "" {
+				nodeName = "localhost"
+			}
+			if endpoint.IsLocal {
+				// Only proceed for local endpoints
+				network[nodeName] = "online"
+				localEndpoints = append(localEndpoints, endpoint)
+				continue
+			}
+			_, present := network[nodeName]
+			if !present {
+				if err := isServerResolvable(endpoint, time.Second); err == nil {
+					network[nodeName] = "online"
+				} else {
+					network[nodeName] = "offline"
+				}
+			}
+		}
+	}
+	localDisks, _ := initStorageDisksWithErrors(localEndpoints)
+	defer closeStorageDisks(localDisks)
+	storageInfo, _ := getStorageInfo(localDisks, localEndpoints.GetAllStrings())
+	return storageInfo.Disks
+}

cmd/api-errors.go

@@ -87,6 +87,7 @@ const (
 	ErrInvalidMaxUploads
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
+	ErrInvalidPartNumber
 	ErrInvalidRequestBody
 	ErrInvalidCopySource
 	ErrInvalidMetadataDirective
@@ -437,6 +438,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Argument partNumberMarker must be an integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidPartNumber: {
+		Code:           "InvalidPartNumber",
+		Description:    "The requested partnumber is not satisfiable",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
 	ErrInvalidPolicyDocument: {
 		Code:           "InvalidPolicyDocument",
 		Description:    "The content of the form does not meet the conditions specified in the policy document.",
@@ -2119,6 +2125,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 				HTTPStatusCode: e.Response().StatusCode,
 			}
 			// Add more Gateway SDKs here if any in future.
+		default:
+			apiErr = APIError{
+				Code:           apiErr.Code,
+				Description:    fmt.Sprintf("%s: cause(%v)", apiErr.Description, err),
+				HTTPStatusCode: apiErr.HTTPStatusCode,
+			}
 		}
 	}
 

cmd/api-headers.go

@@ -156,16 +156,16 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		return err
 	}
 
-	if opts.PartNumber > 0 {
-		rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
-	}
-
 	// For providing ranged content
 	start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
 	if err != nil {
 		return err
 	}
 
+	if rs == nil && opts.PartNumber > 0 {
+		rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
+	}
+
 	// Set content length.
 	w.Header().Set(xhttp.ContentLength, strconv.FormatInt(rangeLen, 10))
 	if rs != nil {

cmd/background-heal-ops.go

@@ -61,7 +61,6 @@ func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
 	}
 
 	// At max 10 attempts to wait with 100 millisecond interval before proceeding
-	waitCount := 10
 	waitTick := 100 * time.Millisecond
 
 	// Bucket notification and http trace are not costly, it is okay to ignore them
@@ -70,14 +69,21 @@ func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
 		return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalHTTPTrace.NumSubscribers())
 	}
 
+	tmpMaxWait := maxWait
 	if httpServer := newHTTPServerFn(); httpServer != nil {
 		// Any requests in progress, delay the heal.
 		for httpServer.GetRequestCount() >= maxIOFn() {
-			time.Sleep(waitTick)
-			waitCount--
-			if waitCount == 0 {
+			if tmpMaxWait > 0 {
+				if tmpMaxWait < waitTick {
+					time.Sleep(tmpMaxWait)
+				} else {
+					time.Sleep(waitTick)
+				}
+				tmpMaxWait = tmpMaxWait - waitTick
+			}
+			if tmpMaxWait <= 0 {
 				if intDataUpdateTracker.debug {
-					logger.Info("waitForLowHTTPReq: waited %d times, resuming", waitCount)
+					logger.Info("waitForLowHTTPReq: waited max %s, resuming", maxWait)
 				}
 				break
 			}
@@ -91,20 +97,22 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 		select {
 		case task, ok := <-h.tasks:
 			if !ok {
-				break
+				return
 			}
 
 			var res madmin.HealResultItem
 			var err error
-			switch {
-			case task.bucket == nopHeal:
+			switch task.bucket {
+			case nopHeal:
 				continue
-			case task.bucket == SlashSeparator:
+			case SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
-			case task.bucket != "" && task.object == "":
-				res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
-			case task.bucket != "" && task.object != "":
-				res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
+			default:
+				if task.object == "" {
+					res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
+				} else {
+					res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
+				}
 			}
 			task.responseCh <- healResult{result: res, err: err}
 

cmd/background-newdisks-heal-ops.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/console"
 )
 
@@ -122,7 +123,7 @@ wait:
 			// Reset to next interval.
 			diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 
-			var erasureSetInZoneDisksToHeal []map[int][]StorageAPI
+			var erasureSetInPoolDisksToHeal []map[int][]StorageAPI
 
 			healDisks := globalBackgroundHealState.getHealLocalDisks()
 			if len(healDisks) > 0 {
@@ -135,14 +136,14 @@ wait:
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
 
-				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
+				erasureSetInPoolDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
 				for i := range z.serverPools {
-					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
+					erasureSetInPoolDisksToHeal[i] = map[int][]StorageAPI{}
 				}
 			}
 
 			if serverDebugLog {
-				console.Debugf("disk check timer fired, attempting to heal %d drives\n", len(healDisks))
+				console.Debugf(color.Green("healDisk:")+" disk check timer fired, attempting to heal %d drives\n", len(healDisks))
 			}
 
 			// heal only if new disks found.
@@ -153,7 +154,7 @@ wait:
 					continue
 				}
 
-				poolIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
+				poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
 				if poolIdx < 0 {
 					continue
 				}
@@ -168,7 +169,7 @@ wait:
 					continue
 				}
 
-				erasureSetInZoneDisksToHeal[poolIdx][setIndex] = append(erasureSetInZoneDisksToHeal[poolIdx][setIndex], disk)
+				erasureSetInPoolDisksToHeal[poolIdx][setIndex] = append(erasureSetInPoolDisksToHeal[poolIdx][setIndex], disk)
 			}
 
 			buckets, _ := z.ListBuckets(ctx)
@@ -178,7 +179,7 @@ wait:
 				return buckets[i].Created.After(buckets[j].Created)
 			})
 
-			for i, setMap := range erasureSetInZoneDisksToHeal {
+			for i, setMap := range erasureSetInPoolDisksToHeal {
 				for setIndex, disks := range setMap {
 					for _, disk := range disks {
 						logger.Info("Healing disk '%s' on %s pool", disk, humanize.Ordinal(i+1))

cmd/bitrot-streaming.go

@@ -149,10 +149,9 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err := &errHashMismatch{fmt.Sprintf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
-			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
-		logger.LogIf(GlobalContext, err)
-		return 0, err
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
+		return 0, errFileCorrupt
 	}
 	b.currOffset += int64(len(buf))
 	return len(buf), nil

cmd/bucket-encryption-handlers.go

@@ -37,7 +37,7 @@ const (
 func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketEncryption")
 
-	defer logger.AuditLog(w, r, "PutBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -102,7 +102,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketEncryption")
 
-	defer logger.AuditLog(w, r, "GetBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketEncryption")
 
-	defer logger.AuditLog(w, r, "DeleteBucketEncryption", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-handlers.go

@@ -25,8 +25,10 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
@@ -72,7 +74,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
-	if err != nil && err != dns.ErrNoEntriesFound && err != dns.ErrNotImplemented {
+	if err != nil && !IsErrIgnored(err, dns.ErrNoEntriesFound, dns.ErrNotImplemented, dns.ErrDomainMissing) {
 		logger.LogIf(GlobalContext, err)
 		return
 	}
@@ -80,6 +82,10 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	bucketsSet := set.NewStringSet()
 	bucketsToBeUpdated := set.NewStringSet()
 	bucketsInConflict := set.NewStringSet()
+
+	// This means that domain is updated, we should update
+	// all bucket entries with new domain name.
+	domainMissing := err == dns.ErrDomainMissing
 	if dnsBuckets != nil {
 		for _, bucket := range buckets {
 			bucketsSet.Add(bucket.Name)
@@ -89,11 +95,16 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 				continue
 			}
 			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() && !domainMissing {
 					// No difference in terms of domainIPs and nothing
 					// has changed so we don't change anything on the etcd.
+					//
+					// Additionally also check if domain is updated/missing with more
+					// entries, if that is the case we should update the
+					// new domain entries as well.
 					continue
 				}
+
 				// if domain IPs intersect then it won't be an empty set.
 				// such an intersection means that bucket exists on etcd.
 				// but if we do see a difference with local domain IPs with
@@ -102,6 +113,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 				bucketsToBeUpdated.Add(bucket.Name)
 				continue
 			}
+
 			// No IPs seem to intersect, this means that bucket exists but has
 			// different IP addresses perhaps from a different deployment.
 			// bucket names are globally unique in federation at a given
@@ -112,8 +124,8 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	}
 
 	// Add/update buckets that are not registered with the DNS
-	g := errgroup.WithNErrs(len(buckets))
 	bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice()
+	g := errgroup.WithNErrs(len(bucketsToBeUpdatedSlice))
 	for index := range bucketsToBeUpdatedSlice {
 		index := index
 		g.Go(func() error {
@@ -128,9 +140,10 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	}
 
 	for _, bucket := range bucketsInConflict.ToSlice() {
-		logger.LogIf(GlobalContext, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", bucket, globalDomainIPs.ToSlice()))
+		logger.LogIf(GlobalContext, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket.", bucket, globalDomainIPs.ToSlice()))
 	}
 
+	var wg sync.WaitGroup
 	// Remove buckets that are in DNS for this server, but aren't local
 	for bucket, records := range dnsBuckets {
 		if bucketsSet.Contains(bucket) {
@@ -142,13 +155,18 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 			continue
 		}
 
-		// We go to here, so we know the bucket no longer exists,
-		// but is registered in DNS to this server
-		if err = globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
-				bucket, err))
-		}
+		wg.Add(1)
+		go func(bucket string) {
+			defer wg.Done()
+			// We go to here, so we know the bucket no longer exists,
+			// but is registered in DNS to this server
+			if err := globalDNSConfig.Delete(bucket); err != nil {
+				logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+					bucket, err))
+			}
+		}(bucket)
 	}
+	wg.Wait()
 }
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -157,7 +175,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
-	defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -205,7 +223,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
-	defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -260,7 +278,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
-	defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -280,7 +298,9 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	var bucketsInfo []BucketInfo
 	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
-		if err != nil && err != dns.ErrNoEntriesFound {
+		if err != nil && !IsErrIgnored(err,
+			dns.ErrNoEntriesFound,
+			dns.ErrDomainMissing) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -290,6 +310,11 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				Created: dnsRecords[0].CreationDate,
 			})
 		}
+
+		sort.Slice(bucketsInfo, func(i, j int) bool {
+			return bucketsInfo[i].Name < bucketsInfo[j].Name
+		})
+
 	} else {
 		// Invoke the list buckets.
 		var err error
@@ -347,7 +372,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteMultipleObjects")
 
-	defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -611,7 +636,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
-	defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -742,7 +767,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
-	defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
@@ -1006,7 +1031,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
-	defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1036,7 +1061,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
-	defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1124,7 +1149,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketObjectLockConfig")
 
-	defer logger.AuditLog(w, r, "PutBucketObjectLockConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1180,7 +1205,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketObjectLockConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketObjectLockConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1218,7 +1243,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketTagging")
 
-	defer logger.AuditLog(w, r, "PutBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1262,7 +1287,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketTagging")
 
-	defer logger.AuditLog(w, r, "GetBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1300,7 +1325,7 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketTagging")
 
-	defer logger.AuditLog(w, r, "DeleteBucketTagging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1330,7 +1355,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 // Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html
 func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketReplicationConfig")
-	defer logger.AuditLog(w, r, "PutBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1394,7 +1419,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketReplicationConfig")
 
-	defer logger.AuditLog(w, r, "GetBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1435,7 +1460,7 @@ func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWr
 // ----------
 func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketReplicationConfig")
-	defer logger.AuditLog(w, r, "DeleteBucketReplicationConfig", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 

cmd/bucket-lifecycle-handlers.go

@@ -38,7 +38,7 @@ const (
 func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "PutBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -103,7 +103,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "GetBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketLifecycle")
 
-	defer logger.AuditLog(w, r, "DeleteBucketLifecycle", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-lifecycle.go

@@ -284,19 +284,17 @@ func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket
 		if isDeleteTierOnly {
 			return nil
 		}
-		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
+
+		objInfo, err := objectAPI.DeleteObject(ctx, bucket, object, opts)
 		if err != nil {
 			return err
 		}
+
 		eventName := event.ObjectRemovedDelete
 		if lcOpts.DeleteMarker {
 			eventName = event.ObjectRemovedDeleteMarkerCreated
 		}
-		objInfo := ObjectInfo{
-			Name:         object,
-			VersionID:    lcOpts.VersionID,
-			DeleteMarker: lcOpts.DeleteMarker,
-		}
+
 		// Notify object deleted event.
 		sendEvent(eventArgs{
 			EventName:  eventName,
@@ -358,19 +356,17 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 	opts.TransitionStatus = lifecycle.TransitionComplete
 	eventName := event.ObjectTransitionComplete
 
-	_, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+	objInfo, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
 	if err != nil {
 		eventName = event.ObjectTransitionFailed
 	}
+
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,
-		BucketName: oi.Bucket,
-		Object: ObjectInfo{
-			Name:      oi.Name,
-			VersionID: opts.VersionID,
-		},
-		Host: "Internal: [ILM-Transition]",
+		BucketName: objInfo.Bucket,
+		Object:     objInfo,
+		Host:       "Internal: [ILM-Transition]",
 	})
 	return err
 }

cmd/bucket-listobjects-handlers.go

@@ -82,7 +82,7 @@ func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int
 func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectVersions")
 
-	defer logger.AuditLog(w, r, "ListObjectVersions", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -143,7 +143,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2M")
 
-	defer logger.AuditLog(w, r, "ListObjectsV2M", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -210,7 +210,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
-	defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -314,7 +314,7 @@ func proxyRequestByStringHash(ctx context.Context, w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
-	defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/bucket-notification-handlers.go

@@ -39,7 +39,7 @@ const (
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
-	defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
@@ -111,7 +111,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
-	defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {

cmd/bucket-policy-handlers.go

@@ -40,7 +40,7 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
-	defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
-	defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -141,7 +141,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
-	defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-replication.go

@@ -129,19 +129,29 @@ func mustReplicater(ctx context.Context, bucket, object string, meta map[string]
 		opts.UserTags = tagStr
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
-	if tgt == nil || tgt.isOffline() {
-		return cfg.Replicate(opts), false
+	// the target online status should not be used here while deciding
+	// whether to replicate as the target could be temporarily down
+	if tgt != nil {
+		return cfg.Replicate(opts), tgt.replicateSync
 	}
-	return cfg.Replicate(opts), tgt.replicateSync
+	return cfg.Replicate(opts), false
 }
 
 // Standard headers that needs to be extracted from User metadata.
 var standardHeaders = []string{
-	"content-type",
-	"content-encoding",
+	xhttp.ContentType,
+	xhttp.CacheControl,
+	xhttp.ContentEncoding,
+	xhttp.ContentLanguage,
+	xhttp.ContentDisposition,
 	xhttp.AmzStorageClass,
 	xhttp.AmzObjectTagging,
 	xhttp.AmzBucketReplicationStatus,
+	xhttp.AmzObjectLockMode,
+	xhttp.AmzObjectLockRetainUntilDate,
+	xhttp.AmzObjectLockLegalHold,
+	xhttp.AmzTagCount,
+	xhttp.AmzServerSideEncryption,
 }
 
 // returns true if any of the objects being deleted qualifies for replication.
@@ -189,7 +199,9 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 		return oi.DeleteMarker, false, sync
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
-	if tgt == nil || tgt.isOffline() {
+	// the target online status should not be used here while deciding
+	// whether to replicate deletes as the target could be temporarily down
+	if tgt == nil {
 		return oi.DeleteMarker, false, false
 	}
 	opts := replication.ObjectOpts{
@@ -216,10 +228,12 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
 	bucket := dobj.Bucket
 	rcfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || rcfg == nil {
+		logger.LogIf(ctx, err)
 		return
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
 	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, rcfg.RoleArn))
 		return
 	}
 	versionID := dobj.DeleteMarkerVersionID
@@ -251,37 +265,31 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
 			versionPurgeStatus = Complete
 		}
 	}
+
 	var eventName = event.ObjectReplicationComplete
 	if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed {
 		eventName = event.ObjectReplicationFailed
 	}
-	objInfo := ObjectInfo{
-		Name:               dobj.ObjectName,
-		DeleteMarker:       dobj.DeleteMarker,
-		VersionID:          versionID,
-		ReplicationStatus:  replication.StatusType(dobj.DeleteMarkerReplicationStatus),
-		VersionPurgeStatus: versionPurgeStatus,
-	}
-
-	eventArg := &eventArgs{
-		BucketName: bucket,
-		Object:     objInfo,
-		Host:       "Internal: [Replication]",
-		EventName:  eventName,
-	}
-	sendEvent(*eventArg)
 
 	// Update metadata on the delete marker or purge permanent delete if replication success.
-	if _, err = objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
+	objInfo, err := objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
 		VersionID:                     versionID,
 		DeleteMarker:                  dobj.DeleteMarker,
 		DeleteMarkerReplicationStatus: replicationStatus,
 		Versioned:                     globalBucketVersioningSys.Enabled(bucket),
 		VersionPurgeStatus:            versionPurgeStatus,
 		VersionSuspended:              globalBucketVersioningSys.Suspended(bucket),
-	}); err != nil {
+	})
+	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s %s: %w", bucket, dobj.ObjectName, dobj.VersionID, err))
 	}
+
+	sendEvent(eventArgs{
+		BucketName: bucket,
+		Object:     objInfo,
+		Host:       "Internal: [Replication]",
+		EventName:  eventName,
+	})
 }
 
 func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]string {
@@ -355,6 +363,15 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
 			SourceETag:        objInfo.ETag,
 		},
 	}
+	if lang, ok := objInfo.UserDefined[xhttp.ContentLanguage]; ok {
+		putOpts.ContentLanguage = lang
+	}
+	if disp, ok := objInfo.UserDefined[xhttp.ContentDisposition]; ok {
+		putOpts.ContentDisposition = disp
+	}
+	if cc, ok := objInfo.UserDefined[xhttp.CacheControl]; ok {
+		putOpts.CacheControl = cc
+	}
 	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
 		rmode := miniogo.RetentionMode(mode)
 		putOpts.Mode = rmode
@@ -390,29 +407,40 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationActio
 	if oi1.ETag != oi2.ETag ||
 		oi1.VersionID != oi2.VersionID ||
 		oi1.Size != oi2.Size ||
-		oi1.DeleteMarker != oi2.IsDeleteMarker {
+		oi1.DeleteMarker != oi2.IsDeleteMarker ||
+		!oi1.ModTime.Equal(oi2.LastModified) {
 		return replicateAll
 	}
-
-	if !oi1.ModTime.Equal(oi2.LastModified) ||
-		oi1.ContentType != oi2.ContentType ||
-		oi1.StorageClass != oi2.StorageClass {
+	if oi1.ContentType != oi2.ContentType {
 		return replicateMetadata
 	}
 	if oi1.ContentEncoding != "" {
-		enc, ok := oi2.UserMetadata[xhttp.ContentEncoding]
-		if !ok || enc != oi1.ContentEncoding {
+		enc, ok := oi2.Metadata[xhttp.ContentEncoding]
+		if !ok || strings.Join(enc, "") != oi1.ContentEncoding {
 			return replicateMetadata
 		}
 	}
-	for k, v := range oi2.UserMetadata {
-		oi2.Metadata[k] = []string{v}
-	}
-	if len(oi2.Metadata) != len(oi1.UserDefined) {
+	// compare metadata on both maps to see if meta is identical
+	for k1, v1 := range oi1.UserDefined {
+		if v2, ok := oi2.UserMetadata[k1]; ok && v1 == v2 {
+			continue
+		}
+		if v2, ok := oi2.Metadata[k1]; ok && v1 == strings.Join(v2, "") {
+			continue
+		}
 		return replicateMetadata
 	}
-	for k1, v1 := range oi1.UserDefined {
-		if v2, ok := oi2.Metadata[k1]; !ok || v1 != strings.Join(v2, "") {
+	for k1, v1 := range oi2.UserMetadata {
+		if v2, ok := oi1.UserDefined[k1]; !ok || v1 != v2 {
+			return replicateMetadata
+		}
+	}
+	for k1, v1slc := range oi2.Metadata {
+		v1 := strings.Join(v1slc, "")
+		if k1 == xhttp.ContentEncoding { //already compared
+			continue
+		}
+		if v2, ok := oi1.UserDefined[k1]; !ok || v1 != v2 {
 			return replicateMetadata
 		}
 	}
@@ -461,7 +489,11 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	}
 
 	rtype := replicateAll
-	oi, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
+	oi, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{
+		VersionID: objInfo.VersionID,
+		Internal: miniogo.AdvancedGetOptions{
+			ReplicationProxyRequest: "false",
+		}})
 	if err == nil {
 		rtype = getReplicationAction(objInfo, oi)
 		if rtype == replicateNone {
@@ -474,7 +506,6 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	replicationStatus := replication.Completed
 	if rtype != replicateAll {
 		gr.Close()
-
 		// replicate metadata for object tagging/copy with metadata replacement
 		dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
 		c := &miniogo.Core{Client: tgt.Client}
@@ -511,6 +542,7 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		}
 		r.Close()
 	}
+
 	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
 	if objInfo.UserTags != "" {
 		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
@@ -524,20 +556,23 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
 	}
+
+	objInfo.metadataOnly = true // Perform only metadata updates.
+	objInfo, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	}, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	})
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
+	}
+
 	sendEvent(eventArgs{
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
 		Host:       "Internal: [Replication]",
 	})
-	objInfo.metadataOnly = true // Perform only metadata updates.
-	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
-		VersionID: objInfo.VersionID,
-	}, ObjectOptions{
-		VersionID: objInfo.VersionID,
-	}); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
-	}
 }
 
 // filterReplicationStatusMetadata filters replication status metadata for COPY
@@ -670,7 +705,7 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		VersionID:            opts.VersionID,
 		ServerSideEncryption: opts.ServerSideEncryption,
 		Internal: miniogo.AdvancedGetOptions{
-			ReplicationProxyRequest: true,
+			ReplicationProxyRequest: "true",
 		},
 	}
 	// get correct offsets for encrypted object
@@ -703,10 +738,11 @@ func isProxyable(ctx context.Context, bucket string) bool {
 	dest := cfg.GetDestination()
 	return dest.Bucket == bucket
 }
+
 func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (tgt *TargetClient, oi ObjectInfo, proxy bool, err error) {
 	// this option is set when active-active replication is in place between site A -> B,
 	// and site B does not have the object yet.
-	if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
 		return nil, oi, false, nil
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
@@ -730,14 +766,14 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 	}
 	tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil || tgt.isOffline() {
-		return nil, oi, false, fmt.Errorf("missing target")
+		return nil, oi, false, fmt.Errorf("target is offline or not configured")
 	}
 
 	gopts := miniogo.GetObjectOptions{
 		VersionID:            opts.VersionID,
 		ServerSideEncryption: opts.ServerSideEncryption,
 		Internal: miniogo.AdvancedGetOptions{
-			ReplicationProxyRequest: true,
+			ReplicationProxyRequest: "true",
 		},
 	}
 

cmd/bucket-targets.go

@@ -36,7 +36,7 @@ import (
 )
 
 const (
-	defaultHealthCheckDuration = 60 * time.Second
+	defaultHealthCheckDuration = 100 * time.Second
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -461,10 +461,10 @@ func (tc *TargetClient) healthCheck() {
 		_, err := tc.BucketExists(GlobalContext, tc.bucket)
 		if err != nil {
 			atomic.StoreInt32(&tc.up, 0)
-			time.Sleep(tc.healthCheckDuration * time.Second)
+			time.Sleep(tc.healthCheckDuration)
 			continue
 		}
 		atomic.StoreInt32(&tc.up, 1)
-		time.Sleep(tc.healthCheckDuration * time.Second)
+		time.Sleep(tc.healthCheckDuration)
 	}
 }

cmd/bucket-versioning-handler.go

@@ -40,7 +40,7 @@ const (
 func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketVersioning")
 
-	defer logger.AuditLog(w, r, "PutBucketVersioning", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -98,7 +98,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketVersioning")
 
-	defer logger.AuditLog(w, r, "GetBucketVersioning", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/common-main.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
@@ -27,6 +28,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"time"
 
@@ -41,6 +43,7 @@ import (
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/handlers"
 )
 
 // serverDebugLog will enable debug printing
@@ -51,10 +54,21 @@ func init() {
 	logger.RegisterError(config.FmtError)
 
 	rand.Seed(time.Now().UTC().UnixNano())
+
 	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second, logger.LogOnceIf)
 
 	initGlobalContext()
 
+	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
+		PassHost:     true,
+		RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
+		Logger: func(err error) {
+			if err != nil && !errors.Is(err, context.Canceled) {
+				logger.LogIf(GlobalContext, err)
+			}
+		},
+	})
+
 	globalReplicationState = newReplicationState()
 	globalTransitionState = newTransitionState()
 
@@ -241,6 +255,14 @@ func handleCommonEnvVars() {
 			}
 			globalDomainNames = append(globalDomainNames, domainName)
 		}
+		sort.Strings(globalDomainNames)
+		lcpSuf := lcpSuffix(globalDomainNames)
+		for _, domainName := range globalDomainNames {
+			if domainName == lcpSuf && len(globalDomainNames) > 1 {
+				logger.Fatal(config.ErrOverlappingDomainValue(nil).Msg("Overlapping domains `%s` not allowed", globalDomainNames),
+					"Invalid MINIO_DOMAIN value in environment variable")
+			}
+		}
 	}
 
 	publicIPs := env.Get(config.EnvPublicIPs, "")

cmd/config-current.go

@@ -228,7 +228,7 @@ var (
 	globalServerConfigMu sync.RWMutex
 )
 
-func validateConfig(s config.Config, setDriveCount int) error {
+func validateConfig(s config.Config, setDriveCounts []int) error {
 	// We must have a global lock for this so nobody else modifies env while we do.
 	defer env.LockSetEnv()()
 
@@ -251,8 +251,10 @@ func validateConfig(s config.Config, setDriveCount int) error {
 	}
 
 	if globalIsErasure {
-		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
-			return err
+		for _, setDriveCount := range setDriveCounts {
+			if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -342,7 +344,7 @@ func validateConfig(s config.Config, setDriveCount int) error {
 	return notify.TestNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs())
 }
 
-func lookupConfigs(s config.Config, setDriveCount int) {
+func lookupConfigs(s config.Config, setDriveCounts []int) {
 	ctx := GlobalContext
 
 	var err error
@@ -429,7 +431,7 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	globalAPIConfig.init(apiConfig, setDriveCount)
+	globalAPIConfig.init(apiConfig, setDriveCounts)
 
 	// Initialize remote instance transport once.
 	getRemoteInstanceTransportOnce.Do(func() {
@@ -437,9 +439,17 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	})
 
 	if globalIsErasure {
-		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+		for i, setDriveCount := range setDriveCounts {
+			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+				break
+			}
+			// if we validated all setDriveCounts and it was successful
+			// proceed to store the correct storage class globally.
+			if i == len(setDriveCounts)-1 {
+				globalStorageClass = sc
+			}
 		}
 	}
 
@@ -553,12 +563,16 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	}
 
 	// Apply dynamic config values
-	logger.LogIf(ctx, applyDynamicConfig(ctx, s))
+	logger.LogIf(ctx, applyDynamicConfig(ctx, newObjectLayerFn(), s))
 }
 
 // applyDynamicConfig will apply dynamic config values.
 // Dynamic systems should be in config.SubSystemsDynamic as well.
-func applyDynamicConfig(ctx context.Context, s config.Config) error {
+func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error {
+	if objAPI == nil {
+		return nil
+	}
+
 	// Read all dynamic configs.
 	// API
 	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
@@ -571,17 +585,16 @@ func applyDynamicConfig(ctx context.Context, s config.Config) error {
 	if err != nil {
 		return fmt.Errorf("Unable to setup Compression: %w", err)
 	}
-	objAPI := newObjectLayerFn()
-	if objAPI != nil {
-		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
-			return fmt.Errorf("Backend does not support compression")
-		}
+
+	// Validate if the object layer supports compression.
+	if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
+		return fmt.Errorf("Backend does not support compression")
 	}
 
 	// Heal
 	healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to apply heal config: %w", err))
+		return fmt.Errorf("Unable to apply heal config: %w", err)
 	}
 
 	// Crawler
@@ -592,7 +605,7 @@ func applyDynamicConfig(ctx context.Context, s config.Config) error {
 
 	// Apply configurations.
 	// We should not fail after this.
-	globalAPIConfig.init(apiConfig, globalAPIConfig.setDriveCount)
+	globalAPIConfig.init(apiConfig, objAPI.SetDriveCounts())
 
 	globalCompressConfigMu.Lock()
 	globalCompressConfig = cmpCfg
@@ -723,7 +736,7 @@ func loadConfig(objAPI ObjectLayer) error {
 	}
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg, objAPI.SetDriveCount())
+	lookupConfigs(srvCfg, objAPI.SetDriveCounts())
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/config/dns/etcd_dns.go

@@ -34,6 +34,9 @@ import (
 // ErrNoEntriesFound - Indicates no entries were found for the given key (directory)
 var ErrNoEntriesFound = errors.New("No entries found for this key")
 
+// ErrDomainMissing - Indicates domain is missing
+var ErrDomainMissing = errors.New("domain is missing")
+
 const etcdPathSeparator = "/"
 
 // create a new coredns service record for the bucket.
@@ -57,9 +60,9 @@ func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
 	var srvRecords = map[string][]SrvRecord{}
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.", domainName), c.prefixPath)
-		records, err := c.list(key)
+		records, err := c.list(key+etcdPathSeparator, true)
 		if err != nil {
-			return nil, err
+			return srvRecords, err
 		}
 		for _, record := range records {
 			if record.Key == "" {
@@ -76,7 +79,7 @@ func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) {
 	var srvRecords []SrvRecord
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
-		records, err := c.list(key)
+		records, err := c.list(key+etcdPathSeparator, false)
 		if err != nil {
 			return nil, err
 		}
@@ -109,19 +112,25 @@ func msgUnPath(s string) string {
 
 // Retrieves list of entries under the key passed.
 // Note that this method fetches entries upto only two levels deep.
-func (c *CoreDNS) list(key string) ([]SrvRecord, error) {
+func (c *CoreDNS) list(key string, domain bool) ([]SrvRecord, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix())
 	defer cancel()
 	if err != nil {
 		return nil, err
 	}
+
 	if r.Count == 0 {
 		key = strings.TrimSuffix(key, etcdPathSeparator)
 		r, err = c.etcdClient.Get(ctx, key)
 		if err != nil {
 			return nil, err
 		}
+		// only if we are looking at `domain` as true
+		// we should return error here.
+		if domain && r.Count == 0 {
+			return nil, ErrDomainMissing
+		}
 	}
 
 	var srvRecords []SrvRecord
@@ -166,11 +175,11 @@ func (c *CoreDNS) Put(bucket string) error {
 			key = key + etcdPathSeparator + ip
 			ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 			_, err = c.etcdClient.Put(ctx, key, string(bucketMsg))
-			defer cancel()
+			cancel()
 			if err != nil {
 				ctx, cancel = context.WithTimeout(context.Background(), defaultContextTimeout)
 				c.etcdClient.Delete(ctx, key)
-				defer cancel()
+				cancel()
 				return err
 			}
 		}
@@ -182,18 +191,12 @@ func (c *CoreDNS) Put(bucket string) error {
 func (c *CoreDNS) Delete(bucket string) error {
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath)
-		srvRecords, err := c.list(key)
+		ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+		_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator, clientv3.WithPrefix())
+		cancel()
 		if err != nil {
 			return err
 		}
-		for _, record := range srvRecords {
-			dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-			if _, err = c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
-				dcancel()
-				return err
-			}
-			dcancel()
-		}
 	}
 	return nil
 }
@@ -203,12 +206,12 @@ func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
 	for _, domainName := range c.domainNames {
 		key := msg.Path(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath)
 
-		dctx, dcancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-		if _, err := c.etcdClient.Delete(dctx, key+etcdPathSeparator+record.Host); err != nil {
-			dcancel()
+		ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+		_, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator+record.Host)
+		cancel()
+		if err != nil {
 			return err
 		}
-		dcancel()
 	}
 	return nil
 }

cmd/config/errors.go

@@ -30,6 +30,12 @@ var (
 		"Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`",
 	)
 
+	ErrOverlappingDomainValue = newErrFn(
+		"Overlapping domain values",
+		"Please check the passed value",
+		"MINIO_DOMAIN only accepts non-overlapping domain values",
+	)
+
 	ErrInvalidDomainValue = newErrFn(
 		"Invalid domain value",
 		"Please check the passed value",

cmd/config/etcd/etcd.go

@@ -27,6 +27,7 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 	"go.etcd.io/etcd/clientv3"
 	"go.etcd.io/etcd/clientv3/namespace"
+	"go.uber.org/zap"
 )
 
 const (
@@ -144,6 +145,13 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
 	cfg.Enabled = true
 	cfg.DialTimeout = defaultDialTimeout
 	cfg.DialKeepAliveTime = defaultDialKeepAlive
+	// Disable etcd client SDK logging, etcd client
+	// incorrectly starts logging in unexpected data
+	// format.
+	cfg.LogConfig = &zap.Config{
+		Level:    zap.NewAtomicLevelAt(zap.FatalLevel),
+		Encoding: "console",
+	}
 	cfg.Endpoints = etcdEndpoints
 	cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath))
 	// Default path prefix for all keys on etcd, other than CoreDNSPath.

cmd/config/identity/ldap/config.go

@@ -32,6 +32,8 @@ import (
 
 const (
 	defaultLDAPExpiry = time.Hour * 1
+
+	dnDelimiter = ";"
 )
 
 // Config contains AD/LDAP server connectivity information.
@@ -45,50 +47,67 @@ type Config struct {
 	STSExpiryDuration string `json:"stsExpiryDuration"`
 
 	// Format string for usernames
-	UsernameFormat        string   `json:"usernameFormat"`
-	UsernameFormats       []string `json:"-"`
-	UsernameSearchFilter  string   `json:"-"`
-	UsernameSearchBaseDNS []string `json:"-"`
+	UsernameFormat  string   `json:"usernameFormat"`
+	UsernameFormats []string `json:"-"`
 
-	GroupSearchBaseDN  string   `json:"groupSearchBaseDN"`
-	GroupSearchBaseDNS []string `json:"-"`
-	GroupSearchFilter  string   `json:"groupSearchFilter"`
-	GroupNameAttribute string   `json:"groupNameAttribute"`
+	// User DN search parameters
+	UserDNSearchBaseDN string `json:"userDNSearchBaseDN"`
+	UserDNSearchFilter string `json:"userDNSearchFilter"`
+
+	// Group search parameters
+	GroupSearchBaseDistName  string   `json:"groupSearchBaseDN"`
+	GroupSearchBaseDistNames []string `json:"-"`
+	GroupSearchFilter        string   `json:"groupSearchFilter"`
+	GroupNameAttribute       string   `json:"groupNameAttribute"`
+
+	// Lookup bind LDAP service account
+	LookupBindDN       string `json:"lookupBindDN"`
+	LookupBindPassword string `json:"lookupBindPassword"`
 
 	stsExpiryDuration time.Duration // contains converted value
 	tlsSkipVerify     bool          // allows skipping TLS verification
-	serverInsecure    bool          // allows plain text connection to LDAP Server
-	serverStartTLS    bool          // allows plain text connection to LDAP Server
+	serverInsecure    bool          // allows plain text connection to LDAP server
+	serverStartTLS    bool          // allows using StartTLS connection to LDAP server
+	isUsingLookupBind bool
 	rootCAs           *x509.CertPool
 }
 
 // LDAP keys and envs.
 const (
-	ServerAddr           = "server_addr"
-	STSExpiry            = "sts_expiry"
-	UsernameFormat       = "username_format"
-	UsernameSearchFilter = "username_search_filter"
-	UsernameSearchBaseDN = "username_search_base_dn"
-	GroupSearchFilter    = "group_search_filter"
-	GroupNameAttribute   = "group_name_attribute"
-	GroupSearchBaseDN    = "group_search_base_dn"
-	TLSSkipVerify        = "tls_skip_verify"
-	ServerInsecure       = "server_insecure"
-	ServerStartTLS       = "server_starttls"
+	ServerAddr         = "server_addr"
+	STSExpiry          = "sts_expiry"
+	LookupBindDN       = "lookup_bind_dn"
+	LookupBindPassword = "lookup_bind_password"
+	UserDNSearchBaseDN = "user_dn_search_base_dn"
+	UserDNSearchFilter = "user_dn_search_filter"
+	UsernameFormat     = "username_format"
+	GroupSearchFilter  = "group_search_filter"
+	GroupNameAttribute = "group_name_attribute"
+	GroupSearchBaseDN  = "group_search_base_dn"
+	TLSSkipVerify      = "tls_skip_verify"
+	ServerInsecure     = "server_insecure"
+	ServerStartTLS     = "server_starttls"
 
-	EnvServerAddr           = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
-	EnvSTSExpiry            = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
-	EnvTLSSkipVerify        = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
-	EnvServerInsecure       = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
-	EnvServerStartTLS       = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
-	EnvUsernameFormat       = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
-	EnvUsernameSearchFilter = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_FILTER"
-	EnvUsernameSearchBaseDN = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_BASE_DN"
-	EnvGroupSearchFilter    = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
-	EnvGroupNameAttribute   = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
-	EnvGroupSearchBaseDN    = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+	EnvServerAddr         = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
+	EnvSTSExpiry          = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
+	EnvTLSSkipVerify      = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
+	EnvServerInsecure     = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
+	EnvServerStartTLS     = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
+	EnvUsernameFormat     = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
+	EnvUserDNSearchBaseDN = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN"
+	EnvUserDNSearchFilter = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER"
+	EnvGroupSearchFilter  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
+	EnvGroupNameAttribute = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
+	EnvGroupSearchBaseDN  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+	EnvLookupBindDN       = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN"
+	EnvLookupBindPassword = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD"
 )
 
+var removedKeys = []string{
+	"username_search_filter",
+	"username_search_base_dn",
+}
+
 // DefaultKVS - default config for LDAP config
 var (
 	DefaultKVS = config.KVS{
@@ -101,11 +120,11 @@ var (
 			Value: "",
 		},
 		config.KV{
-			Key:   UsernameSearchFilter,
+			Key:   UserDNSearchBaseDN,
 			Value: "",
 		},
 		config.KV{
-			Key:   UsernameSearchBaseDN,
+			Key:   UserDNSearchFilter,
 			Value: "",
 		},
 		config.KV{
@@ -136,17 +155,26 @@ var (
 			Key:   ServerStartTLS,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   LookupBindDN,
+			Value: "",
+		},
+		config.KV{
+			Key:   LookupBindPassword,
+			Value: "",
+		},
 	}
 )
 
-const (
-	dnDelimiter = ";"
-)
-
 func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
 	var groups []string
 	sres, err := conn.Search(sreq)
 	if err != nil {
+		// Check if there is no matching result and return empty slice.
+		// Ref: https://ldap.com/ldap-result-code-reference/
+		if ldap.IsErrorWithCode(err, 32) {
+			return nil, nil
+		}
 		return nil, err
 	}
 	for _, entry := range sres.Entries {
@@ -157,96 +185,149 @@ func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
 	return groups, nil
 }
 
-func (l *Config) bind(conn *ldap.Conn, username, password string) ([]string, error) {
-	var bindDNS = make([]string, len(l.UsernameFormats))
+func (l *Config) lookupBind(conn *ldap.Conn) error {
+	return conn.Bind(l.LookupBindDN, l.LookupBindPassword)
+}
+
+// usernameFormatsBind - Iterates over all given username formats and expects
+// that only one will succeed if the credentials are valid. The succeeding
+// bindDN is returned or an error.
+//
+// In the rare case that multiple username formats succeed, implying that two
+// (or more) distinct users in the LDAP directory have the same username and
+// password, we return an error as we cannot identify the account intended by
+// the user.
+func (l *Config) usernameFormatsBind(conn *ldap.Conn, username, password string) (string, error) {
+	var bindDistNames []string
+	var errs = make([]error, len(l.UsernameFormats))
+	var successCount = 0
 	for i, usernameFormat := range l.UsernameFormats {
 		bindDN := fmt.Sprintf(usernameFormat, username)
 		// Bind with user credentials to validate the password
-		if err := conn.Bind(bindDN, password); err != nil {
-			return nil, err
+		errs[i] = conn.Bind(bindDN, password)
+		if errs[i] == nil {
+			bindDistNames = append(bindDistNames, bindDN)
+			successCount++
 		}
-		bindDNS[i] = bindDN
 	}
-	return bindDNS, nil
+	if successCount == 0 {
+		var errStrings []string
+		for _, err := range errs {
+			if err != nil {
+				errStrings = append(errStrings, err.Error())
+			}
+		}
+		outErr := fmt.Sprintf("All username formats failed with: %s", strings.Join(errStrings, "; "))
+		return "", errors.New(outErr)
+	}
+	if successCount > 1 {
+		successDistNames := strings.Join(bindDistNames, ", ")
+		errMsg := fmt.Sprintf("Multiple username formats succeeded - ambiguous user login (succeeded for: %s)", successDistNames)
+		return "", errors.New(errMsg)
+	}
+	return bindDistNames[0], nil
 }
 
-var standardAttributes = []string{
-	"givenName",
-	"sn",
-	"cn",
-	"memberOf",
-	"email",
+// lookupUserDN searches for the DN of the user given their username. conn is
+// assumed to be using the lookup bind service account. It is required that the
+// search result in at most one result.
+func (l *Config) lookupUserDN(conn *ldap.Conn, username string) (string, error) {
+	filter := strings.Replace(l.UserDNSearchFilter, "%s", ldap.EscapeFilter(username), -1)
+	searchRequest := ldap.NewSearchRequest(
+		l.UserDNSearchBaseDN,
+		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+		filter,
+		[]string{}, // only need DN, so no pass no attributes here
+		nil,
+	)
+
+	searchResult, err := conn.Search(searchRequest)
+	if err != nil {
+		return "", err
+	}
+	if len(searchResult.Entries) == 0 {
+		return "", fmt.Errorf("User DN for %s not found", username)
+	}
+	if len(searchResult.Entries) != 1 {
+		return "", fmt.Errorf("Multiple DNs for %s found - please fix the search filter", username)
+	}
+	return searchResult.Entries[0].DN, nil
 }
 
-// Bind - binds to ldap, searches LDAP and returns list of groups.
-func (l *Config) Bind(username, password string) ([]string, error) {
+// Bind - binds to ldap, searches LDAP and returns the distinguished name of the
+// user and the list of groups.
+func (l *Config) Bind(username, password string) (string, []string, error) {
 	conn, err := l.Connect()
 	if err != nil {
-		return nil, err
+		return "", nil, err
 	}
 	defer conn.Close()
 
-	bindDNS, err := l.bind(conn, username, password)
-	if err != nil {
-		return nil, err
+	var bindDN string
+	if l.isUsingLookupBind {
+		// Bind to the lookup user account
+		if err = l.lookupBind(conn); err != nil {
+			return "", nil, err
+		}
+
+		// Lookup user DN
+		bindDN, err = l.lookupUserDN(conn, username)
+		if err != nil {
+			errRet := fmt.Errorf("Unable to find user DN: %s", err)
+			return "", nil, errRet
+		}
+
+		// Authenticate the user credentials.
+		err = conn.Bind(bindDN, password)
+		if err != nil {
+			errRet := fmt.Errorf("LDAP auth failed for DN %s: %v", bindDN, err)
+			return "", nil, errRet
+		}
+
+		// Bind to the lookup user account again to perform group search.
+		if err = l.lookupBind(conn); err != nil {
+			return "", nil, err
+		}
+	} else {
+		// Verify login credentials by checking the username formats.
+		bindDN, err = l.usernameFormatsBind(conn, username, password)
+		if err != nil {
+			return "", nil, err
+		}
+
+		// Bind to the successful bindDN again.
+		err = conn.Bind(bindDN, password)
+		if err != nil {
+			errRet := fmt.Errorf("LDAP conn failed though auth for DN %s succeeded: %v", bindDN, err)
+			return "", nil, errRet
+		}
 	}
 
+	// User groups lookup.
 	var groups []string
-	if l.UsernameSearchFilter != "" {
-		for _, userSearchBase := range l.UsernameSearchBaseDNS {
-			filter := strings.Replace(l.UsernameSearchFilter, "%s",
-				ldap.EscapeFilter(username), -1)
-
+	if l.GroupSearchFilter != "" {
+		for _, groupSearchBase := range l.GroupSearchBaseDistNames {
+			filter := strings.Replace(l.GroupSearchFilter, "%s", ldap.EscapeFilter(bindDN), -1)
 			searchRequest := ldap.NewSearchRequest(
-				userSearchBase,
+				groupSearchBase,
 				ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
 				filter,
-				standardAttributes,
+				[]string{l.GroupNameAttribute},
 				nil,
 			)
 
-			groups, err = getGroups(conn, searchRequest)
+			var newGroups []string
+			newGroups, err = getGroups(conn, searchRequest)
 			if err != nil {
-				return nil, err
+				errRet := fmt.Errorf("Error finding groups of %s: %v", bindDN, err)
+				return "", nil, errRet
 			}
+
+			groups = append(groups, newGroups...)
 		}
 	}
 
-	if l.GroupSearchFilter != "" {
-		for _, groupSearchBase := range l.GroupSearchBaseDNS {
-			var filters []string
-			if l.GroupNameAttribute == "" {
-				filters = []string{strings.Replace(l.GroupSearchFilter, "%s",
-					ldap.EscapeFilter(username), -1)}
-			} else {
-				// With group name attribute specified, make sure to
-				// include search queries for CN distinguished name
-				for _, bindDN := range bindDNS {
-					filters = append(filters, strings.Replace(l.GroupSearchFilter, "%s",
-						ldap.EscapeFilter(bindDN), -1))
-				}
-			}
-			for _, filter := range filters {
-				searchRequest := ldap.NewSearchRequest(
-					groupSearchBase,
-					ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
-					filter,
-					standardAttributes,
-					nil,
-				)
-
-				var newGroups []string
-				newGroups, err = getGroups(conn, searchRequest)
-				if err != nil {
-					return nil, err
-				}
-
-				groups = append(groups, newGroups...)
-			}
-		}
-	}
-
-	return groups, nil
+	return bindDN, groups, nil
 }
 
 // Connect connect to ldap server.
@@ -295,6 +376,12 @@ func Enabled(kvs config.KVS) bool {
 // Lookup - initializes LDAP config, overrides config, if any ENV values are set.
 func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 	l = Config{}
+
+	// Purge all removed keys first
+	for _, k := range removedKeys {
+		kvs.Delete(k)
+	}
+
 	if err = config.CheckValidKeys(config.IdentityLDAPSubSys, kvs, DefaultKVS); err != nil {
 		return l, err
 	}
@@ -316,6 +403,8 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 		l.STSExpiryDuration = v
 		l.stsExpiryDuration = expDur
 	}
+
+	// LDAP connection configuration
 	if v := env.Get(EnvServerInsecure, kvs.Get(ServerInsecure)); v != "" {
 		l.serverInsecure, err = config.ParseBool(v)
 		if err != nil {
@@ -334,26 +423,45 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 			return l, err
 		}
 	}
+
+	// Lookup bind user configuration
+	lookupBindDN := env.Get(EnvLookupBindDN, kvs.Get(LookupBindDN))
+	lookupBindPassword := env.Get(EnvLookupBindPassword, kvs.Get(LookupBindPassword))
+	if lookupBindDN != "" && lookupBindPassword != "" {
+		l.LookupBindDN = lookupBindDN
+		l.LookupBindPassword = lookupBindPassword
+		l.isUsingLookupBind = true
+
+		// User DN search configuration
+		userDNSearchBaseDN := env.Get(EnvUserDNSearchBaseDN, kvs.Get(UserDNSearchBaseDN))
+		userDNSearchFilter := env.Get(EnvUserDNSearchFilter, kvs.Get(EnvUserDNSearchFilter))
+		if userDNSearchFilter == "" || userDNSearchBaseDN == "" {
+			return l, errors.New("In lookup bind mode, userDN search base DN and userDN search filter are both required")
+		}
+		l.UserDNSearchBaseDN = userDNSearchBaseDN
+		l.UserDNSearchFilter = userDNSearchFilter
+	}
+
+	// Username format configuration.
 	if v := env.Get(EnvUsernameFormat, kvs.Get(UsernameFormat)); v != "" {
 		if !strings.Contains(v, "%s") {
 			return l, errors.New("LDAP username format doesn't have '%s' substitution")
 		}
 		l.UsernameFormats = strings.Split(v, dnDelimiter)
-	} else {
-		return l, fmt.Errorf("'%s' cannot be empty and must have a value", UsernameFormat)
 	}
 
-	if v := env.Get(EnvUsernameSearchFilter, kvs.Get(UsernameSearchFilter)); v != "" {
-		if !strings.Contains(v, "%s") {
-			return l, errors.New("LDAP username search filter doesn't have '%s' substitution")
-		}
-		l.UsernameSearchFilter = v
+	// Either lookup bind mode or username format is supported, but not
+	// both.
+	if l.isUsingLookupBind && len(l.UsernameFormats) > 0 {
+		return l, errors.New("Lookup Bind mode and Username Format mode are not supported at the same time")
 	}
 
-	if v := env.Get(EnvUsernameSearchBaseDN, kvs.Get(UsernameSearchBaseDN)); v != "" {
-		l.UsernameSearchBaseDNS = strings.Split(v, dnDelimiter)
+	// At least one of bind mode or username format must be used.
+	if !l.isUsingLookupBind && len(l.UsernameFormats) == 0 {
+		return l, errors.New("Either Lookup Bind mode or Username Format mode is required.")
 	}
 
+	// Group search params configuration
 	grpSearchFilter := env.Get(EnvGroupSearchFilter, kvs.Get(GroupSearchFilter))
 	grpSearchNameAttr := env.Get(EnvGroupNameAttribute, kvs.Get(GroupNameAttribute))
 	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, kvs.Get(GroupSearchBaseDN))
@@ -370,7 +478,8 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 	if allSet {
 		l.GroupSearchFilter = grpSearchFilter
 		l.GroupNameAttribute = grpSearchNameAttr
-		l.GroupSearchBaseDNS = strings.Split(grpSearchBaseDN, dnDelimiter)
+		l.GroupSearchBaseDistName = grpSearchBaseDN
+		l.GroupSearchBaseDistNames = strings.Split(l.GroupSearchBaseDistName, dnDelimiter)
 	}
 
 	l.rootCAs = rootCAs

cmd/config/identity/ldap/help.go

@@ -27,30 +27,46 @@ var (
 			Type:        "address",
 		},
 		config.HelpKV{
-			Key:         UsernameFormat,
-			Description: `";" separated list of username bind DNs e.g. "uid=%s,cn=accounts,dc=myldapserver,dc=com"`,
-			Type:        "list",
+			Key:         STSExpiry,
+			Description: `temporary credentials validity duration in s,m,h,d. Default is "1h"`,
+			Optional:    true,
+			Type:        "duration",
 		},
 		config.HelpKV{
-			Key:         UsernameSearchFilter,
-			Description: `user search filter, for example "(cn=%s)" or "(sAMAccountName=%s)" or "(uid=%s)"`,
+			Key:         LookupBindDN,
+			Description: `DN for LDAP read-only service account used to perform DN and group lookups`,
+			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         LookupBindPassword,
+			Description: `Password for LDAP read-only service account used to perform DN and group lookups`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         UserDNSearchBaseDN,
+			Description: `Base LDAP DN to search for user DN`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         UserDNSearchFilter,
+			Description: `Search filter to lookup user DN`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         UsernameFormat,
+			Description: `";" separated list of username bind DNs e.g. "uid=%s,cn=accounts,dc=myldapserver,dc=com"`,
+			Optional:    true,
+			Type:        "list",
+		},
 		config.HelpKV{
 			Key:         GroupSearchFilter,
 			Description: `search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"`,
-			Type:        "string",
-		},
-		config.HelpKV{
-			Key:         GroupSearchBaseDN,
-			Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"`,
-			Type:        "list",
-		},
-		config.HelpKV{
-			Key:         UsernameSearchBaseDN,
-			Description: `";" separated list of username search DNs`,
-			Type:        "list",
 			Optional:    true,
+			Type:        "string",
 		},
 		config.HelpKV{
 			Key:         GroupNameAttribute,
@@ -59,10 +75,10 @@ var (
 			Type:        "string",
 		},
 		config.HelpKV{
-			Key:         STSExpiry,
-			Description: `temporary credentials validity duration in s,m,h,d. Default is "1h"`,
+			Key:         GroupSearchBaseDN,
+			Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"`,
 			Optional:    true,
-			Type:        "duration",
+			Type:        "list",
 		},
 		config.HelpKV{
 			Key:         TLSSkipVerify,
@@ -76,6 +92,12 @@ var (
 			Optional:    true,
 			Type:        "on|off",
 		},
+		config.HelpKV{
+			Key:         ServerStartTLS,
+			Description: `use StartTLS connection to AD/LDAP server, defaults to "off"`,
+			Optional:    true,
+			Type:        "on|off",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/identity/ldap/legacy.go

@@ -47,7 +47,7 @@ func SetIdentityLDAP(s config.Config, ldapArgs Config) {
 		},
 		config.KV{
 			Key:   GroupSearchBaseDN,
-			Value: ldapArgs.GroupSearchBaseDN,
+			Value: ldapArgs.GroupSearchBaseDistName,
 		},
 	}
 }

cmd/config/storageclass/storage-class.go

@@ -64,7 +64,7 @@ const (
 	defaultRRSParity = minParityDisks
 
 	// Default DMA value
-	defaultDMA = DMAWrite
+	defaultDMA = DMAReadWrite
 )
 
 // DefaultKVS - default storage class config
@@ -88,14 +88,13 @@ var (
 // StorageClass - holds storage class information
 type StorageClass struct {
 	Parity int
-	DMA    string
 }
 
 // Config storage class configuration
 type Config struct {
 	Standard StorageClass `json:"standard"`
 	RRS      StorageClass `json:"rrs"`
-	DMA      StorageClass `json:"dma"`
+	DMA      string       `json:"dma"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -112,7 +111,7 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 // IsValid - returns true if input string is a valid
 // storage class kind supported.
 func IsValid(sc string) bool {
-	return sc == RRS || sc == STANDARD || sc == DMA
+	return sc == RRS || sc == STANDARD
 }
 
 // UnmarshalText unmarshals storage class from its textual form into
@@ -122,14 +121,6 @@ func (sc *StorageClass) UnmarshalText(b []byte) error {
 	if scStr == "" {
 		return nil
 	}
-	if scStr == DMAWrite {
-		sc.DMA = DMAWrite
-		return nil
-	}
-	if scStr == DMAReadWrite {
-		sc.DMA = DMAReadWrite
-		return nil
-	}
 	s, err := parseStorageClass(scStr)
 	if err != nil {
 		return err
@@ -143,14 +134,14 @@ func (sc *StorageClass) MarshalText() ([]byte, error) {
 	if sc.Parity != 0 {
 		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
 	}
-	return []byte(sc.DMA), nil
+	return []byte{}, nil
 }
 
 func (sc *StorageClass) String() string {
 	if sc.Parity != 0 {
 		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
 	}
-	return sc.DMA
+	return ""
 }
 
 // Parses given storageClassEnv and returns a storageClass structure.
@@ -182,22 +173,34 @@ func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
 	}, nil
 }
 
-// Validates the parity disks.
-func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
-	if ssParity == 0 && rrsParity == 0 {
-		return nil
-	}
-
+// ValidateParity validate standard storage class parity.
+func ValidateParity(ssParity, setDriveCount int) error {
 	// SS parity disks should be greater than or equal to minParityDisks.
 	// Parity below minParityDisks is not supported.
-	if ssParity < minParityDisks {
+	if ssParity > 0 && ssParity < minParityDisks {
+		return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
+			ssParity, minParityDisks)
+	}
+
+	if ssParity > setDriveCount/2 {
+		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
+	}
+
+	return nil
+}
+
+// Validates the parity disks.
+func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
+	// SS parity disks should be greater than or equal to minParityDisks.
+	// Parity below minParityDisks is not supported.
+	if ssParity > 0 && ssParity < minParityDisks {
 		return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
 			ssParity, minParityDisks)
 	}
 
 	// RRS parity disks should be greater than or equal to minParityDisks.
 	// Parity below minParityDisks is not supported.
-	if rrsParity < minParityDisks {
+	if rrsParity > 0 && rrsParity < minParityDisks {
 		return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
 	}
 
@@ -210,7 +213,7 @@ func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
 	}
 
 	if ssParity > 0 && rrsParity > 0 {
-		if ssParity < rrsParity {
+		if ssParity > 0 && ssParity < rrsParity {
 			return fmt.Errorf("Standard storage class parity disks %d should be greater than or equal to Reduced redundancy storage class parity disks %d", ssParity, rrsParity)
 		}
 	}
@@ -218,14 +221,16 @@ func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
 }
 
 // GetParityForSC - Returns the data and parity drive count based on storage class
-// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and MINIO_STORAGE_CLASS_STANDARD
-// or config.json fields
-// -- corresponding values are returned
-// If storage class is not set during startup, default values are returned
-// -- Default for Reduced Redundancy Storage class is, parity = 2 and data = N-Parity
-// -- Default for Standard Storage class is, parity = N/2, data = N/2
-// If storage class is empty
-// -- standard storage class is assumed and corresponding data and parity is returned
+// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and
+// MINIO_STORAGE_CLASS_STANDARD or server config fields corresponding values are
+// returned.
+//
+// -- if input storage class is empty then standard is assumed
+// -- if input is RRS but RRS is not configured default '2' parity
+//    for RRS is assumed
+// -- if input is STANDARD but STANDARD is not configured '0' parity
+//    is returned, the caller is expected to choose the right parity
+//    at that point.
 func (sCfg Config) GetParityForSC(sc string) (parity int) {
 	switch strings.TrimSpace(sc) {
 	case RRS:
@@ -241,7 +246,7 @@ func (sCfg Config) GetParityForSC(sc string) (parity int) {
 
 // GetDMA - returns DMA configuration.
 func (sCfg Config) GetDMA() string {
-	return sCfg.DMA.DMA
+	return sCfg.DMA
 }
 
 // Enabled returns if etcd is enabled.
@@ -254,8 +259,6 @@ func Enabled(kvs config.KVS) bool {
 // LookupConfig - lookup storage class config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 	cfg = Config{}
-	cfg.Standard.Parity = setDriveCount / 2
-	cfg.RRS.Parity = defaultRRSParity
 
 	if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS); err != nil {
 		return Config{}, err
@@ -271,9 +274,6 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 			return Config{}, err
 		}
 	}
-	if cfg.Standard.Parity == 0 {
-		cfg.Standard.Parity = setDriveCount / 2
-	}
 
 	if rrsc != "" {
 		cfg.RRS, err = parseStorageClass(rrsc)
@@ -291,7 +291,7 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 	if dma != DMAReadWrite && dma != DMAWrite {
 		return Config{}, errors.New(`valid dma values are "read-write" and "write"`)
 	}
-	cfg.DMA.DMA = dma
+	cfg.DMA = dma
 
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.

cmd/crypto/json.go

@@ -0,0 +1,200 @@
+// MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"unicode/utf8"
+)
+
+// Adapted from Go stdlib.
+
+var hexTable = "0123456789abcdef"
+
+// EscapeStringJSON will escape a string for JSON and write it to dst.
+func EscapeStringJSON(dst *bytes.Buffer, s string) {
+	start := 0
+	for i := 0; i < len(s); {
+		if b := s[i]; b < utf8.RuneSelf {
+			if htmlSafeSet[b] {
+				i++
+				continue
+			}
+			if start < i {
+				dst.WriteString(s[start:i])
+			}
+			dst.WriteByte('\\')
+			switch b {
+			case '\\', '"':
+				dst.WriteByte(b)
+			case '\n':
+				dst.WriteByte('n')
+			case '\r':
+				dst.WriteByte('r')
+			case '\t':
+				dst.WriteByte('t')
+			default:
+				// This encodes bytes < 0x20 except for \t, \n and \r.
+				// If escapeHTML is set, it also escapes <, >, and &
+				// because they can lead to security holes when
+				// user-controlled strings are rendered into JSON
+				// and served to some browsers.
+				dst.WriteString(`u00`)
+				dst.WriteByte(hexTable[b>>4])
+				dst.WriteByte(hexTable[b&0xF])
+			}
+			i++
+			start = i
+			continue
+		}
+		c, size := utf8.DecodeRuneInString(s[i:])
+		if c == utf8.RuneError && size == 1 {
+			if start < i {
+				dst.WriteString(s[start:i])
+			}
+			dst.WriteString(`\ufffd`)
+			i += size
+			start = i
+			continue
+		}
+		// U+2028 is LINE SEPARATOR.
+		// U+2029 is PARAGRAPH SEPARATOR.
+		// They are both technically valid characters in JSON strings,
+		// but don't work in JSONP, which has to be evaluated as JavaScript,
+		// and can lead to security holes there. It is valid JSON to
+		// escape them, so we do so unconditionally.
+		// See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion.
+		if c == '\u2028' || c == '\u2029' {
+			if start < i {
+				dst.WriteString(s[start:i])
+			}
+			dst.WriteString(`\u202`)
+			dst.WriteByte(hexTable[c&0xF])
+			i += size
+			start = i
+			continue
+		}
+		i += size
+	}
+	if start < len(s) {
+		dst.WriteString(s[start:])
+	}
+}
+
+// htmlSafeSet holds the value true if the ASCII character with the given
+// array position can be safely represented inside a JSON string, embedded
+// inside of HTML <script> tags, without any additional escaping.
+//
+// All values are true except for the ASCII control characters (0-31), the
+// double quote ("), the backslash character ("\"), HTML opening and closing
+// tags ("<" and ">"), and the ampersand ("&").
+var htmlSafeSet = [utf8.RuneSelf]bool{
+	' ':      true,
+	'!':      true,
+	'"':      false,
+	'#':      true,
+	'$':      true,
+	'%':      true,
+	'&':      false,
+	'\'':     true,
+	'(':      true,
+	')':      true,
+	'*':      true,
+	'+':      true,
+	',':      true,
+	'-':      true,
+	'.':      true,
+	'/':      true,
+	'0':      true,
+	'1':      true,
+	'2':      true,
+	'3':      true,
+	'4':      true,
+	'5':      true,
+	'6':      true,
+	'7':      true,
+	'8':      true,
+	'9':      true,
+	':':      true,
+	';':      true,
+	'<':      false,
+	'=':      true,
+	'>':      false,
+	'?':      true,
+	'@':      true,
+	'A':      true,
+	'B':      true,
+	'C':      true,
+	'D':      true,
+	'E':      true,
+	'F':      true,
+	'G':      true,
+	'H':      true,
+	'I':      true,
+	'J':      true,
+	'K':      true,
+	'L':      true,
+	'M':      true,
+	'N':      true,
+	'O':      true,
+	'P':      true,
+	'Q':      true,
+	'R':      true,
+	'S':      true,
+	'T':      true,
+	'U':      true,
+	'V':      true,
+	'W':      true,
+	'X':      true,
+	'Y':      true,
+	'Z':      true,
+	'[':      true,
+	'\\':     false,
+	']':      true,
+	'^':      true,
+	'_':      true,
+	'`':      true,
+	'a':      true,
+	'b':      true,
+	'c':      true,
+	'd':      true,
+	'e':      true,
+	'f':      true,
+	'g':      true,
+	'h':      true,
+	'i':      true,
+	'j':      true,
+	'k':      true,
+	'l':      true,
+	'm':      true,
+	'n':      true,
+	'o':      true,
+	'p':      true,
+	'q':      true,
+	'r':      true,
+	's':      true,
+	't':      true,
+	'u':      true,
+	'v':      true,
+	'w':      true,
+	'x':      true,
+	'y':      true,
+	'z':      true,
+	'{':      true,
+	'|':      true,
+	'}':      true,
+	'~':      true,
+	'\u007f': true,
+}

cmd/crypto/kes.go

@@ -116,9 +116,9 @@ func NewKes(cfg KesConfig) (KMS, error) {
 	if err != nil {
 		return nil, err
 	}
-	if cfg.Transport.TLSClientConfig != nil {
-		if err = loadCACertificates(cfg.CAPath,
-			cfg.Transport.TLSClientConfig.RootCAs); err != nil {
+
+	if cfg.Transport.TLSClientConfig != nil && cfg.Transport.TLSClientConfig.RootCAs != nil {
+		if err = loadCACertificates(cfg.CAPath, cfg.Transport.TLSClientConfig.RootCAs); err != nil {
 			return nil, err
 		}
 	} else {
@@ -132,8 +132,12 @@ func NewKes(cfg KesConfig) (KMS, error) {
 		if err = loadCACertificates(cfg.CAPath, rootCAs); err != nil {
 			return nil, err
 		}
-		cfg.Transport.TLSClientConfig = &tls.Config{
-			RootCAs: rootCAs,
+		if cfg.Transport.TLSClientConfig == nil {
+			cfg.Transport.TLSClientConfig = &tls.Config{
+				RootCAs: rootCAs,
+			}
+		} else {
+			cfg.Transport.TLSClientConfig.RootCAs = rootCAs
 		}
 	}
 	cfg.Transport.TLSClientConfig.Certificates = []tls.Certificate{cert}

cmd/crypto/kms.go

@@ -39,6 +39,7 @@ type Context map[string]string
 //
 // WriteTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+// Sort order is based on the un-escaped keys.
 //
 // Note that neither keys nor values are escaped for JSON.
 func (c Context) WriteTo(w io.Writer) (n int64, err error) {
@@ -48,13 +49,19 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	}
 	sort.Sort(sortedKeys)
 
+	escape := func(s string) string {
+		buf := bytes.NewBuffer(make([]byte, 0, len(s)))
+		EscapeStringJSON(buf, s)
+		return buf.String()
+	}
+
 	nn, err := io.WriteString(w, "{")
 	if err != nil {
 		return n + int64(nn), err
 	}
 	n += int64(nn)
 	for i, k := range sortedKeys {
-		s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
+		s := fmt.Sprintf("\"%s\":\"%s\",", escape(k), escape(c[k]))
 		if i == len(sortedKeys)-1 {
 			s = s[:len(s)-1] // remove last ','
 		}
@@ -73,6 +80,7 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 //
 // AppendTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+// Sort order is based on the un-escaped keys.
 //
 // Note that neither keys nor values are escaped for JSON.
 func (c Context) AppendTo(dst []byte) (output []byte) {
@@ -87,9 +95,9 @@ func (c Context) AppendTo(dst []byte) (output []byte) {
 	if len(c) == 1 {
 		for k, v := range c {
 			out.WriteString(`{"`)
-			out.WriteString(k)
+			EscapeStringJSON(out, k)
 			out.WriteString(`":"`)
-			out.WriteString(v)
+			EscapeStringJSON(out, v)
 			out.WriteString(`"}`)
 		}
 		return out.Bytes()
@@ -104,9 +112,9 @@ func (c Context) AppendTo(dst []byte) (output []byte) {
 	out.WriteByte('{')
 	for i, k := range sortedKeys {
 		out.WriteByte('"')
-		out.WriteString(k)
+		EscapeStringJSON(out, k)
 		out.WriteString(`":"`)
-		out.WriteString(c[k])
+		EscapeStringJSON(out, c[k])
 		out.WriteByte('"')
 		if i < len(sortedKeys)-1 {
 			out.WriteByte(',')

cmd/crypto/kms_test.go

@@ -65,11 +65,15 @@ var contextWriteToTests = []struct {
 	Context      Context
 	ExpectedJSON string
 }{
-	{Context: Context{}, ExpectedJSON: "{}"},                                                    // 0
-	{Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},                                     // 1
-	{Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 2
-	{Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},                   // 3
-	{Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`}, // 4
+	0: {Context: Context{}, ExpectedJSON: "{}"},
+	1: {Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`},
+	2: {Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`},
+	3: {Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`},
+	4: {Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`},
+	// rfc 8259 escapes
+	5: {Context: Context{"0": "1", "key\\": "val\tue\r\n", "\"": "\""}, ExpectedJSON: `{"\"":"\"","0":"1","key\\":"val\tue\r\n"}`},
+	// html sensitive escapes
+	6: {Context: Context{"a": "<>&"}, ExpectedJSON: `{"a":"\u003c\u003e\u0026"}`},
 }
 
 func TestContextWriteTo(t *testing.T) {
@@ -101,7 +105,7 @@ func TestContextAppendTo(t *testing.T) {
 }
 
 func BenchmarkContext_AppendTo(b *testing.B) {
-	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}}
+	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%<>&"}}
 	for _, test := range tests {
 		b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) {
 			dst := make([]byte, 0, 1024)

cmd/data-crawler.go

@@ -182,6 +182,8 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		return cache, errors.New("internal error: root scan attempted")
 	}
 
+	skipHeal := cache.Info.SkipHealing
+
 	s := folderScanner{
 		root:                basePath,
 		getSize:             getSize,
@@ -244,7 +246,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		default:
 		}
 		var err error
-		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1, skipHeal)
 		if err != nil {
 			// No useful information...
 			return cache, err
@@ -262,7 +264,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 			return s.newCache, ctx.Err()
 		default:
 		}
-		du, err := s.deepScanFolder(ctx, folder)
+		du, err := s.deepScanFolder(ctx, folder, skipHeal)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -324,7 +326,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		}
 
 		// Update on this cycle...
-		du, err := s.deepScanFolder(ctx, folder)
+		du, err := s.deepScanFolder(ctx, folder, skipHeal)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -347,7 +349,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 // Files found in the folders will be added to f.newCache.
 // If final is provided folders will be put into f.newFolders or f.existingFolders.
 // If final is not provided the folders found are returned from the function.
-func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool, skipHeal bool) ([]cachedFolder, error) {
 	var nextFolders []cachedFolder
 	done := ctx.Done()
 	scannerLogPrefix := color.Green("folder-scanner:")
@@ -455,6 +457,11 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
 
+			// if the drive belongs to an erasure set
+			// that is already being healed, skip the
+			// healing attempt on this drive.
+			item.heal = item.heal && !skipHeal
+
 			sizeSummary, err := f.getSize(item)
 			if err == errSkipFile {
 				wait() // wait to proceed to next entry.
@@ -659,7 +666,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 }
 
 // deepScanFolder will deep scan a folder and return the size if no error occurs.
-func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder) (*dataUsageEntry, error) {
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder, skipHeal bool) (*dataUsageEntry, error) {
 	var cache dataUsageEntry
 
 	done := ctx.Done()
@@ -700,18 +707,23 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 			activeLifeCycle = f.oldCache.Info.lifeCycle
 		}
 
-		sizeSummary, err := f.getSize(
-			crawlItem{
-				Path:       fileName,
-				Typ:        typ,
-				bucket:     bucket,
-				prefix:     path.Dir(prefix),
-				objectName: path.Base(entName),
-				debug:      f.dataUsageCrawlDebug,
-				lifeCycle:  activeLifeCycle,
-				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
-			})
+		item := crawlItem{
+			Path:       fileName,
+			Typ:        typ,
+			bucket:     bucket,
+			prefix:     path.Dir(prefix),
+			objectName: path.Base(entName),
+			debug:      f.dataUsageCrawlDebug,
+			lifeCycle:  activeLifeCycle,
+			heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
+		}
 
+		// if the drive belongs to an erasure set
+		// that is already being healed, skip the
+		// healing attempt on this drive.
+		item.heal = item.heal && !skipHeal
+
+		sizeSummary, err := f.getSize(item)
 		if err == errSkipFile {
 			// Wait to throttle IO
 			wait()
@@ -863,8 +875,8 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 				logger.LogIf(ctx, err)
 				return size
 			}
-		case ObjectNotFound:
-			// object not found return 0
+		case ObjectNotFound, VersionNotFound:
+			// object not found or version not found return 0
 			return 0
 		default:
 			// All other errors proceed.
@@ -930,6 +942,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			opts.VersionID = obj.VersionID
 			opts.TransitionStatus = lifecycle.TransitionPending
 			if _, err = o.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
+				if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+					return 0
+				}
 				// Assume it is still there.
 				logger.LogIf(ctx, err)
 				return size
@@ -941,6 +956,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 
 	if obj.TransitionStatus != "" {
 		if err := deleteTransitionedObject(ctx, o, i.bucket, i.objectPath(), lcOpts, action, false); err != nil {
+			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+				return 0
+			}
 			logger.LogIf(ctx, err)
 			return size
 		}
@@ -950,6 +968,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 
 	obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
 	if err != nil {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			return 0
+		}
 		// Assume it is still there.
 		logger.LogIf(ctx, err)
 		return size

cmd/data-usage-cache.go

@@ -88,9 +88,12 @@ type dataUsageEntryInfo struct {
 
 type dataUsageCacheInfo struct {
 	// Name of the bucket. Also root element.
-	Name        string
-	LastUpdate  time.Time
-	NextCycle   uint32
+	Name       string
+	LastUpdate time.Time
+	NextCycle  uint32
+	// indicates if the disk is being healed and crawler
+	// should skip healing the disk
+	SkipHealing bool
 	BloomFilter []byte               `msg:"BloomFilter,omitempty"`
 	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
 }

cmd/data-usage-cache_gen.go

@@ -313,6 +313,12 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "SkipHealing":
+			z.SkipHealing, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "SkipHealing")
+				return
+			}
 		case "BloomFilter":
 			z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
 			if err != nil {
@@ -333,11 +339,11 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
-	zb0001Len := uint32(4)
-	var zb0001Mask uint8 /* 4 bits */
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
 	if z.BloomFilter == nil {
 		zb0001Len--
-		zb0001Mask |= 0x8
+		zb0001Mask |= 0x10
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -377,7 +383,17 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "NextCycle")
 		return
 	}
-	if (zb0001Mask & 0x8) == 0 { // if not empty
+	// write "SkipHealing"
+	err = en.Append(0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.SkipHealing)
+	if err != nil {
+		err = msgp.WrapError(err, "SkipHealing")
+		return
+	}
+	if (zb0001Mask & 0x10) == 0 { // if not empty
 		// write "BloomFilter"
 		err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
 		if err != nil {
@@ -396,11 +412,11 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// omitempty: check for empty values
-	zb0001Len := uint32(4)
-	var zb0001Mask uint8 /* 4 bits */
+	zb0001Len := uint32(5)
+	var zb0001Mask uint8 /* 5 bits */
 	if z.BloomFilter == nil {
 		zb0001Len--
-		zb0001Mask |= 0x8
+		zb0001Mask |= 0x10
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -416,7 +432,10 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "NextCycle"
 	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
 	o = msgp.AppendUint32(o, z.NextCycle)
-	if (zb0001Mask & 0x8) == 0 { // if not empty
+	// string "SkipHealing"
+	o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
+	o = msgp.AppendBool(o, z.SkipHealing)
+	if (zb0001Mask & 0x10) == 0 { // if not empty
 		// string "BloomFilter"
 		o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
 		o = msgp.AppendBytes(o, z.BloomFilter)
@@ -460,6 +479,12 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "SkipHealing":
+			z.SkipHealing, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "SkipHealing")
+				return
+			}
 		case "BloomFilter":
 			z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
 			if err != nil {
@@ -480,7 +505,7 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint32Size + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint32Size + 12 + msgp.BoolSize + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
 	return
 }
 

cmd/disk-cache-stats.go

@@ -34,6 +34,14 @@ type CacheDiskStats struct {
 	Dir          string
 }
 
+// GetUsageLevelString gets the string representation for the usage level.
+func (c *CacheDiskStats) GetUsageLevelString() (u string) {
+	if atomic.LoadInt32(&c.UsageState) == 0 {
+		return "low"
+	}
+	return "high"
+}
+
 // CacheStats - represents bytes served from cache,
 // cache hits and cache misses.
 type CacheStats struct {

cmd/dummy-handlers.go

@@ -32,7 +32,7 @@ import (
 func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketWebsite")
 
-	defer logger.AuditLog(w, r, "GetBucketWebsite", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -64,7 +64,7 @@ func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketAccelerate")
 
-	defer logger.AuditLog(w, r, "GetBucketAccelerate", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -97,7 +97,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketRequestPayment")
 
-	defer logger.AuditLog(w, r, "GetBucketRequestPayment", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -131,7 +131,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLogging")
 
-	defer logger.AuditLog(w, r, "GetBucketLogging", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -170,7 +170,7 @@ func (api objectAPIHandlers) DeleteBucketWebsiteHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketCors")
 
-	defer logger.AuditLog(w, r, "GetBucketCors", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

cmd/endpoint-ellipses.go

@@ -276,7 +276,16 @@ func parseEndpointSet(customSetDriveCount uint64, args ...string) (ep endpointSe
 // specific set size.
 // For example: {1...64} is divided into 4 sets each of size 16.
 // This applies to even distributed setup syntax as well.
-func GetAllSets(customSetDriveCount uint64, args ...string) ([][]string, error) {
+func GetAllSets(args ...string) ([][]string, error) {
+	var customSetDriveCount uint64
+	if v := env.Get(EnvErasureSetDriveCount, ""); v != "" {
+		driveCount, err := strconv.Atoi(v)
+		if err != nil {
+			return nil, config.ErrInvalidErasureSetSize(err)
+		}
+		customSetDriveCount = uint64(driveCount)
+	}
+
 	var setArgs [][]string
 	if !ellipses.HasEllipses(args...) {
 		var setIndexes [][]uint64
@@ -335,16 +344,8 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		return nil, -1, errInvalidArgument
 	}
 
-	var setDriveCount int
-	if v := env.Get(EnvErasureSetDriveCount, ""); v != "" {
-		setDriveCount, err = strconv.Atoi(v)
-		if err != nil {
-			return nil, -1, config.ErrInvalidErasureSetSize(err)
-		}
-	}
-
 	if !ellipses.HasEllipses(args...) {
-		setArgs, err := GetAllSets(uint64(setDriveCount), args...)
+		setArgs, err := GetAllSets(args...)
 		if err != nil {
 			return nil, -1, err
 		}
@@ -352,7 +353,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, err
 		}
-		endpointServerPools = append(endpointServerPools, ZoneEndpoints{
+		endpointServerPools = append(endpointServerPools, PoolEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
@@ -363,18 +364,16 @@ func createServerEndpoints(serverAddr string, args ...string) (
 
 	var foundPrevLocal bool
 	for _, arg := range args {
-		setArgs, err := GetAllSets(uint64(setDriveCount), arg)
+		setArgs, err := GetAllSets(arg)
 		if err != nil {
 			return nil, -1, err
 		}
+
 		endpointList, gotSetupType, err := CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
 		if err != nil {
 			return nil, -1, err
 		}
-		if setDriveCount != 0 && setDriveCount != len(setArgs[0]) {
-			return nil, -1, fmt.Errorf("All serverPools should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
-		}
-		if err = endpointServerPools.Add(ZoneEndpoints{
+		if err = endpointServerPools.Add(PoolEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
@@ -382,9 +381,6 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			return nil, -1, err
 		}
 		foundPrevLocal = endpointList.atleastOneEndpointLocal()
-		if setDriveCount == 0 {
-			setDriveCount = len(setArgs[0])
-		}
 		if setupType == UnknownSetupType {
 			setupType = gotSetupType
 		}

cmd/endpoint.go

@@ -196,20 +196,20 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}, nil
 }
 
-// ZoneEndpoints represent endpoints in a given pool
+// PoolEndpoints represent endpoints in a given pool
 // along with its setCount and setDriveCount.
-type ZoneEndpoints struct {
+type PoolEndpoints struct {
 	SetCount     int
 	DrivesPerSet int
 	Endpoints    Endpoints
 }
 
 // EndpointServerPools - list of list of endpoints
-type EndpointServerPools []ZoneEndpoints
+type EndpointServerPools []PoolEndpoints
 
-// GetLocalZoneIdx returns the pool which endpoint belongs to locally.
+// GetLocalPoolIdx returns the pool which endpoint belongs to locally.
 // if ep is remote this code will return -1 poolIndex
-func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
+func (l EndpointServerPools) GetLocalPoolIdx(ep Endpoint) int {
 	for i, zep := range l {
 		for _, cep := range zep.Endpoints {
 			if cep.IsLocal && ep.IsLocal {
@@ -223,7 +223,7 @@ func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
 }
 
 // Add add pool endpoints
-func (l *EndpointServerPools) Add(zeps ZoneEndpoints) error {
+func (l *EndpointServerPools) Add(zeps PoolEndpoints) error {
 	existSet := set.NewStringSet()
 	for _, zep := range *l {
 		for _, ep := range zep.Endpoints {

cmd/endpoint_test.go

@@ -357,7 +357,7 @@ func TestGetLocalPeer(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...)
+		zendpoints := mustGetPoolEndpoints(testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
 			if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
@@ -390,7 +390,7 @@ func TestGetRemotePeers(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...)
+		zendpoints := mustGetPoolEndpoints(testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
 			if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil {
 				t.Errorf("error: expected = <nil>, got = %v", err)

cmd/erasure-common.go

@@ -18,8 +18,6 @@ package cmd
 
 import (
 	"context"
-	"sort"
-	"strings"
 	"sync"
 
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -38,65 +36,6 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
-type sortSlices struct {
-	disks []StorageAPI
-	infos []DiskInfo
-}
-
-type sortByOther sortSlices
-
-func (sbo sortByOther) Len() int {
-	return len(sbo.disks)
-}
-
-func (sbo sortByOther) Swap(i, j int) {
-	sbo.disks[i], sbo.disks[j] = sbo.disks[j], sbo.disks[i]
-	sbo.infos[i], sbo.infos[j] = sbo.infos[j], sbo.infos[i]
-}
-
-func (sbo sortByOther) Less(i, j int) bool {
-	return sbo.infos[i].UsedInodes < sbo.infos[j].UsedInodes
-}
-
-func (er erasureObjects) getOnlineDisksSortedByUsedInodes() (newDisks []StorageAPI) {
-	disks := er.getDisks()
-	var wg sync.WaitGroup
-	var mu sync.Mutex
-	var infos []DiskInfo
-	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
-		i := i
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if disks[i-1] == nil {
-				return
-			}
-			di, err := disks[i-1].DiskInfo(context.Background())
-			if err != nil || di.Healing {
-
-				// - Do not consume disks which are not reachable
-				//   unformatted or simply not accessible for some reason.
-				//
-				// - Do not consume disks which are being healed
-				//
-				// - Future: skip busy disks
-				return
-			}
-
-			mu.Lock()
-			newDisks = append(newDisks, disks[i-1])
-			infos = append(infos, di)
-			mu.Unlock()
-		}()
-	}
-	wg.Wait()
-
-	slices := sortSlices{newDisks, infos}
-	sort.Sort(sortByOther(slices))
-
-	return slices.disks
-}
-
 func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	var wg sync.WaitGroup
@@ -204,32 +143,12 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 }
 
 // This function does the following check, suppose
-// object is "a/b/c/d", stat makes sure that objects ""a/b/c""
-// "a/b" and "a" do not exist.
+// object is "a/b/c/d", stat makes sure that objects
+// - "a/b/c"
+// - "a/b"
+// - "a"
+// do not exist on the namespace.
 func (er erasureObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
-	path := ""
-	segments := strings.Split(parent, slashSeparator)
-	for _, s := range segments {
-		if s == "" {
-			break
-		}
-		path += s
-		isObject, pathNotExist := er.isObject(ctx, bucket, path)
-		if pathNotExist {
-			return false
-		}
-		if isObject {
-			// If there is already a file at prefix "p", return true.
-			return true
-		}
-		path += slashSeparator
-	}
-	return false
-}
-
-// isObject - returns `true` if the prefix is an object i.e if
-// `xl.meta` exists at the leaf, false otherwise.
-func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (ok, pathDoesNotExist bool) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -241,7 +160,7 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o
 				return errDiskNotFound
 			}
 			// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
-			return storageDisks[index].CheckFile(ctx, bucket, prefix)
+			return storageDisks[index].CheckFile(ctx, bucket, parent)
 		}, index)
 	}
 
@@ -251,6 +170,5 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o
 	// ignored if necessary.
 	readQuorum := getReadQuorum(len(storageDisks))
 
-	err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum)
-	return err == nil, err == errPathNotFound
+	return reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum) == nil
 }

cmd/erasure-common_test.go

@@ -28,7 +28,7 @@ func TestErasureParentDirIsObject(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	obj, fsDisks, err := prepareErasure16(ctx)
+	obj, fsDisks, err := prepareErasureSets32(ctx)
 	if err != nil {
 		t.Fatalf("Unable to initialize 'Erasure' object layer.")
 	}
@@ -45,59 +45,38 @@ func TestErasureParentDirIsObject(t *testing.T) {
 	if err = obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
+
 	objectContent := "12345"
-	objInfo, err := obj.PutObject(GlobalContext, bucketName, objectName,
+	_, err = obj.PutObject(GlobalContext, bucketName, objectName,
 		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if objInfo.Name != objectName {
-		t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName)
-	}
 
-	z := obj.(*erasureServerPools)
-	xl := z.serverPools[0].sets[0]
 	testCases := []struct {
-		parentIsObject bool
-		objectName     string
+		expectedErr bool
+		objectName  string
 	}{
-		// parentIsObject is true if object is available.
 		{
-			parentIsObject: true,
-			objectName:     objectName,
+			expectedErr: true,
+			objectName:  pathJoin(objectName, "parent-is-object"),
 		},
 		{
-			parentIsObject: false,
-			objectName:     "new-dir",
-		},
-		{
-			parentIsObject: false,
-			objectName:     "",
-		},
-		{
-			parentIsObject: false,
-			objectName:     ".",
-		},
-		// Should not cause infinite loop.
-		{
-			parentIsObject: false,
-			objectName:     SlashSeparator,
-		},
-		{
-			parentIsObject: false,
-			objectName:     "\\",
-		},
-		// Should not cause infinite loop with double forward slash.
-		{
-			parentIsObject: false,
-			objectName:     "//",
+			expectedErr: false,
+			objectName:  pathJoin("no-parent", "object"),
 		},
 	}
 
-	for i, testCase := range testCases {
-		gotValue := xl.parentDirIsObject(GlobalContext, bucketName, testCase.objectName)
-		if testCase.parentIsObject != gotValue {
-			t.Errorf("Test %d: Unexpected value returned got %t, expected %t", i+1, gotValue, testCase.parentIsObject)
-		}
+	for _, testCase := range testCases {
+		t.Run("", func(t *testing.T) {
+			_, err = obj.PutObject(GlobalContext, bucketName, testCase.objectName,
+				mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
+			if testCase.expectedErr && err == nil {
+				t.Error("Expected error but got nil")
+			}
+			if !testCase.expectedErr && err != nil {
+				t.Errorf("Expected nil but got %v", err)
+			}
+		})
 	}
 }

cmd/erasure-decode.go

@@ -26,8 +26,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-var errHealRequired = errors.New("heal required")
-
 // Reads in parallel from readers.
 type parallelReader struct {
 	readers       []io.ReaderAt
@@ -125,7 +123,8 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		readTriggerCh <- true
 	}
 
-	healRequired := int32(0) // Atomic bool flag.
+	bitrotHeal := int32(0)       // Atomic bool flag.
+	missingPartsHeal := int32(0) // Atomic bool flag.
 	readerIndex := 0
 	var wg sync.WaitGroup
 	// if readTrigger is true, it implies next disk.ReadAt() should be tried
@@ -164,8 +163,10 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 			p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize]
 			_, err := rr.ReadAt(p.buf[bufIdx], p.offset)
 			if err != nil {
-				if _, ok := err.(*errHashMismatch); ok {
-					atomic.StoreInt32(&healRequired, 1)
+				if errors.Is(err, errFileNotFound) {
+					atomic.StoreInt32(&missingPartsHeal, 1)
+				} else if errors.Is(err, errFileCorrupt) {
+					atomic.StoreInt32(&bitrotHeal, 1)
 				}
 
 				// This will be communicated upstream.
@@ -188,8 +189,10 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 	wg.Wait()
 	if p.canDecode(newBuf) {
 		p.offset += p.shardSize
-		if healRequired != 0 {
-			return newBuf, errHealRequired
+		if atomic.LoadInt32(&missingPartsHeal) == 1 {
+			return newBuf, errFileNotFound
+		} else if atomic.LoadInt32(&bitrotHeal) == 1 {
+			return newBuf, errFileCorrupt
 		}
 		return newBuf, nil
 	}
@@ -197,41 +200,20 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 	return nil, reduceReadQuorumErrs(context.Background(), p.errs, objectOpIgnoredErrs, p.dataBlocks)
 }
 
-type errDecodeHealRequired struct {
-	err error
-}
-
-func (err *errDecodeHealRequired) Error() string {
-	return err.err.Error()
-}
-
-func (err *errDecodeHealRequired) Unwrap() error {
-	return err.err
-}
-
 // Decode reads from readers, reconstructs data if needed and writes the data to the writer.
 // A set of preferred drives can be supplied. In that case they will be used and the data reconstructed.
-func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64, prefer []bool) error {
-	healRequired, err := e.decode(ctx, writer, readers, offset, length, totalLength, prefer)
-	if healRequired {
-		return &errDecodeHealRequired{err}
-	}
-
-	return err
-}
-
-// Decode reads from readers, reconstructs data if needed and writes the data to the writer.
-func (e Erasure) decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64, prefer []bool) (bool, error) {
+func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64, prefer []bool) (written int64, derr error) {
 	if offset < 0 || length < 0 {
 		logger.LogIf(ctx, errInvalidArgument)
-		return false, errInvalidArgument
+		return -1, errInvalidArgument
 	}
 	if offset+length > totalLength {
 		logger.LogIf(ctx, errInvalidArgument)
-		return false, errInvalidArgument
+		return -1, errInvalidArgument
 	}
+
 	if length == 0 {
-		return false, nil
+		return 0, nil
 	}
 
 	reader := newParallelReader(readers, e, offset, totalLength)
@@ -242,7 +224,6 @@ func (e Erasure) decode(ctx context.Context, writer io.Writer, readers []io.Read
 	startBlock := offset / e.blockSize
 	endBlock := (offset + length) / e.blockSize
 
-	var healRequired bool
 	var bytesWritten int64
 	var bufs [][]byte
 	for block := startBlock; block <= endBlock; block++ {
@@ -264,32 +245,39 @@ func (e Erasure) decode(ctx context.Context, writer io.Writer, readers []io.Read
 		if blockLength == 0 {
 			break
 		}
+
 		var err error
 		bufs, err = reader.Read(bufs)
-		if err != nil {
-			if errors.Is(err, errHealRequired) {
-				// errHealRequired is only returned if there are be enough data for reconstruction.
-				healRequired = true
-			} else {
-				return healRequired, err
+		if len(bufs) > 0 {
+			// Set only if there are be enough data for reconstruction.
+			// and only for expected errors, also set once.
+			if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) {
+				if derr == nil {
+					derr = err
+				}
 			}
+		} else if err != nil {
+			// For all errors that cannot be reconstructed fail the read operation.
+			return -1, err
 		}
 
 		if err = e.DecodeDataBlocks(bufs); err != nil {
 			logger.LogIf(ctx, err)
-			return healRequired, err
+			return -1, err
 		}
 
 		n, err := writeDataBlocks(ctx, writer, bufs, e.dataBlocks, blockOffset, blockLength)
 		if err != nil {
-			return healRequired, err
+			return -1, err
 		}
+
 		bytesWritten += n
 	}
+
 	if bytesWritten != length {
 		logger.LogIf(ctx, errLessData)
-		return healRequired, errLessData
+		return bytesWritten, errLessData
 	}
 
-	return healRequired, nil
+	return bytesWritten, derr
 }

cmd/erasure-decode_test.go

@@ -138,7 +138,7 @@ func TestErasureDecode(t *testing.T) {
 		}
 
 		writer := bytes.NewBuffer(nil)
-		err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data, nil)
+		_, err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data, nil)
 		closeBitrotReaders(bitrotReaders)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
@@ -181,7 +181,7 @@ func TestErasureDecode(t *testing.T) {
 				bitrotReaders[0] = nil
 			}
 			writer.Reset()
-			err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data, nil)
+			_, err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data, nil)
 			closeBitrotReaders(bitrotReaders)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
@@ -272,7 +272,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 			tillOffset := erasure.ShardFileOffset(offset, readLen, length)
 			bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
-		err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length, nil)
+		_, err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length, nil)
 		closeBitrotReaders(bitrotReaders)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
@@ -334,7 +334,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 			tillOffset := erasure.ShardFileOffset(0, size, size)
 			bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
-		if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil {
+		if _, err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil {
 			panic(err)
 		}
 		closeBitrotReaders(bitrotReaders)

cmd/erasure-healing.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -42,7 +43,10 @@ func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts mad
 	storageEndpoints := er.getEndpoints()
 
 	// get write quorum for an object
-	writeQuorum := getWriteQuorum(len(storageDisks))
+	writeQuorum := len(storageDisks) - er.defaultParityCount
+	if writeQuorum == er.defaultParityCount {
+		writeQuorum++
+	}
 
 	// Heal bucket.
 	return healBucket(ctx, storageDisks, storageEndpoints, bucket, writeQuorum, opts)
@@ -98,16 +102,13 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 
 	errs := g.Wait()
 
-	reducedErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum-1)
-	if errors.Is(reducedErr, errVolumeNotFound) && !opts.Recreate {
-		return res, nil
-	}
-
 	// Initialize heal result info
 	res = madmin.HealResultItem{
-		Type:      madmin.HealItemBucket,
-		Bucket:    bucket,
-		DiskCount: len(storageDisks),
+		Type:         madmin.HealItemBucket,
+		Bucket:       bucket,
+		DiskCount:    len(storageDisks),
+		ParityBlocks: len(storageDisks) / 2,
+		DataBlocks:   len(storageDisks) / 2,
 	}
 
 	for i := range beforeState {
@@ -118,6 +119,18 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 		})
 	}
 
+	reducedErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum-1)
+	if errors.Is(reducedErr, errVolumeNotFound) && !opts.Recreate {
+		for i := range beforeState {
+			res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
+				UUID:     "",
+				Endpoint: storageEndpoints[i],
+				State:    madmin.DriveStateOk,
+			})
+		}
+		return res, nil
+	}
+
 	// Initialize sync waitgroup.
 	g = errgroup.WithNErrs(len(storageDisks))
 
@@ -223,8 +236,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	dryRun := opts.DryRun
 	scanMode := opts.ScanMode
 
-	dataBlocks := lfi.Erasure.DataBlocks
-
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
@@ -241,8 +252,8 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		Bucket:       bucket,
 		Object:       object,
 		DiskCount:    len(storageDisks),
-		ParityBlocks: len(storageDisks) / 2,
-		DataBlocks:   len(storageDisks) / 2,
+		ParityBlocks: er.defaultParityCount,
+		DataBlocks:   len(storageDisks) - er.defaultParityCount,
 	}
 
 	// Loop to find number of disks with valid data, per-drive
@@ -308,12 +319,12 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
 		}
 		// File is fully gone, fileInfo is empty.
-		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object, versionID), err
+		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
 	}
 
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
-	if numAvailableDisks < dataBlocks {
+	if numAvailableDisks < result.DataBlocks {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, dataErrs, opts)
 	}
 
@@ -330,7 +341,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, dataBlocks)
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, result.DataBlocks)
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, versionID)
 	}
@@ -360,7 +371,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		dataDir = migrateDataDir
 	}
 
-	if !latestMeta.Deleted {
+	if !latestMeta.Deleted || latestMeta.TransitionStatus != lifecycle.TransitionComplete {
 		result.DataBlocks = latestMeta.Erasure.DataBlocks
 		result.ParityBlocks = latestMeta.Erasure.ParityBlocks
 
@@ -490,8 +501,8 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 		Bucket:       bucket,
 		Object:       object,
 		DiskCount:    len(storageDisks),
-		ParityBlocks: getDefaultParityBlocks(len(storageDisks)),
-		DataBlocks:   getDefaultDataBlocks(len(storageDisks)),
+		ParityBlocks: er.defaultParityCount,
+		DataBlocks:   len(storageDisks) - er.defaultParityCount,
 		ObjectSize:   0,
 	}
 
@@ -562,7 +573,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 
 // Populates default heal result item entries with possible values when we are returning prematurely.
 // This is to ensure that in any circumstance we are not returning empty arrays with wrong values.
-func defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []string, errs []error, bucket, object, versionID string) madmin.HealResultItem {
+func defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []string, errs []error, bucket, object, versionID string, defaultParityCount int) madmin.HealResultItem {
 	// Initialize heal result object
 	result := madmin.HealResultItem{
 		Type:      madmin.HealItemObject,
@@ -608,8 +619,8 @@ func defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints
 
 	if !lfi.IsValid() {
 		// Default to most common configuration for erasure blocks.
-		result.ParityBlocks = getDefaultParityBlocks(len(storageDisks))
-		result.DataBlocks = getDefaultDataBlocks(len(storageDisks))
+		result.ParityBlocks = defaultParityCount
+		result.DataBlocks = len(storageDisks) - defaultParityCount
 	} else {
 		result.ParityBlocks = lfi.Erasure.ParityBlocks
 		result.DataBlocks = lfi.Erasure.DataBlocks
@@ -690,7 +701,7 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 	if ok {
 		writeQuorum := m.Erasure.DataBlocks
 		if m.Erasure.DataBlocks == 0 || m.Erasure.DataBlocks == m.Erasure.ParityBlocks {
-			writeQuorum = getWriteQuorum(len(storageDisks))
+			writeQuorum++
 		}
 		var err error
 		var returnNotFound bool
@@ -722,18 +733,15 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 			if versionID != "" {
 				err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
 			}
-			return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID), err
+			return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
 		}
-		return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID)
+		return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), toObjectErr(err, bucket, object, versionID)
 	}
 
-	readQuorum := m.Erasure.DataBlocks
-	if m.Erasure.DataBlocks == 0 || m.Erasure.DataBlocks == m.Erasure.ParityBlocks {
-		readQuorum = getReadQuorum(len(storageDisks))
-	}
+	readQuorum := len(storageDisks) - er.defaultParityCount
 
 	err := toObjectErr(reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum), bucket, object, versionID)
-	return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID), err
+	return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
 }
 
 // Object is considered dangling/corrupted if any only
@@ -775,8 +783,10 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 		break
 	}
 
-	if validMeta.Deleted {
-		// notFoundParts is ignored since a delete marker does not have any parts
+	if validMeta.Deleted || validMeta.TransitionStatus == lifecycle.TransitionComplete {
+		// notFoundParts is ignored since a
+		// - delete marker does not have any parts
+		// - transition status of complete has no parts
 		return validMeta, corruptedErasureMeta+notFoundErasureMeta > len(errs)/2
 	}
 
@@ -819,7 +829,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
 		}
 		// Nothing to do, file is already gone.
-		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object, versionID), err
+		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
 	}
 
 	fi, err := getLatestFileInfo(healCtx, partsMetadata, errs)

cmd/erasure-healing_test.go

@@ -160,7 +160,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	objLayer, _, err := initObjectLayer(ctx, mustGetZoneEndpoints(fsDirs...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -277,8 +277,8 @@ func TestHealObjectCorrupted(t *testing.T) {
 
 	// Test 4: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
-	for i := 0; i <= len(er.getDisks())/2; i++ {
-		er.getDisks()[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -310,7 +310,7 @@ func TestHealObjectErasure(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	obj, _, err := initObjectLayer(ctx, mustGetZoneEndpoints(fsDirs...))
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -400,7 +400,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	obj, _, err := initObjectLayer(ctx, mustGetZoneEndpoints(fsDirs...))
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/erasure-lowlevel-heal.go

@@ -28,7 +28,7 @@ import (
 func (e Erasure) Heal(ctx context.Context, readers []io.ReaderAt, writers []io.Writer, size int64) error {
 	r, w := io.Pipe()
 	go func() {
-		if err := e.Decode(ctx, w, readers, 0, size, size, nil); err != nil {
+		if _, err := e.Decode(ctx, w, readers, 0, size, size, nil); err != nil {
 			w.CloseWithError(err)
 			return
 		}

cmd/erasure-metadata-utils.go

@@ -128,7 +128,12 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 			}
 			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, readData)
 			if err != nil {
-				if !IsErr(err, errFileNotFound, errVolumeNotFound, errFileVersionNotFound, errDiskNotFound) {
+				if !IsErr(err, []error{
+					errFileNotFound,
+					errVolumeNotFound,
+					errFileVersionNotFound,
+					errDiskNotFound,
+				}...) {
 					logger.LogOnceIf(ctx, err, disks[index].String())
 				}
 			}

cmd/erasure-metadata-utils_test.go

@@ -142,7 +142,7 @@ func TestShuffleDisks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	objLayer, _, err := initObjectLayer(ctx, mustGetZoneEndpoints(disks...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(disks...))
 	if err != nil {
 		removeRoots(disks)
 		t.Fatal(err)
@@ -190,7 +190,7 @@ func TestEvalDisks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	objLayer, _, err := initObjectLayer(ctx, mustGetZoneEndpoints(disks...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(disks...))
 	if err != nil {
 		removeRoots(disks)
 		t.Fatal(err)

cmd/erasure-metadata.go

@@ -334,7 +334,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 // Returns per object readQuorum and writeQuorum
 // readQuorum is the min required disks to read data.
 // writeQuorum is the min required disks to write data.
-func objectQuorumFromMeta(ctx context.Context, er erasureObjects, partsMetaData []FileInfo, errs []error) (objectReadQuorum, objectWriteQuorum int, err error) {
+func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
 	// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
 	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs)
 	if err != nil {
@@ -343,13 +343,13 @@ func objectQuorumFromMeta(ctx context.Context, er erasureObjects, partsMetaData
 
 	dataBlocks := latestFileInfo.Erasure.DataBlocks
 	parityBlocks := globalStorageClass.GetParityForSC(latestFileInfo.Metadata[xhttp.AmzStorageClass])
-	if parityBlocks == 0 {
-		parityBlocks = dataBlocks
+	if parityBlocks <= 0 {
+		parityBlocks = defaultParityCount
 	}
 
 	writeQuorum := dataBlocks
 	if dataBlocks == parityBlocks {
-		writeQuorum = dataBlocks + 1
+		writeQuorum++
 	}
 
 	// Since all the valid erasure code meta updated at the same time are equivalent, pass dataBlocks

cmd/erasure-multipart.go

@@ -48,7 +48,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	// Read metadata associated with the object from all disks.
 	metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "", false)
 
-	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		return err
 	}
@@ -242,18 +242,18 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 
 	onlineDisks := er.getDisks()
 	parityBlocks := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
-	if parityBlocks == 0 {
-		parityBlocks = len(onlineDisks) / 2
+	if parityBlocks <= 0 {
+		parityBlocks = er.defaultParityCount
 	}
-	dataBlocks := len(onlineDisks) - parityBlocks
 
+	dataBlocks := len(onlineDisks) - parityBlocks
 	fi := newFileInfo(object, dataBlocks, parityBlocks)
 
 	// we now know the number of blocks this object needs for data and parity.
 	// establish the writeQuorum using this data
 	writeQuorum := dataBlocks
 	if dataBlocks == parityBlocks {
-		writeQuorum = dataBlocks + 1
+		writeQuorum++
 	}
 
 	if opts.UserDefined["content-type"] == "" {
@@ -374,7 +374,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		uploadIDPath, "", false)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
 		return pi, toObjectErr(err, bucket, object)
 	}
@@ -555,7 +555,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID, false)
 
 	// get Quorum for this object
-	readQuorum, _, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
+	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
 		return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
@@ -603,7 +603,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
 		return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
@@ -690,7 +690,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	// Check if an object is present as one of the parent dir.
 	// -- FIXME. (needs a new kind of lock).
-	if er.parentDirIsObject(ctx, bucket, path.Dir(object)) {
+	if opts.ParentIsObject != nil && opts.ParentIsObject(ctx, bucket, path.Dir(object)) {
 		return oi, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
@@ -707,7 +707,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
@@ -892,7 +892,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 	partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "", false)
 
 	// get Quorum for this object
-	_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
 		return toObjectErr(err, bucket, object, uploadID)
 	}

cmd/erasure-object.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
+	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -51,6 +52,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	}
 
 	defer ObjectPathUpdated(pathJoin(dstBucket, dstObject))
+
 	lk := er.NewNSLock(dstBucket, dstObject)
 	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
@@ -62,7 +64,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, false)
 
 	// get Quorum for this object
-	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
@@ -99,7 +101,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		modTime = dstOpts.MTime
 		fi.ModTime = dstOpts.MTime
 	}
-
+	fi.Metadata = srcInfo.UserDefined
 	srcInfo.UserDefined["etag"] = srcInfo.ETag
 
 	// Update `xl.meta` content on each disks.
@@ -314,17 +316,29 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 			// Prefer local disks
 			prefer[index] = disk.Hostname() == ""
 		}
-		err = erasure.Decode(ctx, writer, readers, partOffset, partLength, partSize, prefer)
+
+		written, err := erasure.Decode(ctx, writer, readers, partOffset, partLength, partSize, prefer)
 		// Note: we should not be defer'ing the following closeBitrotReaders() call as
 		// we are inside a for loop i.e if we use defer, we would accumulate a lot of open files by the time
 		// we return from this function.
 		closeBitrotReaders(readers)
 		if err != nil {
-			if decodeHealErr, ok := err.(*errDecodeHealRequired); ok {
-				healOnce.Do(func() {
-					go deepHealObject(bucket, object, fi.VersionID)
-				})
-				err = decodeHealErr.err
+			// If we have successfully written all the content that was asked
+			// by the client, but we still see an error - this would mean
+			// that we have some parts or data blocks missing or corrupted
+			// - attempt a heal to successfully heal them for future calls.
+			if written == partLength {
+				var scan madmin.HealScanMode
+				if errors.Is(err, errFileNotFound) {
+					scan = madmin.HealNormalScan
+				} else if errors.Is(err, errFileCorrupt) {
+					scan = madmin.HealDeepScan
+				}
+				if scan != madmin.HealUnknownScan {
+					healOnce.Do(func() {
+						go healObject(bucket, object, fi.VersionID, scan)
+					})
+				}
 			}
 			if err != nil {
 				return toObjectErr(err, bucket, object)
@@ -380,7 +394,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	// Read metadata associated with the object from all disks.
 	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)
 
-	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		return fi, nil, nil, err
 	}
@@ -415,6 +429,24 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	if err != nil {
 		return fi, nil, nil, err
 	}
+
+	var missingBlocks int
+	for i, err := range errs {
+		if err != nil && errors.Is(err, errFileNotFound) {
+			missingBlocks++
+			continue
+		}
+		if metaArr[i].IsValid() && metaArr[i].ModTime.Equal(fi.ModTime) {
+			continue
+		}
+		missingBlocks++
+	}
+
+	// if missing metadata can be reconstructed, attempt to reconstruct.
+	if missingBlocks > 0 && missingBlocks < readQuorum {
+		go healObject(bucket, object, fi.VersionID, madmin.HealNormalScan)
+	}
+
 	return fi, metaArr, onlineDisks, nil
 }
 
@@ -580,7 +612,9 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	defer ObjectPathUpdated(pathJoin(bucket, object))
+	defer func() {
+		ObjectPathUpdated(pathJoin(bucket, object))
+	}()
 
 	data := r.Reader
 
@@ -595,8 +629,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// Get parity and data drive count based on storage class metadata
 	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
-	if parityDrives == 0 {
-		parityDrives = getDefaultParityBlocks(len(storageDisks))
+	if parityDrives <= 0 {
+		parityDrives = er.defaultParityCount
 	}
 	dataDrives := len(storageDisks) - parityDrives
 
@@ -604,7 +638,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// writeQuorum is dataBlocks + 1
 	writeQuorum := dataDrives
 	if dataDrives == parityDrives {
-		writeQuorum = dataDrives + 1
+		writeQuorum++
 	}
 
 	// Delete temporary object in the event of failure.
@@ -621,7 +655,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// Check if an object is present as one of the parent dir.
 	// -- FIXME. (needs a new kind of lock).
 	// -- FIXME (this also causes performance issue when disks are down).
-	if er.parentDirIsObject(ctx, bucket, path.Dir(object)) {
+	if opts.ParentIsObject != nil && opts.ParentIsObject(ctx, bucket, path.Dir(object)) {
 		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
@@ -783,6 +817,31 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
 }
 
+// deleteEmptyDir knows only how to remove an empty directory (not the empty object with a
+// trailing slash), this is called for the healing code to remove such directories.
+func (er erasureObjects) deleteEmptyDir(ctx context.Context, bucket, object string) error {
+	defer ObjectPathUpdated(pathJoin(bucket, object))
+
+	if bucket == minioMetaTmpBucket {
+		return nil
+	}
+
+	disks := er.getDisks()
+	g := errgroup.WithNErrs(len(disks))
+	for index := range disks {
+		index := index
+		g.Go(func() error {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			return disks[index].Delete(ctx, bucket, object, false)
+		}, index)
+	}
+
+	// return errors if any during deletion
+	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, len(disks)/2+1)
+}
+
 // deleteObject - wrapper for delete object, deletes an object from
 // all the disks in parallel, including `xl.meta` associated with the
 // object.
@@ -1093,7 +1152,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	// Read metadata associated with the object from all disks.
 	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
 
-	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
 	}
@@ -1154,7 +1213,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	// Read metadata associated with the object from all disks.
 	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
 
-	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
 	}

cmd/erasure-object_test.go

@@ -196,6 +196,13 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 }
 
 func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
+	restoreGlobalStorageClass := globalStorageClass
+	defer func() {
+		globalStorageClass = restoreGlobalStorageClass
+	}()
+
+	globalStorageClass = storageclass.Config{}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -230,7 +237,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	erasureDisks := xl.getDisks()
 	z.serverPools[0].erasureDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
-		for i := range erasureDisks[:7] {
+		for i := range erasureDisks[:4] {
 			erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk)
 		}
 		return erasureDisks
@@ -501,6 +508,8 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		globalStorageClass = restoreGlobalStorageClass
 	}()
 
+	globalStorageClass = storageclass.Config{}
+
 	bucket := getRandomBucketName()
 
 	var opts ObjectOptions
@@ -528,7 +537,6 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	}
 
 	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "", false)
-
 	parts1SC := globalStorageClass
 
 	// Object for test case 2 - No StorageClass defined, MetaData in PutObject requesting RRS Class
@@ -626,7 +634,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	// Reset global storage class flags
 	object7 := "object7"
 	metadata7 := make(map[string]string)
-	metadata7["x-amz-storage-class"] = storageclass.RRS
+	metadata7["x-amz-storage-class"] = storageclass.STANDARD
 	globalStorageClass = storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 5,
@@ -653,19 +661,19 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		storageClassCfg     storageclass.Config
 		expectedError       error
 	}{
-		{parts1, errs1, 8, 9, parts1SC, nil},
+		{parts1, errs1, 12, 12, parts1SC, nil},
 		{parts2, errs2, 14, 14, parts2SC, nil},
-		{parts3, errs3, 8, 9, parts3SC, nil},
+		{parts3, errs3, 12, 12, parts3SC, nil},
 		{parts4, errs4, 10, 10, parts4SC, nil},
 		{parts5, errs5, 14, 14, parts5SC, nil},
-		{parts6, errs6, 8, 9, parts6SC, nil},
-		{parts7, errs7, 14, 14, parts7SC, nil},
+		{parts6, errs6, 12, 12, parts6SC, nil},
+		{parts7, errs7, 11, 11, parts7SC, nil},
 	}
 	for _, tt := range tests {
 		tt := tt
 		t.(*testing.T).Run("", func(t *testing.T) {
 			globalStorageClass = tt.storageClassCfg
-			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, *xl, tt.parts, tt.errs)
+			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, getDefaultParityBlocks(len(erasureDisks)))
 			if tt.expectedError != nil && err == nil {
 				t.Errorf("Expected %s, got %s", tt.expectedError, err)
 			}

cmd/erasure-server-pool.go

@@ -34,7 +34,6 @@ import (
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -48,15 +47,17 @@ type erasureServerPools struct {
 	shutdown context.CancelFunc
 }
 
-func (z *erasureServerPools) SingleZone() bool {
+func (z *erasureServerPools) SinglePool() bool {
 	return len(z.serverPools) == 1
 }
 
 // Initialize new pool of erasure sets.
 func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) {
 	var (
-		deploymentID string
-		err          error
+		deploymentID       string
+		distributionAlgo   string
+		commonParityDrives int
+		err                error
 
 		formats      = make([]*formatErasureV3, len(endpointServerPools))
 		storageDisks = make([][]StorageAPI, len(endpointServerPools))
@@ -72,18 +73,50 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 				localDrives = append(localDrives, endpoint.Path)
 			}
 		}
+
+		// If storage class is not set during startup, default values are used
+		// -- Default for Reduced Redundancy Storage class is, parity = 2
+		// -- Default for Standard Storage class is, parity = 2 - disks 4, 5
+		// -- Default for Standard Storage class is, parity = 3 - disks 6, 7
+		// -- Default for Standard Storage class is, parity = 4 - disks 8 to 16
+		if commonParityDrives == 0 {
+			commonParityDrives = ecDrivesNoConfig(ep.DrivesPerSet)
+		}
+
+		if err = storageclass.ValidateParity(commonParityDrives, ep.DrivesPerSet); err != nil {
+			return nil, fmt.Errorf("All current serverPools should have same parity ratio - expected %d, got %d", commonParityDrives, ecDrivesNoConfig(ep.DrivesPerSet))
+		}
+
 		storageDisks[i], formats[i], err = waitForFormatErasure(local, ep.Endpoints, i+1,
-			ep.SetCount, ep.DrivesPerSet, deploymentID)
+			ep.SetCount, ep.DrivesPerSet, deploymentID, distributionAlgo)
 		if err != nil {
 			return nil, err
 		}
+
 		if deploymentID == "" {
+			// all zones should have same deployment ID
 			deploymentID = formats[i].ID
 		}
-		z.serverPools[i], err = newErasureSets(ctx, ep.Endpoints, storageDisks[i], formats[i])
+
+		if distributionAlgo == "" {
+			distributionAlgo = formats[i].Erasure.DistributionAlgo
+		}
+
+		// Validate if users brought different DeploymentID pools.
+		if deploymentID != formats[i].ID {
+			return nil, fmt.Errorf("All serverPools should have same deployment ID expected %s, got %s", deploymentID, formats[i].ID)
+		}
+
+		// Validate if users brought different different distribution algo pools.
+		if distributionAlgo != formats[i].Erasure.DistributionAlgo {
+			return nil, fmt.Errorf("All serverPools should have same distributionAlgo expected %s, got %s", distributionAlgo, formats[i].Erasure.DistributionAlgo)
+		}
+
+		z.serverPools[i], err = newErasureSets(ctx, ep.Endpoints, storageDisks[i], formats[i], commonParityDrives)
 		if err != nil {
 			return nil, err
 		}
+		z.serverPools[i].poolNumber = i
 	}
 	ctx, z.shutdown = context.WithCancel(ctx)
 	go intDataUpdateTracker.start(ctx, localDrives...)
@@ -120,12 +153,12 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 	return res
 }
 
-func (z *erasureServerPools) GetAllLockers() []dsync.NetLocker {
-	return z.serverPools[0].GetAllLockers()
-}
-
-func (z *erasureServerPools) SetDriveCount() int {
-	return z.serverPools[0].SetDriveCount()
+func (z *erasureServerPools) SetDriveCounts() []int {
+	setDriveCounts := make([]int, len(z.serverPools))
+	for i := range z.serverPools {
+		setDriveCounts[i] = z.serverPools[i].SetDriveCount()
+	}
+	return setDriveCounts
 }
 
 type serverPoolsAvailableSpace []poolAvailableSpace
@@ -144,9 +177,9 @@ func (p serverPoolsAvailableSpace) TotalAvailable() uint64 {
 	return total
 }
 
-// getAvailableZoneIdx will return an index that can hold size bytes.
+// getAvailablePoolIdx will return an index that can hold size bytes.
 // -1 is returned if no serverPools have available space for the size given.
-func (z *erasureServerPools) getAvailableZoneIdx(ctx context.Context, size int64) int {
+func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, size int64) int {
 	serverPools := z.getServerPoolsAvailableSpace(ctx, size)
 	total := serverPools.TotalAvailable()
 	if total == 0 {
@@ -217,10 +250,10 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
 	return serverPools
 }
 
-// getZoneIdx returns the found previous object and its corresponding pool idx,
+// getPoolIdx returns the found previous object and its corresponding pool idx,
 // if none are found falls back to most available space pool.
-func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object string, opts ObjectOptions, size int64) (idx int, err error) {
-	if z.SingleZone() {
+func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions, size int64) (idx int, err error) {
+	if z.SinglePool() {
 		return 0, nil
 	}
 	for i, pool := range z.serverPools {
@@ -246,7 +279,7 @@ func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object stri
 	}
 
 	// We multiply the size by 2 to account for erasure coding.
-	idx = z.getAvailableZoneIdx(ctx, size*2)
+	idx = z.getAvailablePoolIdx(ctx, size*2)
 	if idx < 0 {
 		return -1, toObjectErr(errDiskFull)
 	}
@@ -278,14 +311,18 @@ func (z *erasureServerPools) BackendInfo() (b BackendInfo) {
 	b.Type = BackendErasure
 
 	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-	if scParity == 0 {
-		scParity = z.SetDriveCount() / 2
+	if scParity <= 0 {
+		scParity = z.serverPools[0].defaultParityCount
 	}
-	b.StandardSCData = z.SetDriveCount() - scParity
-	b.StandardSCParity = scParity
-
 	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
-	b.RRSCData = z.SetDriveCount() - rrSCParity
+
+	// Data blocks can vary per pool, but parity is same.
+	for _, setDriveCount := range z.SetDriveCounts() {
+		b.StandardSCData = append(b.StandardSCData, setDriveCount-scParity)
+		b.RRSCData = append(b.RRSCData, setDriveCount-rrSCParity)
+	}
+
+	b.StandardSCParity = scParity
 	b.RRSCParity = rrSCParity
 	return
 }
@@ -554,11 +591,11 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 
 	object = encodeDirObject(object)
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
 
-	idx, err := z.getZoneIdx(ctx, bucket, object, opts, data.Size())
+	idx, err := z.getPoolIdx(ctx, bucket, object, opts, data.Size())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -574,7 +611,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 
 	object = encodeDirObject(object)
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
 	for _, pool := range z.serverPools {
@@ -611,7 +648,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 	}
 	defer multiDeleteLock.Unlock()
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
 	}
 
@@ -633,7 +670,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
-	poolIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
+	poolIdx, err := z.getPoolIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
@@ -786,7 +823,7 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
 		return ListMultipartsInfo{}, err
 	}
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	}
 
@@ -812,12 +849,12 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 		return "", err
 	}
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
 
 	// We don't know the exact size, so we ask for at least 1GiB file.
-	idx, err := z.getZoneIdx(ctx, bucket, object, opts, 1<<30)
+	idx, err := z.getPoolIdx(ctx, bucket, object, opts, 1<<30)
 	if err != nil {
 		return "", err
 	}
@@ -841,7 +878,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		return PartInfo{}, err
 	}
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
@@ -871,7 +908,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		return MultipartInfo{}, err
 	}
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
 	for _, pool := range z.serverPools {
@@ -901,7 +938,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		return ListPartsInfo{}, err
 	}
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
 	for _, pool := range z.serverPools {
@@ -928,7 +965,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		return err
 	}
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
 
@@ -957,7 +994,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		return objInfo, err
 	}
 
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
 
@@ -984,7 +1021,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 
 // GetBucketInfo - returns bucket info from one of the erasure coded serverPools.
 func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
-	if z.SingleZone() {
+	if z.SinglePool() {
 		bucketInfo, err = z.serverPools[0].GetBucketInfo(ctx, bucket)
 		if err != nil {
 			return bucketInfo, err
@@ -1042,7 +1079,7 @@ func (z *erasureServerPools) IsTaggingSupported() bool {
 // even if one of the serverPools fail to delete buckets, we proceed to
 // undo a successful operation.
 func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].DeleteBucket(ctx, bucket, forceDelete)
 	}
 	g := errgroup.WithNErrs(len(z.serverPools))
@@ -1118,7 +1155,7 @@ func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all serverPools.
 func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
-	if z.SingleZone() {
+	if z.SinglePool() {
 		buckets, err = z.serverPools[0].ListBuckets(ctx)
 	} else {
 		for _, pool := range z.serverPools {
@@ -1312,11 +1349,19 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 					break
 				}
 
+				// Remove empty directories if found - they have no meaning.
+				// Can be left over from highly concurrent put/remove.
+				if quorumCount > set.setDriveCount/2 && entry.IsEmptyDir {
+					if !opts.DryRun && opts.Remove {
+						set.deleteEmptyDir(ctx, bucket, entry.Name)
+					}
+				}
+
 				// Indicate that first attempt was a success and subsequent loop
 				// knows that its not our first attempt at 'prefix'
 				err = nil
 
-				if quorumCount == z.SetDriveCount() && opts.ScanMode == madmin.HealNormalScan {
+				if quorumCount == set.setDriveCount && opts.ScanMode == madmin.HealNormalScan {
 					continue
 				}
 
@@ -1353,6 +1398,7 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 
 	for _, pool := range z.serverPools {
 		result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
+		result.Object = decodeDirObject(result.Object)
 		if err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
@@ -1375,12 +1421,12 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
 }
 
 // GetMetrics - no op
-func (z *erasureServerPools) GetMetrics(ctx context.Context) (*Metrics, error) {
+func (z *erasureServerPools) GetMetrics(ctx context.Context) (*BackendMetrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return &Metrics{}, NotImplemented{}
+	return &BackendMetrics{}, NotImplemented{}
 }
 
-func (z *erasureServerPools) getZoneAndSet(id string) (int, int, error) {
+func (z *erasureServerPools) getPoolAndSet(id string) (int, int, error) {
 	for poolIdx := range z.serverPools {
 		format := z.serverPools[poolIdx].format
 		for setIdx, set := range format.Erasure.Sets {
@@ -1405,7 +1451,7 @@ type HealthOptions struct {
 type HealthResult struct {
 	Healthy       bool
 	HealingDrives int
-	ZoneID, SetID int
+	PoolID, SetID int
 	WriteQuorum   int
 }
 
@@ -1426,7 +1472,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 
 	for _, localDiskIDs := range diskIDs {
 		for _, id := range localDiskIDs {
-			poolIdx, setIdx, err := z.getZoneAndSet(id)
+			poolIdx, setIdx, err := z.getPoolAndSet(id)
 			if err != nil {
 				logger.LogIf(ctx, err)
 				continue
@@ -1437,15 +1483,9 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 
 	reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance))
 
-	parityDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-	diskCount := z.SetDriveCount()
-
-	if parityDrives == 0 {
-		parityDrives = getDefaultParityBlocks(diskCount)
-	}
-	dataDrives := diskCount - parityDrives
-	writeQuorum := dataDrives
-	if dataDrives == parityDrives {
+	b := z.BackendInfo()
+	writeQuorum := b.StandardSCData[0]
+	if writeQuorum == b.StandardSCParity {
 		writeQuorum++
 	}
 
@@ -1477,7 +1517,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 				return HealthResult{
 					Healthy:       false,
 					HealingDrives: len(aggHealStateResult.HealDisks),
-					ZoneID:        poolIdx,
+					PoolID:        poolIdx,
 					SetID:         setIdx,
 					WriteQuorum:   writeQuorum,
 				}
@@ -1504,7 +1544,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 // PutObjectTags - replace or add tags to an existing object
 func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) error {
 	object = encodeDirObject(object)
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}
 
@@ -1534,7 +1574,7 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 // DeleteObjectTags - delete object tags from an existing object
 func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) error {
 	object = encodeDirObject(object)
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
 	for _, pool := range z.serverPools {
@@ -1563,7 +1603,7 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 // GetObjectTags - get object tags from an existing object
 func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) {
 	object = encodeDirObject(object)
-	if z.SingleZone() {
+	if z.SinglePool() {
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
 	for _, pool := range z.serverPools {

cmd/erasure-sets.go

@@ -81,6 +81,9 @@ type erasureSets struct {
 
 	// Total number of sets and the number of disks per set.
 	setCount, setDriveCount int
+	defaultParityCount      int
+
+	poolNumber int
 
 	disksConnectEvent chan diskConnectInfo
 
@@ -291,27 +294,6 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 	}
 }
 
-// GetAllLockers return a list of all lockers for all sets.
-func (s *erasureSets) GetAllLockers() []dsync.NetLocker {
-	var allLockers []dsync.NetLocker
-	lockEpSet := set.NewStringSet()
-	for _, lockers := range s.erasureLockers {
-		for _, locker := range lockers {
-			if locker == nil || !locker.IsOnline() {
-				// Skip any offline lockers.
-				continue
-			}
-			if lockEpSet.Contains(locker.String()) {
-				// Skip duplicate lockers.
-				continue
-			}
-			lockEpSet.Add(locker.String())
-			allLockers = append(allLockers, locker)
-		}
-	}
-	return allLockers
-}
-
 func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) {
 	return func() ([]dsync.NetLocker, string) {
 		lockers := make([]dsync.NetLocker, len(s.erasureLockers[setIndex]))
@@ -349,7 +331,7 @@ func (s *erasureSets) GetDisks(setIndex int) func() []StorageAPI {
 const defaultMonitorConnectEndpointInterval = defaultMonitorNewDiskInterval + time.Second*5
 
 // Initialize new set of erasure coded sets.
-func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageAPI, format *formatErasureV3) (*erasureSets, error) {
+func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageAPI, format *formatErasureV3, defaultParityCount int) (*erasureSets, error) {
 	setCount := len(format.Erasure.Sets)
 	setDriveCount := len(format.Erasure.Sets[0])
 
@@ -357,19 +339,20 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
-		sets:              make([]*erasureObjects, setCount),
-		erasureDisks:      make([][]StorageAPI, setCount),
-		erasureLockers:    make([][]dsync.NetLocker, setCount),
-		erasureLockOwner:  GetLocalPeer(globalEndpoints),
-		endpoints:         endpoints,
-		endpointStrings:   endpointStrings,
-		setCount:          setCount,
-		setDriveCount:     setDriveCount,
-		format:            format,
-		disksConnectEvent: make(chan diskConnectInfo),
-		distributionAlgo:  format.Erasure.DistributionAlgo,
-		deploymentID:      uuid.MustParse(format.ID),
-		mrfOperations:     make(map[healSource]int),
+		sets:               make([]*erasureObjects, setCount),
+		erasureDisks:       make([][]StorageAPI, setCount),
+		erasureLockers:     make([][]dsync.NetLocker, setCount),
+		erasureLockOwner:   GetLocalPeer(globalEndpoints),
+		endpoints:          endpoints,
+		endpointStrings:    endpointStrings,
+		setCount:           setCount,
+		setDriveCount:      setDriveCount,
+		defaultParityCount: defaultParityCount,
+		format:             format,
+		disksConnectEvent:  make(chan diskConnectInfo),
+		distributionAlgo:   format.Erasure.DistributionAlgo,
+		deploymentID:       uuid.MustParse(format.ID),
+		mrfOperations:      make(map[healSource]int),
 	}
 
 	mutex := newNSLock(globalIsDistErasure)
@@ -416,13 +399,15 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 
 		// Initialize erasure objects for a given set.
 		s.sets[i] = &erasureObjects{
-			setDriveCount: setDriveCount,
-			getDisks:      s.GetDisks(i),
-			getLockers:    s.GetLockers(i),
-			getEndpoints:  s.GetEndpoints(i),
-			nsMutex:       mutex,
-			bp:            bp,
-			mrfOpCh:       make(chan partialOperation, 10000),
+			setNumber:          i,
+			setDriveCount:      setDriveCount,
+			defaultParityCount: defaultParityCount,
+			getDisks:           s.GetDisks(i),
+			getLockers:         s.GetLockers(i),
+			getEndpoints:       s.GetEndpoints(i),
+			nsMutex:            mutex,
+			bp:                 bp,
+			mrfOpCh:            make(chan partialOperation, 10000),
 		}
 	}
 
@@ -453,6 +438,44 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval,
 	}
 }
 
+const objectErasureMapKey = "objectErasureMap"
+
+type auditObjectOp struct {
+	Pool  int      `json:"poolId"`
+	Set   int      `json:"setId"`
+	Disks []string `json:"disks"`
+}
+
+func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjects, poolNum int) {
+	if len(logger.AuditTargets) == 0 {
+		return
+	}
+
+	object = decodeDirObject(object)
+
+	op := auditObjectOp{
+		Pool:  poolNum + 1,
+		Set:   set.setNumber + 1,
+		Disks: set.getEndpoints(),
+	}
+
+	var objectErasureSetTag map[string]auditObjectOp
+	reqInfo := logger.GetReqInfo(ctx)
+	for _, kv := range reqInfo.GetTags() {
+		if kv.Key == objectErasureMapKey {
+			objectErasureSetTag = kv.Val.(map[string]auditObjectOp)
+			break
+		}
+	}
+
+	if objectErasureSetTag == nil {
+		objectErasureSetTag = make(map[string]auditObjectOp)
+	}
+
+	objectErasureSetTag[object] = op
+	reqInfo.SetTags(objectErasureMapKey, objectErasureSetTag)
+}
+
 // NewNSLock - initialize a new namespace RWLocker instance.
 func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
 	if len(objects) == 1 {
@@ -466,6 +489,12 @@ func (s *erasureSets) SetDriveCount() int {
 	return s.setDriveCount
 }
 
+// ParityCount returns the default parity count used while erasure
+// coding objects
+func (s *erasureSets) ParityCount() int {
+	return s.defaultParityCount
+}
+
 // StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets.
 // This only returns disk usage info for ServerPools to perform placement decision, this call
 // is not implemented in Object interface and is not meant to be used by other object
@@ -616,9 +645,9 @@ func crcHashMod(key string, cardinality int) int {
 
 func hashKey(algo string, key string, cardinality int, id [16]byte) int {
 	switch algo {
-	case formatErasureVersionV2DistributionAlgoLegacy:
+	case formatErasureVersionV2DistributionAlgoV1:
 		return crcHashMod(key, cardinality)
-	case formatErasureVersionV3DistributionAlgo:
+	case formatErasureVersionV3DistributionAlgoV2, formatErasureVersionV3DistributionAlgoV3:
 		return sipHashMod(key, cardinality, id)
 	default:
 		// Unknown algorithm returns -1, also if cardinality is lesser than 0.
@@ -742,27 +771,45 @@ func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, er
 
 // GetObjectNInfo - returns object info and locked object ReadCloser
 func (s *erasureSets) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	return s.getHashedSet(object).GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 }
 
 // GetObject - reads an object from the hashedSet based on the object name.
 func (s *erasureSets) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
-	return s.getHashedSet(object).GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
+}
+
+func (s *erasureSets) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
+	if parent == "." {
+		return false
+	}
+	return s.getHashedSet(parent).parentDirIsObject(ctx, bucket, parent)
 }
 
 // PutObject - writes an object to hashedSet based on the object name.
 func (s *erasureSets) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return s.getHashedSet(object).PutObject(ctx, bucket, object, data, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	opts.ParentIsObject = s.parentDirIsObject
+	return set.PutObject(ctx, bucket, object, data, opts)
 }
 
 // GetObjectInfo - reads object metadata from the hashedSet based on the object name.
 func (s *erasureSets) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return s.getHashedSet(object).GetObjectInfo(ctx, bucket, object, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.GetObjectInfo(ctx, bucket, object, opts)
 }
 
 // DeleteObject - deletes an object from the hashedSet based on the object name.
 func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return s.getHashedSet(object).DeleteObject(ctx, bucket, object, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.DeleteObject(ctx, bucket, object, opts)
 }
 
 // DeleteObjects - bulk delete of objects
@@ -807,10 +854,14 @@ func (s *erasureSets) DeleteObjects(ctx context.Context, bucket string, objects
 	// Invoke bulk delete on objects per set and save
 	// the result of the delete operation
 	for _, objsGroup := range objSetMap {
-		dobjects, errs := s.getHashedSet(objsGroup[0].object.ObjectName).DeleteObjects(ctx, bucket, toNames(objsGroup), opts)
+		set := s.getHashedSet(objsGroup[0].object.ObjectName)
+		dobjects, errs := set.DeleteObjects(ctx, bucket, toNames(objsGroup), opts)
 		for i, obj := range objsGroup {
 			delErrs[obj.origIndex] = errs[i]
 			delObjects[obj.origIndex] = dobjects[i]
+			if errs[i] == nil {
+				auditObjectErasureSet(ctx, obj.object.ObjectName, set, s.poolNumber)
+			}
 		}
 	}
 
@@ -822,6 +873,8 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 	srcSet := s.getHashedSet(srcObject)
 	dstSet := s.getHashedSet(dstObject)
 
+	auditObjectErasureSet(ctx, dstObject, dstSet, s.poolNumber)
+
 	cpSrcDstSame := srcSet == dstSet
 	// Check if this request is only metadata update.
 	if cpSrcDstSame && srcInfo.metadataOnly {
@@ -1003,45 +1056,60 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
 func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	// In list multipart uploads we are going to treat input prefix as the object,
 	// this means that we are not supporting directory navigation.
-	return s.getHashedSet(prefix).ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+	set := s.getHashedSet(prefix)
+	auditObjectErasureSet(ctx, prefix, set, s.poolNumber)
+	return set.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 }
 
 // Initiate a new multipart upload on a hashedSet based on object name.
 func (s *erasureSets) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
-	return s.getHashedSet(object).NewMultipartUpload(ctx, bucket, object, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.NewMultipartUpload(ctx, bucket, object, opts)
 }
 
 // Copies a part of an object from source hashedSet to destination hashedSet.
 func (s *erasureSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int,
 	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (partInfo PartInfo, err error) {
 	destSet := s.getHashedSet(destObject)
-
+	auditObjectErasureSet(ctx, destObject, destSet, s.poolNumber)
 	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
 }
 
 // PutObjectPart - writes part of an object to hashedSet based on the object name.
 func (s *erasureSets) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
-	return s.getHashedSet(object).PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 }
 
 // GetMultipartInfo - return multipart metadata info uploaded at hashedSet.
 func (s *erasureSets) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (result MultipartInfo, err error) {
-	return s.getHashedSet(object).GetMultipartInfo(ctx, bucket, object, uploadID, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 }
 
 // ListObjectParts - lists all uploaded parts to an object in hashedSet.
 func (s *erasureSets) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
-	return s.getHashedSet(object).ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 }
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
 func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
-	return s.getHashedSet(object).AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	return set.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 }
 
 // CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name.
 func (s *erasureSets) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return s.getHashedSet(object).CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
+	set := s.getHashedSet(object)
+	auditObjectErasureSet(ctx, object, set, s.poolNumber)
+	opts.ParentIsObject = s.parentDirIsObject
+	return set.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 }
 
 /*
@@ -1171,7 +1239,8 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 			if storageDisks[i] != nil && infos[i].RootDisk {
 				// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 				// defective drive we should not heal a path on the root disk.
-				logger.Info("Disk `%s` is a root disk. Please ensure the disk is mounted properly, refusing to use root disk.",
+				logger.Info("Disk `%s` the same as the system root disk.\n"+
+					"Disk will not be used. Please supply a separate disk and restart the server.",
 					storageDisks[i].String())
 				storageDisks[i] = nil
 			}
@@ -1195,7 +1264,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	}(storageDisks)
 
 	formats, sErrs := loadFormatErasureAll(storageDisks, true)
-	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
+	if err = checkFormatErasureValues(formats, storageDisks, s.setDriveCount); err != nil {
 		return madmin.HealResultItem{}, err
 	}
 
@@ -1316,17 +1385,20 @@ func (s *erasureSets) HealObject(ctx context.Context, bucket, object, versionID
 
 // PutObjectTags - replace or add tags to an existing object
 func (s *erasureSets) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) error {
-	return s.getHashedSet(object).PutObjectTags(ctx, bucket, object, tags, opts)
+	er := s.getHashedSet(object)
+	return er.PutObjectTags(ctx, bucket, object, tags, opts)
 }
 
 // DeleteObjectTags - delete object tags from an existing object
 func (s *erasureSets) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) error {
-	return s.getHashedSet(object).DeleteObjectTags(ctx, bucket, object, opts)
+	er := s.getHashedSet(object)
+	return er.DeleteObjectTags(ctx, bucket, object, opts)
 }
 
 // GetObjectTags - get object tags from an existing object
 func (s *erasureSets) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) {
-	return s.getHashedSet(object).GetObjectTags(ctx, bucket, object, opts)
+	er := s.getHashedSet(object)
+	return er.GetObjectTags(ctx, bucket, object, opts)
 }
 
 // maintainMRFList gathers the list of successful partial uploads

cmd/erasure-sets_test.go

@@ -173,23 +173,23 @@ func TestNewErasureSets(t *testing.T) {
 	}
 
 	endpoints := mustGetNewEndpoints(erasureDisks...)
-	_, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "")
+	_, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "", "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)
 	}
 
-	_, _, err = waitForFormatErasure(true, nil, 1, 1, 16, "")
+	_, _, err = waitForFormatErasure(true, nil, 1, 1, 16, "", "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)
 	}
 
 	// Initializes all erasure disks
-	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "")
+	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "", "")
 	if err != nil {
 		t.Fatalf("Unable to format disks for erasure, %s", err)
 	}
 
-	if _, err := newErasureSets(ctx, endpoints, storageDisks, format); err != nil {
+	if _, err := newErasureSets(ctx, endpoints, storageDisks, format, ecDrivesNoConfig(16)); err != nil {
 		t.Fatalf("Unable to initialize erasure")
 	}
 }

cmd/erasure.go

@@ -48,7 +48,10 @@ type partialOperation struct {
 type erasureObjects struct {
 	GatewayUnsupported
 
-	setDriveCount int
+	setDriveCount      int
+	defaultParityCount int
+
+	setNumber int
 
 	// getDisks returns list of storageAPIs.
 	getDisks func() []StorageAPI
@@ -222,6 +225,51 @@ func (er erasureObjects) StorageInfo(ctx context.Context) (StorageInfo, []error)
 	return getStorageInfo(disks, endpoints)
 }
 
+func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, healing bool) {
+	var wg sync.WaitGroup
+	disks := er.getDisks()
+	infos := make([]DiskInfo, len(disks))
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			disk := disks[i-1]
+
+			if disk == nil {
+				return
+			}
+
+			di, err := disk.DiskInfo(context.Background())
+			if err != nil {
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				//
+				// - Future: skip busy disks
+				return
+			}
+
+			infos[i-1] = di
+		}()
+	}
+	wg.Wait()
+
+	for i, info := range infos {
+		// Check if one of the drives in the set is being healed.
+		// this information is used by crawler to skip healing
+		// this erasure set while it calculates the usage.
+		if info.Healing {
+			healing = true
+			continue
+		}
+		newDisks = append(newDisks, disks[i])
+	}
+
+	return newDisks, healing
+}
+
 // CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
@@ -230,8 +278,8 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 		return nil
 	}
 
-	// Collect disks we can use, sorted by least inode usage.
-	disks := er.getOnlineDisksSortedByUsedInodes()
+	// Collect disks we can use.
+	disks, healing := er.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
 		logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl")
 		return nil
@@ -246,6 +294,11 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 			continue
 		}
 		id, _ := disk.GetDiskID()
+		if id == "" {
+			// its possible that disk is unformatted
+			// or just went offline
+			continue
+		}
 		allDiskIDs = append(allDiskIDs, id)
 	}
 
@@ -347,6 +400,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 					cache.Info.Name = bucket.Name
 				}
 				cache.Info.BloomFilter = bloom
+				cache.Info.SkipHealing = healing
 				cache.Disks = allDiskIDs
 				if cache.Info.Name != bucket.Name {
 					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))

cmd/format-erasure.go

@@ -27,6 +27,7 @@ import (
 	"sync"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/color"
@@ -48,10 +49,13 @@ const (
 	formatErasureVersionV3 = "3"
 
 	// Distribution algorithm used, legacy
-	formatErasureVersionV2DistributionAlgoLegacy = "CRCMOD"
+	formatErasureVersionV2DistributionAlgoV1 = "CRCMOD"
 
-	// Distributed algorithm used, current
-	formatErasureVersionV3DistributionAlgo = "SIPMOD"
+	// Distributed algorithm used, with N/2 default parity
+	formatErasureVersionV3DistributionAlgoV2 = "SIPMOD"
+
+	// Distributed algorithm used, with EC:4 default parity
+	formatErasureVersionV3DistributionAlgoV3 = "SIPMOD+PARITY"
 )
 
 // Offline disk UUID represents an offline disk.
@@ -130,17 +134,13 @@ func (f *formatErasureV3) Clone() *formatErasureV3 {
 }
 
 // Returns formatErasure.Erasure.Version
-func newFormatErasureV3(numSets int, setLen int, distributionAlgo string) *formatErasureV3 {
+func newFormatErasureV3(numSets int, setLen int) *formatErasureV3 {
 	format := &formatErasureV3{}
 	format.Version = formatMetaVersionV1
 	format.Format = formatBackendErasure
 	format.ID = mustGetUUID()
 	format.Erasure.Version = formatErasureVersionV3
-	if distributionAlgo == "" {
-		format.Erasure.DistributionAlgo = formatErasureVersionV3DistributionAlgo
-	} else {
-		format.Erasure.DistributionAlgo = distributionAlgo
-	}
+	format.Erasure.DistributionAlgo = formatErasureVersionV3DistributionAlgoV3
 	format.Erasure.Sets = make([][]string, numSets)
 
 	for i := 0; i < numSets; i++ {
@@ -230,7 +230,7 @@ func formatErasureMigrateV1ToV2(export, version string) error {
 	formatV2.Version = formatMetaVersionV1
 	formatV2.Format = formatBackendErasure
 	formatV2.Erasure.Version = formatErasureVersionV2
-	formatV2.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoLegacy
+	formatV2.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
 	formatV2.Erasure.This = formatV1.Erasure.Disk
 	formatV2.Erasure.Sets = make([][]string, 1)
 	formatV2.Erasure.Sets[0] = make([]string, len(formatV1.Erasure.JBOD))
@@ -444,38 +444,38 @@ func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
 }
 
 // Valid formatErasure basic versions.
-func checkFormatErasureValue(formatErasure *formatErasureV3) error {
+func checkFormatErasureValue(formatErasure *formatErasureV3, disk StorageAPI) error {
 	// Validate format version and format type.
 	if formatErasure.Version != formatMetaVersionV1 {
-		return fmt.Errorf("Unsupported version of backend format [%s] found", formatErasure.Version)
+		return fmt.Errorf("Unsupported version of backend format [%s] found on %s", formatErasure.Version, disk)
 	}
 	if formatErasure.Format != formatBackendErasure {
-		return fmt.Errorf("Unsupported backend format [%s] found", formatErasure.Format)
+		return fmt.Errorf("Unsupported backend format [%s] found on %s", formatErasure.Format, disk)
 	}
 	if formatErasure.Erasure.Version != formatErasureVersionV3 {
-		return fmt.Errorf("Unsupported Erasure backend format found [%s]", formatErasure.Erasure.Version)
+		return fmt.Errorf("Unsupported Erasure backend format found [%s] on %s", formatErasure.Erasure.Version, disk)
 	}
 	return nil
 }
 
 // Check all format values.
-func checkFormatErasureValues(formats []*formatErasureV3, setDriveCount int) error {
+func checkFormatErasureValues(formats []*formatErasureV3, disks []StorageAPI, setDriveCount int) error {
 	for i, formatErasure := range formats {
 		if formatErasure == nil {
 			continue
 		}
-		if err := checkFormatErasureValue(formatErasure); err != nil {
+		if err := checkFormatErasureValue(formatErasure, disks[i]); err != nil {
 			return err
 		}
 		if len(formats) != len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]) {
 			return fmt.Errorf("%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)",
-				humanize.Ordinal(i+1), len(formats), humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]))
+				disks[i], len(formats), humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]))
 		}
-		// Only if custom erasure drive count is set,
-		// we should fail here other proceed to honor what
-		// is present on the disk.
+		// Only if custom erasure drive count is set, verify if the
+		// set_drive_count was manually set - we need to honor what is
+		// present on the drives.
 		if globalCustomErasureDriveCount && len(formatErasure.Erasure.Sets[0]) != setDriveCount {
-			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets[0]), setDriveCount)
+			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", disks[i], len(formatErasure.Erasure.Sets[0]), setDriveCount)
 		}
 	}
 	return nil
@@ -838,8 +838,8 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 }
 
 // initFormatErasure - save Erasure format configuration on all disks.
-func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, distributionAlgo string, deploymentID string, sErrs []error) (*formatErasureV3, error) {
-	format := newFormatErasureV3(setCount, setDriveCount, distributionAlgo)
+func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID, distributionAlgo string, sErrs []error) (*formatErasureV3, error) {
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, len(storageDisks))
 	wantAtMost := ecDrivesNoConfig(setDriveCount)
 
@@ -849,6 +849,9 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 			disk := storageDisks[i*setDriveCount+j]
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
+			if distributionAlgo != "" {
+				newFormat.Erasure.DistributionAlgo = distributionAlgo
+			}
 			if deploymentID != "" {
 				newFormat.ID = deploymentID
 			}
@@ -890,18 +893,26 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 	return getFormatErasureInQuorum(formats)
 }
 
+func getDefaultParityBlocks(drive int) int {
+	switch drive {
+	case 3, 2:
+		return 1
+	case 4, 5:
+		return 2
+	case 6, 7:
+		return 3
+	default:
+		return 4
+	}
+}
+
 // ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
 // It will attempt to read it from env variable and fall back to drives/2.
 func ecDrivesNoConfig(setDriveCount int) int {
-	ecDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-	if ecDrives == 0 {
-		cfg, err := storageclass.LookupConfig(nil, setDriveCount)
-		if err == nil {
-			ecDrives = cfg.Standard.Parity
-		}
-		if ecDrives == 0 {
-			ecDrives = setDriveCount / 2
-		}
+	sc, _ := storageclass.LookupConfig(config.KVS{}, setDriveCount)
+	ecDrives := sc.GetParityForSC(storageclass.STANDARD)
+	if ecDrives <= 0 {
+		ecDrives = getDefaultParityBlocks(setDriveCount)
 	}
 	return ecDrives
 }

cmd/format-erasure_test.go

@@ -43,7 +43,8 @@ func TestFixFormatV3(t *testing.T) {
 		}
 	}
 
-	format := newFormatErasureV3(1, 8, "CRCMOD")
+	format := newFormatErasureV3(1, 8)
+	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
 	formats := make([]*formatErasureV3, 8)
 
 	for j := 0; j < 8; j++ {
@@ -77,7 +78,8 @@ func TestFixFormatV3(t *testing.T) {
 
 // tests formatErasureV3ThisEmpty conditions.
 func TestFormatErasureEmpty(t *testing.T) {
-	format := newFormatErasureV3(1, 16, "CRCMOD")
+	format := newFormatErasureV3(1, 16)
+	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
 	formats := make([]*formatErasureV3, 16)
 
 	for j := 0; j < 16; j++ {
@@ -265,7 +267,7 @@ func TestCheckFormatErasureValue(t *testing.T) {
 
 	// Valid all test cases.
 	for i, testCase := range testCases {
-		if err := checkFormatErasureValue(testCase.format); err != nil && testCase.success {
+		if err := checkFormatErasureValue(testCase.format, nil); err != nil && testCase.success {
 			t.Errorf("Test %d: Expected failure %s", i+1, err)
 		}
 	}
@@ -276,7 +278,8 @@ func TestGetFormatErasureInQuorumCheck(t *testing.T) {
 	setCount := 2
 	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, setDriveCount, "CRCMOD")
+	format := newFormatErasureV3(setCount, setDriveCount)
+	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
 	formats := make([]*formatErasureV3, 32)
 
 	for i := 0; i < setCount; i++ {
@@ -342,7 +345,8 @@ func TestGetErasureID(t *testing.T) {
 	setCount := 2
 	setDriveCount := 8
 
-	format := newFormatErasureV3(setCount, setDriveCount, "CRCMOD")
+	format := newFormatErasureV3(setCount, setDriveCount)
+	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
 	formats := make([]*formatErasureV3, 16)
 
 	for i := 0; i < setCount; i++ {
@@ -397,7 +401,8 @@ func TestNewFormatSets(t *testing.T) {
 	setCount := 2
 	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, setDriveCount, "CRCMOD")
+	format := newFormatErasureV3(setCount, setDriveCount)
+	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
 	formats := make([]*formatErasureV3, 32)
 	errs := make([]error, 32)
 

cmd/fs-v1.go

@@ -123,7 +123,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	}
 
 	var err error
-	if fsPath, err = getValidPath(fsPath, false); err != nil {
+	if fsPath, err = getValidPath(fsPath); err != nil {
 		if err == errMinDiskSize {
 			return nil, config.ErrUnableToWriteInBackend(err).Hint(err.Error())
 		}
@@ -187,9 +187,9 @@ func (fs *FSObjects) NewNSLock(bucket string, objects ...string) RWLocker {
 	return fs.nsMutex.NewNSLock(nil, bucket, objects...)
 }
 
-// SetDriveCount no-op
-func (fs *FSObjects) SetDriveCount() int {
-	return 0
+// SetDriveCounts no-op
+func (fs *FSObjects) SetDriveCounts() []int {
+	return nil
 }
 
 // Shutdown - should be called when process shuts down.
@@ -735,9 +735,9 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		rwPoolUnlocker = func() { fs.rwPool.Close(fsMetaPath) }
 	}
 
-	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, opts, nsUnlocker, rwPoolUnlocker)
-	if rErr != nil {
-		return nil, rErr
+	objReaderFn, off, length, err := NewGetObjectReader(rs, objInfo, opts, nsUnlocker, rwPoolUnlocker)
+	if err != nil {
+		return nil, err
 	}
 
 	// Read the object, doesn't exist returns an s3 compatible error.
@@ -748,10 +748,11 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		nsUnlocker()
 		return nil, toObjectErr(err, bucket, object)
 	}
-	reader := io.LimitReader(readCloser, length)
+
 	closeFn := func() {
 		readCloser.Close()
 	}
+	reader := io.LimitReader(readCloser, length)
 
 	// Check if range is valid
 	if off > size || off+length > size {
@@ -1554,9 +1555,9 @@ func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, opt
 }
 
 // GetMetrics - no op
-func (fs *FSObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
+func (fs *FSObjects) GetMetrics(ctx context.Context) (*BackendMetrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return &Metrics{}, NotImplemented{}
+	return &BackendMetrics{}, NotImplemented{}
 }
 
 // ListObjectsV2 lists all blobs in bucket filtered by prefix

cmd/gateway-common.go

@@ -389,7 +389,7 @@ func shouldMeterRequest(req *http.Request) bool {
 // MetricsTransport is a custom wrapper around Transport to track metrics
 type MetricsTransport struct {
 	Transport *http.Transport
-	Metrics   *Metrics
+	Metrics   *BackendMetrics
 }
 
 // RoundTrip implements the RoundTrip method for MetricsTransport

cmd/gateway-main.go

@@ -233,7 +233,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	srvCfg := newServerConfig()
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg, 0)
+	lookupConfigs(srvCfg, nil)
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/gateway-metrics.go

@@ -29,36 +29,28 @@ type RequestStats struct {
 	Post uint64 `json:"Post"`
 }
 
-// Metrics - represents bytes served from backend
-// only implemented for S3 Gateway
-type Metrics struct {
-	bytesReceived uint64
-	bytesSent     uint64
-	requestStats  RequestStats
-}
-
 // IncBytesReceived - Increase total bytes received from gateway backend
-func (s *Metrics) IncBytesReceived(n uint64) {
+func (s *BackendMetrics) IncBytesReceived(n uint64) {
 	atomic.AddUint64(&s.bytesReceived, n)
 }
 
 // GetBytesReceived - Get total bytes received from gateway backend
-func (s *Metrics) GetBytesReceived() uint64 {
+func (s *BackendMetrics) GetBytesReceived() uint64 {
 	return atomic.LoadUint64(&s.bytesReceived)
 }
 
 // IncBytesSent - Increase total bytes sent to gateway backend
-func (s *Metrics) IncBytesSent(n uint64) {
+func (s *BackendMetrics) IncBytesSent(n uint64) {
 	atomic.AddUint64(&s.bytesSent, n)
 }
 
 // GetBytesSent - Get total bytes received from gateway backend
-func (s *Metrics) GetBytesSent() uint64 {
+func (s *BackendMetrics) GetBytesSent() uint64 {
 	return atomic.LoadUint64(&s.bytesSent)
 }
 
 // IncRequests - Increase request count sent to gateway backend by 1
-func (s *Metrics) IncRequests(method string) {
+func (s *BackendMetrics) IncRequests(method string) {
 	// Only increment for Head & Get requests, else no op
 	if method == http.MethodGet {
 		atomic.AddUint64(&s.requestStats.Get, 1)
@@ -72,11 +64,11 @@ func (s *Metrics) IncRequests(method string) {
 }
 
 // GetRequests - Get total number of Get & Headrequests sent to gateway backend
-func (s *Metrics) GetRequests() RequestStats {
+func (s *BackendMetrics) GetRequests() RequestStats {
 	return s.requestStats
 }
 
-// NewMetrics - Prepare new Metrics structure
-func NewMetrics() *Metrics {
-	return &Metrics{}
+// NewMetrics - Prepare new BackendMetrics structure
+func NewMetrics() *BackendMetrics {
+	return &BackendMetrics{}
 }

cmd/gateway-unsupported.go

@@ -51,9 +51,9 @@ func (a GatewayUnsupported) NewNSLock(bucket string, objects ...string) RWLocker
 	return nil
 }
 
-// SetDriveCount no-op
-func (a GatewayUnsupported) SetDriveCount() int {
-	return 0
+// SetDriveCounts no-op
+func (a GatewayUnsupported) SetDriveCounts() []int {
+	return nil
 }
 
 // ListMultipartUploads lists all multipart uploads.
@@ -202,9 +202,9 @@ func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, sr
 }
 
 // GetMetrics - no op
-func (a GatewayUnsupported) GetMetrics(ctx context.Context) (*Metrics, error) {
+func (a GatewayUnsupported) GetMetrics(ctx context.Context) (*BackendMetrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return &Metrics{}, NotImplemented{}
+	return &BackendMetrics{}, NotImplemented{}
 }
 
 // PutObjectTags - not implemented.

cmd/gateway/azure/gateway-azure.go

@@ -419,7 +419,7 @@ type azureObjects struct {
 	minio.GatewayUnsupported
 	endpoint   *url.URL
 	httpClient *http.Client
-	metrics    *minio.Metrics
+	metrics    *minio.BackendMetrics
 	client     azblob.ServiceURL // Azure sdk client
 }
 
@@ -533,7 +533,7 @@ func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) {
 }
 
 // GetMetrics returns this gateway's metrics
-func (a *azureObjects) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+func (a *azureObjects) GetMetrics(ctx context.Context) (*minio.BackendMetrics, error) {
 	return a.metrics, nil
 }
 

cmd/gateway/gcs/gateway-gcs.go

@@ -341,7 +341,7 @@ type gcsGateway struct {
 	minio.GatewayUnsupported
 	client     *storage.Client
 	httpClient *http.Client
-	metrics    *minio.Metrics
+	metrics    *minio.BackendMetrics
 	projectID  string
 }
 
@@ -359,7 +359,7 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
 }
 
 // GetMetrics returns this gateway's metrics
-func (l *gcsGateway) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+func (l *gcsGateway) GetMetrics(ctx context.Context) (*minio.BackendMetrics, error) {
 	return l.metrics, nil
 }
 

cmd/gateway/s3/gateway-s3.go

@@ -259,11 +259,11 @@ type s3Objects struct {
 	minio.GatewayUnsupported
 	Client     *miniogo.Core
 	HTTPClient *http.Client
-	Metrics    *minio.Metrics
+	Metrics    *minio.BackendMetrics
 }
 
 // GetMetrics returns this gateway's metrics
-func (l *s3Objects) GetMetrics(ctx context.Context) (*minio.Metrics, error) {
+func (l *s3Objects) GetMetrics(ctx context.Context) (*minio.BackendMetrics, error) {
 	return l.Metrics, nil
 }
 

cmd/generic-handlers.go

@@ -30,7 +30,6 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/http/stats"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/handlers"
 )
 
 // Adds limiting body size middleware
@@ -228,7 +227,9 @@ func guessIsMetricsReq(req *http.Request) bool {
 	}
 	aType := getRequestAuthType(req)
 	return (aType == authTypeAnonymous || aType == authTypeJWT) &&
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsPath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsPathLegacy ||
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.
@@ -530,14 +531,6 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 	})
 }
 
-var fwd = handlers.NewForwarder(&handlers.Forwarder{
-	PassHost:     true,
-	RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
-	Logger: func(err error) {
-		logger.LogIf(GlobalContext, err)
-	},
-})
-
 // setBucketForwardingHandler middleware forwards the path style requests
 // on a bucket to the right bucket location, bucket to IP configuration
 // is obtained from centralized etcd configuration service.
@@ -587,7 +580,12 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 					r.URL.Scheme = "https"
 				}
 				r.URL.Host = getHostFromSrv(sr)
-				fwd.ServeHTTP(w, r)
+				// Make sure we remove any existing headers before
+				// proxying the request to another node.
+				for k := range w.Header() {
+					w.Header().Del(k)
+				}
+				globalForwarder.ServeHTTP(w, r)
 				return
 			}
 			h.ServeHTTP(w, r)
@@ -637,7 +635,12 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 				r.URL.Scheme = "https"
 			}
 			r.URL.Host = getHostFromSrv(sr)
-			fwd.ServeHTTP(w, r)
+			// Make sure we remove any existing headers before
+			// proxying the request to another node.
+			for k := range w.Header() {
+				w.Header().Del(k)
+			}
+			globalForwarder.ServeHTTP(w, r)
 			return
 		}
 		h.ServeHTTP(w, r)

cmd/global-heal.go

@@ -22,6 +22,8 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/console"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -146,6 +148,10 @@ func healErasureSet(ctx context.Context, setIndex int, buckets []BucketInfo, dis
 			}
 		}
 
+		if serverDebugLog {
+			console.Debugf(color.Green("healDisk:")+" healing bucket %s content on erasure set %d\n", bucket.Name, setIndex+1)
+		}
+
 		var entryChs []FileInfoVersionsCh
 		var mu sync.Mutex
 		var wg sync.WaitGroup
@@ -194,8 +200,8 @@ func healErasureSet(ctx context.Context, setIndex int, buckets []BucketInfo, dis
 	return nil
 }
 
-// deepHealObject heals given object path in deep to fix bitrot.
-func deepHealObject(bucket, object, versionID string) {
+// healObject heals given object path in deep to fix bitrot.
+func healObject(bucket, object, versionID string, scan madmin.HealScanMode) {
 	// Get background heal sequence to send elements to heal
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if ok {
@@ -205,7 +211,7 @@ func deepHealObject(bucket, object, versionID string) {
 			versionID: versionID,
 			opts: &madmin.HealOpts{
 				Remove:   true, // if found dangling purge it.
-				ScanMode: madmin.HealDeepScan,
+				ScanMode: scan,
 			},
 		}
 	}

cmd/globals.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/pkg/bucket/bandwidth"
+	"github.com/minio/minio/pkg/handlers"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/config/cache"
@@ -285,6 +286,8 @@ var (
 	globalProxyTransport http.RoundTripper
 
 	globalDNSCache *xhttp.DNSCache
+
+	globalForwarder *handlers.Forwarder
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -35,16 +35,19 @@ type apiConfig struct {
 	listQuorum       int
 	extendListLife   time.Duration
 	corsAllowOrigins []string
-	setDriveCount    int
+	// total drives per erasure set across pools.
+	totalDriveCount int
 }
 
-func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
+func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
 	t.clusterDeadline = cfg.ClusterDeadline
 	t.corsAllowOrigins = cfg.CorsAllowOrigin
-	t.setDriveCount = setDriveCount
+	for _, setDriveCount := range setDriveCounts {
+		t.totalDriveCount += setDriveCount
+	}
 
 	var apiRequestsMaxPerNode int
 	if cfg.RequestsMax <= 0 {
@@ -56,8 +59,8 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
 		}
 		// max requests per node is calculated as
 		// total_ram / ram_per_request
-		// ram_per_request is 4MiB * setDriveCount + 2 * 10MiB (default erasure block size)
-		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(setDriveCount*(blockSizeLarge+blockSizeSmall)+blockSizeV1*2))
+		// ram_per_request is 1MiB * driveCount + 2 * 10MiB (default erasure block size)
+		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(t.totalDriveCount*(blockSizeLarge+blockSizeSmall)+blockSizeV1*2))
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
 		if len(globalEndpoints.Hostnames()) > 0 {
@@ -84,13 +87,6 @@ func (t *apiConfig) getListQuorum() int {
 	return t.listQuorum
 }
 
-func (t *apiConfig) getSetDriveCount() int {
-	t.mu.RLock()
-	defer t.mu.RUnlock()
-
-	return t.setDriveCount
-}
-
 func (t *apiConfig) getExtendListLife() time.Duration {
 	t.mu.RLock()
 	defer t.mu.RUnlock()

cmd/http-stats.go

@@ -79,10 +79,10 @@ func (s *ConnStats) getS3OutputBytes() uint64 {
 // Return connection stats (total input/output bytes and total s3 input/output bytes)
 func (s *ConnStats) toServerConnStats() ServerConnStats {
 	return ServerConnStats{
-		TotalInputBytes:  s.getTotalInputBytes(),
-		TotalOutputBytes: s.getTotalOutputBytes(),
-		S3InputBytes:     s.getS3InputBytes(),
-		S3OutputBytes:    s.getS3OutputBytes(),
+		TotalInputBytes:  s.getTotalInputBytes(),  // Traffic including reserved bucket
+		TotalOutputBytes: s.getTotalOutputBytes(), // Traffic including reserved bucket
+		S3InputBytes:     s.getS3InputBytes(),     // Traffic for client buckets
+		S3OutputBytes:    s.getS3OutputBytes(),    // Traffic for client buckets
 	}
 }
 
@@ -163,9 +163,11 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 // Update statistics from http request and response data
 func (st *HTTPStats) updateStats(api string, r *http.Request, w *logger.ResponseWriter) {
 	// A successful request has a 2xx response code
-	successReq := (w.StatusCode >= 200 && w.StatusCode < 300)
+	successReq := w.StatusCode >= 200 && w.StatusCode < 300
 
-	if !strings.HasSuffix(r.URL.Path, prometheusMetricsPath) {
+	if !strings.HasSuffix(r.URL.Path, prometheusMetricsPathLegacy) ||
+		!strings.HasSuffix(r.URL.Path, prometheusMetricsV2ClusterPath) ||
+		!strings.HasSuffix(r.URL.Path, prometheusMetricsV2NodePath) {
 		st.totalS3Requests.Inc(api)
 		if !successReq && w.StatusCode != 0 {
 			st.totalS3Errors.Inc(api)

cmd/http/headers.go

@@ -164,7 +164,7 @@ const (
 	// Header indicates delete-marker replication status.
 	MinIODeleteMarkerReplicationStatus = "X-Minio-Replication-DeleteMarker-Status"
 	// Header indicates if its a GET/HEAD proxy request for active-active replication
-	MinIOSourceProxyRequest = "x-minio-source-proxy-request"
+	MinIOSourceProxyRequest = "X-Minio-Source-Proxy-Request"
 )
 
 // Common http query params S3 API

cmd/iam-etcd-store.go

@@ -264,6 +264,9 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
 	var p iampolicy.Policy
 	err := ies.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
 	if err != nil {
+		if err == errConfigNotFound {
+			return errNoSuchPolicy
+		}
 		return err
 	}
 	m[policy] = p
@@ -282,8 +285,7 @@ func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]iampol
 
 	// Reload config and policies for all policys.
 	for _, policyName := range policies.ToSlice() {
-		err = ies.loadPolicyDoc(ctx, policyName, m)
-		if err != nil {
+		if err = ies.loadPolicyDoc(ctx, policyName, m); err != nil && err != errNoSuchPolicy {
 			return err
 		}
 	}
@@ -358,7 +360,7 @@ func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m
 
 	// Reload config for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadUser(ctx, user, userType, m); err != nil {
+		if err = ies.loadUser(ctx, user, userType, m); err != nil && err != errNoSuchUser {
 			return err
 		}
 	}
@@ -392,7 +394,7 @@ func (ies *IAMEtcdStore) loadGroups(ctx context.Context, m map[string]GroupInfo)
 
 	// Reload config for all groups.
 	for _, group := range groups.ToSlice() {
-		if err = ies.loadGroup(ctx, group, m); err != nil {
+		if err = ies.loadGroup(ctx, group, m); err != nil && err != errNoSuchGroup {
 			return err
 		}
 	}
@@ -440,7 +442,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 	// Reload config and policies for all users.
 	for _, user := range users.ToSlice() {
-		if err = ies.loadMappedPolicy(ctx, user, userType, isGroup, m); err != nil {
+		if err = ies.loadMappedPolicy(ctx, user, userType, isGroup, m); err != nil && err != errNoSuchPolicy {
 			return err
 		}
 	}
@@ -449,117 +451,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 }
 
 func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
-	iamUsersMap := make(map[string]auth.Credentials)
-	iamGroupsMap := make(map[string]GroupInfo)
-	iamUserPolicyMap := make(map[string]MappedPolicy)
-	iamGroupPolicyMap := make(map[string]MappedPolicy)
-
-	ies.rlock()
-	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
-	ies.runlock()
-
-	ies.lock()
-	if err := ies.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
-		ies.unlock()
-		return err
-	}
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
-
-	ies.unlock()
-
-	if isMinIOUsersSys {
-		if err := ies.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
-			return err
-		}
-		if err := ies.loadGroups(ctx, iamGroupsMap); err != nil {
-			return err
-		}
-	}
-
-	// load polices mapped to users
-	if err := ies.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	// load policies mapped to groups
-	if err := ies.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
-		return err
-	}
-
-	if err := ies.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS temp users
-	if err := ies.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS policy mappings
-	if err := ies.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	ies.lock()
-	defer ies.Unlock()
-
-	// Merge the new reloaded entries into global map.
-	// See issue https://github.com/minio/minio/issues/9651
-	// where the present list of entries on disk are not yet
-	// latest, there is a small window where this can make
-	// valid users invalid.
-	for k, v := range iamUsersMap {
-		sys.iamUsersMap[k] = v
-	}
-
-	for k, v := range iamUserPolicyMap {
-		sys.iamUserPolicyMap[k] = v
-	}
-
-	// purge any expired entries which became expired now.
-	var expiredEntries []string
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			expiredEntries = append(expiredEntries, k)
-			// Deleting on the disk is taken care of in the next cycle
-		}
-	}
-
-	for _, v := range sys.iamUsersMap {
-		if v.IsServiceAccount() {
-			for _, accessKey := range expiredEntries {
-				if v.ParentUser == accessKey {
-					_ = ies.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
-					delete(sys.iamUsersMap, v.AccessKey)
-				}
-			}
-		}
-	}
-
-	// purge any expired entries which became expired now.
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			// Deleting on the etcd is taken care of in the next cycle
-		}
-	}
-
-	for k, v := range iamGroupPolicyMap {
-		sys.iamGroupPolicyMap[k] = v
-	}
-
-	for k, v := range iamGroupsMap {
-		sys.iamGroupsMap[k] = v
-	}
-
-	sys.buildUserGroupMemberships()
-	sys.storeFallback = false
-
-	return nil
+	return sys.Load(ctx, ies)
 }
 
 func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
@@ -653,6 +545,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 	usersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigUsersPrefix)
 	groupsPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigGroupsPrefix)
 	stsPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigSTSPrefix)
+	svcPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigServiceAccountsPrefix)
 	policyPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPoliciesPrefix)
 	policyDBUsersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPolicyDBUsersPrefix)
 	policyDBSTSUsersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPolicyDBSTSUsersPrefix)
@@ -672,6 +565,10 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigSTSPrefix))
 			ies.loadUser(ctx, accessKey, stsUser, sys.iamUsersMap)
+		case svcPrefix:
+			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigServiceAccountsPrefix))
+			ies.loadUser(ctx, accessKey, srvAccUser, sys.iamUsersMap)
 		case groupsPrefix:
 			group := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigGroupsPrefix))

cmd/iam-object-store.go

@@ -402,110 +402,9 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 	return nil
 }
 
-// Refresh IAMSys. If an object layer is passed in use that, otherwise
-// load from global.
+// Refresh IAMSys. If an object layer is passed in use that, otherwise load from global.
 func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
-	iamUsersMap := make(map[string]auth.Credentials)
-	iamGroupsMap := make(map[string]GroupInfo)
-	iamUserPolicyMap := make(map[string]MappedPolicy)
-	iamGroupPolicyMap := make(map[string]MappedPolicy)
-
-	iamOS.rlock()
-	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
-	iamOS.runlock()
-
-	iamOS.lock()
-	if err := iamOS.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
-		iamOS.unlock()
-		return err
-	}
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
-	iamOS.unlock()
-
-	if isMinIOUsersSys {
-		if err := iamOS.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
-			return err
-		}
-		if err := iamOS.loadGroups(ctx, iamGroupsMap); err != nil {
-			return err
-		}
-	}
-
-	// load polices mapped to users
-	if err := iamOS.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	// load policies mapped to groups
-	if err := iamOS.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
-		return err
-	}
-
-	if err := iamOS.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS temp users
-	if err := iamOS.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
-		return err
-	}
-
-	// load STS policy mappings
-	if err := iamOS.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
-		return err
-	}
-
-	iamOS.lock()
-	defer iamOS.unlock()
-
-	// Merge the new reloaded entries into global map.
-	// See issue https://github.com/minio/minio/issues/9651
-	// where the present list of entries on disk are not yet
-	// latest, there is a small window where this can make
-	// valid users invalid.
-	for k, v := range iamUsersMap {
-		sys.iamUsersMap[k] = v
-	}
-
-	for k, v := range iamUserPolicyMap {
-		sys.iamUserPolicyMap[k] = v
-	}
-
-	// purge any expired entries which became expired now.
-	var expiredEntries []string
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			expiredEntries = append(expiredEntries, k)
-			// Deleting on the disk is taken care of in the next cycle
-		}
-	}
-
-	for _, v := range sys.iamUsersMap {
-		if v.IsServiceAccount() {
-			for _, accessKey := range expiredEntries {
-				if v.ParentUser == accessKey {
-					_ = iamOS.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
-					delete(sys.iamUsersMap, v.AccessKey)
-				}
-			}
-		}
-	}
-
-	for k, v := range iamGroupPolicyMap {
-		sys.iamGroupPolicyMap[k] = v
-	}
-
-	for k, v := range iamGroupsMap {
-		sys.iamGroupsMap[k] = v
-	}
-
-	sys.buildUserGroupMemberships()
-	sys.storeFallback = false
-
-	return nil
+	return sys.Load(ctx, iamOS)
 }
 
 func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {

cmd/iam.go

@@ -442,6 +442,122 @@ func (sys *IAMSys) Initialized() bool {
 	return sys.store != nil
 }
 
+// Load - loads all credentials
+func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
+	iamUsersMap := make(map[string]auth.Credentials)
+	iamGroupsMap := make(map[string]GroupInfo)
+	iamUserPolicyMap := make(map[string]MappedPolicy)
+	iamGroupPolicyMap := make(map[string]MappedPolicy)
+	iamPolicyDocsMap := make(map[string]iampolicy.Policy)
+
+	store.rlock()
+	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
+	store.runlock()
+
+	if err := store.loadPolicyDocs(ctx, iamPolicyDocsMap); err != nil {
+		return err
+	}
+
+	// Sets default canned policies, if none are set.
+	setDefaultCannedPolicies(iamPolicyDocsMap)
+
+	if isMinIOUsersSys {
+		if err := store.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
+			return err
+		}
+		if err := store.loadGroups(ctx, iamGroupsMap); err != nil {
+			return err
+		}
+	}
+
+	// load polices mapped to users
+	if err := store.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
+		return err
+	}
+
+	// load policies mapped to groups
+	if err := store.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
+		return err
+	}
+
+	if err := store.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
+		return err
+	}
+
+	// load STS temp users
+	if err := store.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
+		return err
+	}
+
+	// load STS policy mappings
+	if err := store.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
+		return err
+	}
+
+	store.lock()
+	defer store.unlock()
+
+	for k, v := range iamPolicyDocsMap {
+		sys.iamPolicyDocsMap[k] = v
+	}
+
+	// Merge the new reloaded entries into global map.
+	// See issue https://github.com/minio/minio/issues/9651
+	// where the present list of entries on disk are not yet
+	// latest, there is a small window where this can make
+	// valid users invalid.
+	for k, v := range iamUsersMap {
+		sys.iamUsersMap[k] = v
+	}
+
+	for k, v := range iamUserPolicyMap {
+		sys.iamUserPolicyMap[k] = v
+	}
+
+	// purge any expired entries which became expired now.
+	var expiredEntries []string
+	for k, v := range sys.iamUsersMap {
+		if v.IsExpired() {
+			delete(sys.iamUsersMap, k)
+			delete(sys.iamUserPolicyMap, k)
+			expiredEntries = append(expiredEntries, k)
+			// Deleting on the disk is taken care of in the next cycle
+		}
+	}
+
+	for _, v := range sys.iamUsersMap {
+		if v.IsServiceAccount() {
+			for _, accessKey := range expiredEntries {
+				if v.ParentUser == accessKey {
+					_ = store.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
+					delete(sys.iamUsersMap, v.AccessKey)
+				}
+			}
+		}
+	}
+
+	// purge any expired entries which became expired now.
+	for k, v := range sys.iamUsersMap {
+		if v.IsExpired() {
+			delete(sys.iamUsersMap, k)
+			delete(sys.iamUserPolicyMap, k)
+			// Deleting on the etcd is taken care of in the next cycle
+		}
+	}
+
+	for k, v := range iamGroupPolicyMap {
+		sys.iamGroupPolicyMap[k] = v
+	}
+
+	for k, v := range iamGroupsMap {
+		sys.iamGroupsMap[k] = v
+	}
+
+	sys.buildUserGroupMemberships()
+	sys.storeFallback = false
+	return nil
+}
+
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	// Initialize IAM store
@@ -664,6 +780,10 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		return errServerNotInitialized
 	}
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	// First we remove the user from their groups.
 	userInfo, getErr := sys.GetUserInfo(accessKey)
 	if getErr != nil {
@@ -681,18 +801,21 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	sys.store.lock()
 	defer sys.store.unlock()
 
-	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
-	}
-
-	// Delete any service accounts if any first.
 	for _, u := range sys.iamUsersMap {
+		// Delete any service accounts if any first.
 		if u.IsServiceAccount() {
 			if u.ParentUser == accessKey {
 				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, srvAccUser)
 				delete(sys.iamUsersMap, u.AccessKey)
 			}
 		}
+		// Delete any associated STS users.
+		if u.IsTemp() {
+			if u.ParentUser == accessKey {
+				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, stsUser)
+				delete(sys.iamUsersMap, u.AccessKey)
+			}
+		}
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
@@ -869,13 +992,19 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	}
 
 	sys.store.rlock()
-	defer sys.store.runlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
+	if fallback {
+		sys.loadUserFromStore(name)
+	}
 
 	if sys.usersSysType != MinIOUsersSysType {
+		sys.store.rlock()
 		// If the user has a mapped policy or is a member of a group, we
 		// return that info. Otherwise we return error.
 		mappedPolicy, ok1 := sys.iamUserPolicyMap[name]
 		memberships, ok2 := sys.iamUserGroupMemberships[name]
+		sys.store.runlock()
 		if !ok1 && !ok2 {
 			return u, errNoSuchUser
 		}
@@ -885,6 +1014,9 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 		}, nil
 	}
 
+	sys.store.rlock()
+	defer sys.store.runlock()
+
 	cred, found := sys.iamUsersMap[name]
 	if !found {
 		return u, errNoSuchUser
@@ -894,7 +1026,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 		return u, errIAMActionNotAllowed
 	}
 
-	u = madmin.UserInfo{
+	return madmin.UserInfo{
 		PolicyName: sys.iamUserPolicyMap[name].Policies,
 		Status: func() madmin.AccountStatus {
 			if cred.IsValid() {
@@ -903,8 +1035,8 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 			return madmin.AccountDisabled
 		}(),
 		MemberOf: sys.iamUserGroupMemberships[name].ToSlice(),
-	}
-	return u, nil
+	}, nil
+
 }
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
@@ -913,6 +1045,10 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		return errServerNotInitialized
 	}
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	if status != madmin.AccountEnabled && status != madmin.AccountDisabled {
 		return errInvalidArgument
 	}
@@ -920,10 +1056,6 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 	sys.store.lock()
 	defer sys.store.unlock()
 
-	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
-	}
-
 	cred, ok := sys.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
@@ -1091,30 +1223,31 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 	return nil
 }
 
-// SetUser - set user credentials and policy.
-func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
+// CreateUser - create new user credentials and policy, if user already exists
+// they shall be rewritten with new inputs.
+func (sys *IAMSys) CreateUser(accessKey string, uinfo madmin.UserInfo) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
+	sys.store.lock()
+	defer sys.store.unlock()
+
+	cr, ok := sys.iamUsersMap[accessKey]
+	if cr.IsTemp() && ok {
+		return errIAMActionNotAllowed
+	}
+
 	u := newUserIdentity(auth.Credentials{
 		AccessKey: accessKey,
 		SecretKey: uinfo.SecretKey,
 		Status:    string(uinfo.Status),
 	})
 
-	sys.store.lock()
-	defer sys.store.unlock()
-
-	if sys.usersSysType != MinIOUsersSysType {
-		return errIAMActionNotAllowed
-	}
-
-	cr, ok := sys.iamUsersMap[accessKey]
-	if cr.IsTemp() && ok {
-		return errIAMActionNotAllowed
-	}
-
 	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, u); err != nil {
 		return err
 	}
@@ -1156,6 +1289,44 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 	return nil
 }
 
+func (sys *IAMSys) loadUserFromStore(accessKey string) {
+	sys.store.lock()
+	// If user is already found proceed.
+	if _, found := sys.iamUsersMap[accessKey]; !found {
+		sys.store.loadUser(context.Background(), accessKey, regularUser, sys.iamUsersMap)
+		if _, found = sys.iamUsersMap[accessKey]; found {
+			// found user, load its mapped policies
+			sys.store.loadMappedPolicy(context.Background(), accessKey, regularUser, false, sys.iamUserPolicyMap)
+		} else {
+			sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
+			if svc, found := sys.iamUsersMap[accessKey]; found {
+				// Found service account, load its parent user and its mapped policies.
+				if sys.usersSysType == MinIOUsersSysType {
+					sys.store.loadUser(context.Background(), svc.ParentUser, regularUser, sys.iamUsersMap)
+				}
+				sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
+			} else {
+				// None found fall back to STS users.
+				sys.store.loadUser(context.Background(), accessKey, stsUser, sys.iamUsersMap)
+				if _, found = sys.iamUsersMap[accessKey]; found {
+					// STS user found, load its mapped policy.
+					sys.store.loadMappedPolicy(context.Background(), accessKey, stsUser, false, sys.iamUserPolicyMap)
+				}
+			}
+		}
+	}
+
+	// Load associated policies if any.
+	for _, policy := range sys.iamUserPolicyMap[accessKey].toSlice() {
+		if _, found := sys.iamPolicyDocsMap[policy]; !found {
+			sys.store.loadPolicyDoc(context.Background(), policy, sys.iamPolicyDocsMap)
+		}
+	}
+
+	sys.buildUserGroupMemberships()
+	sys.store.unlock()
+}
+
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	if !sys.Initialized() {
@@ -1166,47 +1337,26 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	fallback := sys.storeFallback
 	sys.store.runlock()
 	if fallback {
-		sys.store.lock()
-		// If user is already found proceed.
-		if _, found := sys.iamUsersMap[accessKey]; !found {
-			sys.store.loadUser(context.Background(), accessKey, regularUser, sys.iamUsersMap)
-			if _, found = sys.iamUsersMap[accessKey]; found {
-				// found user, load its mapped policies
-				sys.store.loadMappedPolicy(context.Background(), accessKey, regularUser, false, sys.iamUserPolicyMap)
-			} else {
-				sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
-				if svc, found := sys.iamUsersMap[accessKey]; found {
-					// Found service account, load its parent user and its mapped policies.
-					if sys.usersSysType == MinIOUsersSysType {
-						sys.store.loadUser(context.Background(), svc.ParentUser, regularUser, sys.iamUsersMap)
-					}
-					sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
-				} else {
-					// None found fall back to STS users.
-					sys.store.loadUser(context.Background(), accessKey, stsUser, sys.iamUsersMap)
-					if _, found = sys.iamUsersMap[accessKey]; found {
-						// STS user found, load its mapped policy.
-						sys.store.loadMappedPolicy(context.Background(), accessKey, stsUser, false, sys.iamUserPolicyMap)
-					}
-				}
-			}
-		}
-
-		// Load associated policies if any.
-		for _, policy := range sys.iamUserPolicyMap[accessKey].toSlice() {
-			if _, found := sys.iamPolicyDocsMap[policy]; !found {
-				sys.store.loadPolicyDoc(context.Background(), policy, sys.iamPolicyDocsMap)
-			}
-		}
-
-		sys.buildUserGroupMemberships()
-		sys.store.unlock()
+		sys.loadUserFromStore(accessKey)
 	}
 
 	sys.store.rlock()
+	cred, ok = sys.iamUsersMap[accessKey]
+	if !ok && !fallback {
+		sys.store.runlock()
+		// accessKey not found, also
+		// IAM store is not in fallback mode
+		// we can try to reload again from
+		// the IAM store and see if credential
+		// exists now. If it doesn't proceed to
+		// fail.
+		sys.loadUserFromStore(accessKey)
+
+		sys.store.rlock()
+		cred, ok = sys.iamUsersMap[accessKey]
+	}
 	defer sys.store.runlock()
 
-	cred, ok = sys.iamUsersMap[accessKey]
 	if ok && cred.IsValid() {
 		if cred.ParentUser != "" && sys.usersSysType == MinIOUsersSysType {
 			_, ok = sys.iamUsersMap[cred.ParentUser]
@@ -1288,14 +1438,14 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 		return errServerNotInitialized
 	}
 
-	if group == "" {
-		return errInvalidArgument
-	}
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
+	if group == "" {
+		return errInvalidArgument
+	}
+
 	sys.store.lock()
 	defer sys.store.unlock()
 

cmd/listen-notification-handlers.go

@@ -30,7 +30,7 @@ import (
 func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenNotification")
 
-	defer logger.AuditLog(w, r, "ListenNotification", mustGetClaimsFromToken(r))
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
@@ -139,6 +139,9 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
 	})
 
+	if bucketName != "" {
+		values.Set(peerRESTListenBucket, bucketName)
+	}
 	for _, peer := range peers {
 		if peer == nil {
 			continue

cmd/local-locker.go

@@ -27,11 +27,13 @@ import (
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
+	Name          string    // name of the resource lock was requested for
 	Writer        bool      // Bool whether write or read lock.
 	UID           string    // UID to uniquely identify request of client.
 	Timestamp     time.Time // Timestamp set at the time of initialization.
 	TimeLastCheck time.Time // Timestamp for last check of validity of lock.
 	Source        string    // Contains line, function and filename reqesting the lock.
+	Group         bool      // indicates if it was a group lock.
 	// Owner represents the UUID of the owner who originally requested the lock
 	// useful in expiry.
 	Owner string
@@ -91,12 +93,14 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 	for _, resource := range args.Resources {
 		l.lockMap[resource] = []lockRequesterInfo{
 			{
+				Name:          resource,
 				Writer:        true,
 				Source:        args.Source,
 				Owner:         args.Owner,
 				UID:           args.UID,
 				Timestamp:     UTCNow(),
 				TimeLastCheck: UTCNow(),
+				Group:         len(args.Resources) > 1,
 				Quorum:        args.Quorum,
 			},
 		}
@@ -148,7 +152,9 @@ func (l *localLocker) removeEntry(name string, args dsync.LockArgs, lri *[]lockR
 func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
+	resource := args.Resources[0]
 	lrInfo := lockRequesterInfo{
+		Name:          resource,
 		Writer:        false,
 		Source:        args.Source,
 		Owner:         args.Owner,
@@ -157,7 +163,6 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 		TimeLastCheck: UTCNow(),
 		Quorum:        args.Quorum,
 	}
-	resource := args.Resources[0]
 	if lri, ok := l.lockMap[resource]; ok {
 		if reply = !isWriteLock(lri); reply {
 			// Unless there is a write lock
@@ -214,6 +219,23 @@ func (l *localLocker) IsLocal() bool {
 	return true
 }
 
+func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
+		if len(args.UID) != 0 {
+			return false, fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID)
+		}
+		for _, resource := range args.Resources {
+			delete(l.lockMap, resource) // Remove the lock (irrespective of write or read lock)
+		}
+		return true, nil
+	}
+}
+
 func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired bool, err error) {
 	select {
 	case <-ctx.Done():
@@ -222,38 +244,42 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 		l.mutex.Lock()
 		defer l.mutex.Unlock()
 
+		resource := args.Resources[0] // expiry check is always per resource.
+
 		// Lock found, proceed to verify if belongs to given uid.
-		for _, resource := range args.Resources {
-			if lri, ok := l.lockMap[resource]; ok {
-				// Check whether uid is still active
-				for _, entry := range lri {
-					if entry.UID == args.UID && entry.Owner == args.Owner {
-						if ep, ok := globalRemoteEndpoints[args.Owner]; ok {
-							if err = isServerResolvable(ep); err != nil {
-								return true, nil
-							}
-						}
-						return false, nil
-					}
+		lri, ok := l.lockMap[resource]
+		if !ok {
+			return true, nil
+		}
+
+		// Check whether uid is still active
+		for _, entry := range lri {
+			if entry.UID == args.UID && entry.Owner == args.Owner {
+				ep := globalRemoteEndpoints[args.Owner]
+				if !ep.IsLocal {
+					// check if the owner is online
+					return isServerResolvable(ep, 250*time.Millisecond) != nil, nil
 				}
+				return false, nil
 			}
 		}
+
 		return true, nil
 	}
 }
 
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
 // Caller must hold 'l.mutex' lock.
-func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
+func (l *localLocker) removeEntryIfExists(lrip lockRequesterInfo) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 
 	// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
-	if lri, ok := l.lockMap[nlrip.name]; ok {
+	if lri, ok := l.lockMap[lrip.Name]; ok {
 		// Even if the entry exists, it may not be the same entry which was
 		// considered as expired, so we simply an attempt to remove it if its
 		// not possible there is nothing we need to do.
-		l.removeEntry(nlrip.name, dsync.LockArgs{Owner: nlrip.lri.Owner, UID: nlrip.lri.UID}, &lri)
+		l.removeEntry(lrip.Name, dsync.LockArgs{Owner: lrip.Owner, UID: lrip.UID}, &lri)
 	}
 }
 

cmd/lock-rest-client.go

@@ -135,6 +135,11 @@ func (client *lockRESTClient) Expired(ctx context.Context, args dsync.LockArgs)
 	return client.restCall(ctx, lockRESTMethodExpired, args)
 }
 
+// ForceUnlock calls force unlock handler to forcibly unlock an active lock.
+func (client *lockRESTClient) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
+	return client.restCall(ctx, lockRESTMethodForceUnlock, args)
+}
+
 func newLockAPI(endpoint Endpoint) dsync.NetLocker {
 	if endpoint.IsLocal {
 		return globalLockServer
@@ -156,7 +161,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 	healthClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
 	healthClient.ExpectTimeouts = true
 	restClient.HealthCheckFn = func() bool {
-		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
+		ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout)
 		defer cancel()
 		respBody, err := healthClient.Call(ctx, lockRESTMethodHealth, nil, nil, -1)
 		xhttp.DrainBody(respBody)

cmd/lock-rest-server-common.go

@@ -21,18 +21,19 @@ import (
 )
 
 const (
-	lockRESTVersion       = "v4" // Add Quorum query param
+	lockRESTVersion       = "v5" // Add Quorum query param
 	lockRESTVersionPrefix = SlashSeparator + lockRESTVersion
 	lockRESTPrefix        = minioReservedBucketPath + "/lock"
 )
 
 const (
-	lockRESTMethodHealth  = "/health"
-	lockRESTMethodLock    = "/lock"
-	lockRESTMethodRLock   = "/rlock"
-	lockRESTMethodUnlock  = "/unlock"
-	lockRESTMethodRUnlock = "/runlock"
-	lockRESTMethodExpired = "/expired"
+	lockRESTMethodHealth      = "/health"
+	lockRESTMethodLock        = "/lock"
+	lockRESTMethodRLock       = "/rlock"
+	lockRESTMethodUnlock      = "/unlock"
+	lockRESTMethodRUnlock     = "/runlock"
+	lockRESTMethodExpired     = "/expired"
+	lockRESTMethodForceUnlock = "/force-unlock"
 
 	// lockRESTOwner represents owner UUID
 	lockRESTOwner = "owner"

cmd/lock-rest-server.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 	"sort"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -32,10 +33,10 @@ import (
 
 const (
 	// Lock maintenance interval.
-	lockMaintenanceInterval = 30 * time.Second
+	lockMaintenanceInterval = 10 * time.Second
 
 	// Lock validity check interval.
-	lockValidityCheckInterval = 5 * time.Second
+	lockValidityCheckInterval = 30 * time.Second
 )
 
 // To abstract a node over network.
@@ -98,7 +99,7 @@ func (l *lockRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {
 // LockHandler - Acquires a lock.
 func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
+		l.writeErrorResponse(w, errors.New("invalid request"))
 		return
 	}
 
@@ -121,7 +122,7 @@ func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
 // UnlockHandler - releases the acquired lock.
 func (l *lockRESTServer) UnlockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
+		l.writeErrorResponse(w, errors.New("invalid request"))
 		return
 	}
 
@@ -143,7 +144,7 @@ func (l *lockRESTServer) UnlockHandler(w http.ResponseWriter, r *http.Request) {
 // LockHandler - Acquires an RLock.
 func (l *lockRESTServer) RLockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
+		l.writeErrorResponse(w, errors.New("invalid request"))
 		return
 	}
 
@@ -166,7 +167,7 @@ func (l *lockRESTServer) RLockHandler(w http.ResponseWriter, r *http.Request) {
 // RUnlockHandler - releases the acquired read lock.
 func (l *lockRESTServer) RUnlockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
+		l.writeErrorResponse(w, errors.New("invalid request"))
 		return
 	}
 
@@ -184,10 +185,29 @@ func (l *lockRESTServer) RUnlockHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+// ForceUnlockHandler - query expired lock status.
+func (l *lockRESTServer) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
+	if !l.IsValid(w, r) {
+		l.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	args, err := getLockArgs(r)
+	if err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+
+	if _, err = l.ll.ForceUnlock(r.Context(), args); err != nil {
+		l.writeErrorResponse(w, err)
+		return
+	}
+}
+
 // ExpiredHandler - query expired lock status.
 func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
+		l.writeErrorResponse(w, errors.New("invalid request"))
 		return
 	}
 
@@ -208,31 +228,22 @@ func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// nameLockRequesterInfoPair is a helper type for lock maintenance
-type nameLockRequesterInfoPair struct {
-	name string
-	lri  lockRequesterInfo
-}
-
 // getLongLivedLocks returns locks that are older than a certain time and
 // have not been 'checked' for validity too soon enough
-func getLongLivedLocks(interval time.Duration) []nameLockRequesterInfoPair {
-	nlrip := []nameLockRequesterInfoPair{}
+func getLongLivedLocks(interval time.Duration) []lockRequesterInfo {
+	lrips := []lockRequesterInfo{}
 	globalLockServer.mutex.Lock()
-	for name, lriArray := range globalLockServer.lockMap {
+	for _, lriArray := range globalLockServer.lockMap {
 		for idx := range lriArray {
 			// Check whether enough time has gone by since last check
 			if time.Since(lriArray[idx].TimeLastCheck) >= interval {
-				nlrip = append(nlrip, nameLockRequesterInfoPair{
-					name: name,
-					lri:  lriArray[idx],
-				})
+				lrips = append(lrips, lriArray[idx])
 				lriArray[idx].TimeLastCheck = UTCNow()
 			}
 		}
 	}
 	globalLockServer.mutex.Unlock()
-	return nlrip
+	return lrips
 }
 
 // lockMaintenance loops over locks that have been active for some time and checks back
@@ -261,47 +272,68 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 
 	updateNlocks := func(nlripsMap map[string]nlock, name string, writer bool) {
 		nlk, ok := nlripsMap[name]
-		if !ok {
+		if ok {
+			nlk.locks++
+			nlripsMap[name] = nlk
+		} else {
 			nlripsMap[name] = nlock{
 				locks:  1,
 				writer: writer,
 			}
-		} else {
-			nlk.locks++
-			nlripsMap[name] = nlk
 		}
 	}
 
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
-	nlrips := getLongLivedLocks(interval)
-	nlripsMap := make(map[string]nlock, len(nlrips))
-	for _, nlrip := range nlrips {
-		for _, c := range z.GetAllLockers() {
-			ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
-
-			// Call back to original server verify whether the lock is
-			// still active (based on name & uid)
-			expired, err := c.Expired(ctx, dsync.LockArgs{
-				Owner:     nlrip.lri.Owner,
-				UID:       nlrip.lri.UID,
-				Resources: []string{nlrip.name},
-			})
-			cancel()
-			if err != nil {
-				updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
-				continue
-			}
-
-			if !expired {
-				updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
-			}
+	lrips := getLongLivedLocks(interval)
+	lripsMap := make(map[string]nlock, len(lrips))
+	var mutex sync.Mutex // mutex for lripsMap updates
+	for _, lrip := range lrips {
+		// fetch the lockers participated in handing
+		// out locks for `nlrip.name`
+		var lockers []dsync.NetLocker
+		if lrip.Group {
+			lockers, _ = z.serverPools[0].getHashedSet("").getLockers()
+		} else {
+			lockers, _ = z.serverPools[0].getHashedSet(lrip.Name).getLockers()
 		}
+		var wg sync.WaitGroup
+		wg.Add(len(lockers))
+		for _, c := range lockers {
+			go func(lrip lockRequesterInfo, c dsync.NetLocker) {
+				defer wg.Done()
+				ctx, cancel := context.WithTimeout(GlobalContext, 3*time.Second)
+
+				// Call back to all participating servers, verify
+				// if each of those servers think lock is still
+				// active, if not expire it.
+				expired, err := c.Expired(ctx, dsync.LockArgs{
+					Owner:     lrip.Owner,
+					UID:       lrip.UID,
+					Resources: []string{lrip.Name},
+				})
+				cancel()
+
+				if err != nil {
+					mutex.Lock()
+					updateNlocks(lripsMap, lrip.Name, lrip.Writer)
+					mutex.Unlock()
+					return
+				}
+
+				if !expired {
+					mutex.Lock()
+					updateNlocks(lripsMap, lrip.Name, lrip.Writer)
+					mutex.Unlock()
+				}
+			}(lrip, c)
+		}
+		wg.Wait()
 
 		// less than the quorum, we have locks expired.
-		if nlripsMap[nlrip.name].locks < nlrip.lri.Quorum {
+		if lripsMap[lrip.Name].locks < lrip.Quorum {
 			// Purge the stale entry if it exists.
-			globalLockServer.removeEntryIfExists(nlrip)
+			globalLockServer.removeEntryIfExists(lrip)
 		}
 	}
 
@@ -360,6 +392,7 @@ func registerLockRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler))
 	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler))
 	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodExpired).HandlerFunc(httpTraceAll(lockServer.ExpiredHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodForceUnlock).HandlerFunc(httpTraceAll(lockServer.ForceUnlockHandler))
 
 	globalLockServer = lockServer.ll
 

cmd/logger/audit.go

@@ -18,14 +18,13 @@ package logger
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"strconv"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger/message/audit"
 )
 
@@ -125,7 +124,7 @@ func (lrw *ResponseWriter) Size() int {
 }
 
 // AuditLog - logs audit logs to all audit targets.
-func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}, filterKeys ...string) {
+func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request, reqClaims map[string]interface{}, filterKeys ...string) {
 	// Fast exit if there is not audit target configured
 	if len(AuditTargets) == 0 {
 		return
@@ -144,11 +143,9 @@ func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[
 		timeToFirstByte = st.TimeToFirstByte
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	object, err := url.PathUnescape(vars["object"])
-	if err != nil {
-		object = vars["object"]
+	reqInfo := GetReqInfo(ctx)
+	if reqInfo == nil {
+		return
 	}
 
 	entry := audit.ToEntry(w, r, reqClaims, globalDeploymentID)
@@ -158,12 +155,13 @@ func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[
 		delete(entry.ReqHeader, filterKey)
 		delete(entry.RespHeader, filterKey)
 	}
-	entry.API.Name = api
-	entry.API.Bucket = bucket
-	entry.API.Object = object
+	entry.API.Name = reqInfo.API
+	entry.API.Bucket = reqInfo.BucketName
+	entry.API.Object = reqInfo.ObjectName
 	entry.API.Status = http.StatusText(statusCode)
 	entry.API.StatusCode = statusCode
 	entry.API.TimeToResponse = strconv.FormatInt(timeToResponse.Nanoseconds(), 10) + "ns"
+	entry.Tags = reqInfo.GetTagsMap()
 	// ttfb will be recorded only for GET requests, Ignore such cases where ttfb will be empty.
 	if timeToFirstByte != 0 {
 		entry.API.TimeToFirstByte = strconv.FormatInt(timeToFirstByte.Nanoseconds(), 10) + "ns"

cmd/logger/logger.go

@@ -335,7 +335,7 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
 	}
 
 	kv := req.GetTags()
-	tags := make(map[string]string, len(kv))
+	tags := make(map[string]interface{}, len(kv))
 	for _, entry := range kv {
 		tags[entry.Key] = entry.Val
 	}
@@ -376,7 +376,7 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
 		entry.API.Args.Object = hashString(entry.API.Args.Object)
 		entry.RemoteHost = hashString(entry.RemoteHost)
 		entry.Trace.Message = reflect.TypeOf(err).String()
-		entry.Trace.Variables = make(map[string]string)
+		entry.Trace.Variables = make(map[string]interface{})
 	}
 
 	// Iterate over all logger targets to send the log entry

cmd/logger/message/audit/entry.go

@@ -49,6 +49,7 @@ type Entry struct {
 	ReqQuery   map[string]string      `json:"requestQuery,omitempty"`
 	ReqHeader  map[string]string      `json:"requestHeader,omitempty"`
 	RespHeader map[string]string      `json:"responseHeader,omitempty"`
+	Tags       map[string]interface{} `json:"tags,omitempty"`
 }
 
 // ToEntry - constructs an audit entry object.
@@ -67,7 +68,9 @@ func ToEntry(w http.ResponseWriter, r *http.Request, reqClaims map[string]interf
 	for k, v := range wh {
 		respHeader[k] = strings.Join(v, ",")
 	}
-	respHeader[xhttp.ETag] = strings.Trim(respHeader[xhttp.ETag], `"`)
+	if etag := respHeader[xhttp.ETag]; etag != "" {
+		respHeader[xhttp.ETag] = strings.Trim(etag, `"`)
+	}
 
 	entry := Entry{
 		Version:      Version,

cmd/logger/message/log/entry.go

@@ -27,9 +27,9 @@ type Args struct {
 
 // Trace - defines the trace.
 type Trace struct {
-	Message   string            `json:"message,omitempty"`
-	Source    []string          `json:"source,omitempty"`
-	Variables map[string]string `json:"variables,omitempty"`
+	Message   string                 `json:"message,omitempty"`
+	Source    []string               `json:"source,omitempty"`
+	Variables map[string]interface{} `json:"variables,omitempty"`
 }
 
 // API - defines the api type and its args.

cmd/logger/reqinfo.go

@@ -30,7 +30,7 @@ const contextLogKey = contextKeyType("miniolog")
 // KeyVal - appended to ReqInfo.Tags
 type KeyVal struct {
 	Key string
-	Val string
+	Val interface{}
 }
 
 // ReqInfo stores the request info.
@@ -62,7 +62,7 @@ func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, obj
 }
 
 // AppendTags - appends key/val to ReqInfo.tags
-func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
+func (r *ReqInfo) AppendTags(key string, val interface{}) *ReqInfo {
 	if r == nil {
 		return nil
 	}
@@ -73,7 +73,7 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
 }
 
 // SetTags - sets key/val to ReqInfo.tags
-func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
+func (r *ReqInfo) SetTags(key string, val interface{}) *ReqInfo {
 	if r == nil {
 		return nil
 	}
@@ -105,6 +105,20 @@ func (r *ReqInfo) GetTags() []KeyVal {
 	return append([]KeyVal(nil), r.tags...)
 }
 
+// GetTagsMap - returns the user defined tags in a map structure
+func (r *ReqInfo) GetTagsMap() map[string]interface{} {
+	if r == nil {
+		return nil
+	}
+	r.RLock()
+	defer r.RUnlock()
+	m := make(map[string]interface{}, len(r.tags))
+	for _, t := range r.tags {
+		m[t.Key] = t.Val
+	}
+	return m
+}
+
 // SetReqInfo sets ReqInfo in the context.
 func SetReqInfo(ctx context.Context, req *ReqInfo) context.Context {
 	if ctx == nil {

cmd/logger/target/console/console.go

@@ -76,7 +76,7 @@ func (c *Target) Send(e interface{}, logKind string) error {
 			if tagString != "" {
 				tagString += ", "
 			}
-			tagString += key + "=" + value
+			tagString += fmt.Sprintf("%s=%v", key, value)
 		}
 	}
 

cmd/metacache-bucket.go

@@ -437,7 +437,7 @@ func (b *bucketMetacache) deleteAll() {
 	ctx := context.Background()
 	ez, ok := newObjectLayerFn().(*erasureServerPools)
 	if !ok {
-		logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be *erasureZones"))
+		logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be *erasurePools"))
 		return
 	}
 

cmd/metacache-set.go

@@ -540,10 +540,6 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 	}
 }
 
-func (er erasureObjects) SetDriveCount() int {
-	return er.setDriveCount
-}
-
 // Will return io.EOF if continuing would not yield more results.
 func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
 	o.debugf(color.Green("listPath:")+" with options: %#v", o)
@@ -598,9 +594,9 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
 	askDisks := o.AskDisks
 	listingQuorum := askDisks - 1
 	// Special case: ask all disks if the drive count is 4
-	if askDisks == -1 || er.SetDriveCount() == 4 {
+	if askDisks == -1 || er.setDriveCount == 4 {
 		askDisks = len(disks) // with 'strict' quorum list on all online disks.
-		listingQuorum = getReadQuorum(er.SetDriveCount())
+		listingQuorum = getReadQuorum(er.setDriveCount)
 	}
 
 	if len(disks) < askDisks {
@@ -680,8 +676,9 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
 				logger.LogIf(ctx, err)
 				custom := b.headerKV()
 				_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r, nil, nil), ObjectOptions{
-					UserDefined: custom,
-					NoLock:      true, // No need to hold namespace lock, each prefix caches uniquely.
+					UserDefined:    custom,
+					NoLock:         true, // No need to hold namespace lock, each prefix caches uniquely.
+					ParentIsObject: nil,
 				})
 				if err != nil {
 					metaMu.Lock()

cmd/metrics-router.go

@@ -24,7 +24,9 @@ import (
 )
 
 const (
-	prometheusMetricsPath = "/prometheus/metrics"
+	prometheusMetricsPathLegacy    = "/prometheus/metrics"
+	prometheusMetricsV2ClusterPath = "/v2/metrics/cluster"
+	prometheusMetricsV2NodePath    = "/v2/metrics/node"
 )
 
 // Standard env prometheus auth type
@@ -43,14 +45,17 @@ const (
 func registerMetricsRouter(router *mux.Router) {
 	// metrics router
 	metricsRouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-
 	authType := strings.ToLower(os.Getenv(EnvPrometheusAuthType))
 	switch prometheusAuthType(authType) {
 	case prometheusPublic:
-		metricsRouter.Handle(prometheusMetricsPath, metricsHandler())
+		metricsRouter.Handle(prometheusMetricsPathLegacy, metricsHandler())
+		metricsRouter.Handle(prometheusMetricsV2ClusterPath, metricsServerHandler())
+		metricsRouter.Handle(prometheusMetricsV2NodePath, metricsNodeHandler())
 	case prometheusJWT:
 		fallthrough
 	default:
-		metricsRouter.Handle(prometheusMetricsPath, AuthMiddleware(metricsHandler()))
+		metricsRouter.Handle(prometheusMetricsPathLegacy, AuthMiddleware(metricsHandler()))
+		metricsRouter.Handle(prometheusMetricsV2ClusterPath, AuthMiddleware(metricsServerHandler()))
+		metricsRouter.Handle(prometheusMetricsV2NodePath, AuthMiddleware(metricsNodeHandler()))
 	}
 }

cmd/metrics.go

@@ -51,6 +51,17 @@ var (
 	)
 )
 
+const (
+	healMetricsNamespace = "self_heal"
+	gatewayNamespace     = "gateway"
+	cacheNamespace       = "cache"
+	s3Namespace          = "s3"
+	bucketNamespace      = "bucket"
+	minioNamespace       = "minio"
+	diskNamespace        = "disk"
+	interNodeNamespace   = "internode"
+)
+
 func init() {
 	prometheus.MustRegister(httpRequestsDuration)
 	prometheus.MustRegister(newMinioCollector())
@@ -81,9 +92,10 @@ func (c *minioCollector) Describe(ch chan<- *prometheus.Desc) {
 func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 
 	// Expose MinIO's version information
-	minioVersionInfo.WithLabelValues(Version, CommitID).Set(float64(1.0))
+	minioVersionInfo.WithLabelValues(Version, CommitID).Set(1.0)
 
 	storageMetricsPrometheus(ch)
+	nodeHealthMetricsPrometheus(ch)
 	bucketUsageMetricsPrometheus(ch)
 	networkMetricsPrometheus(ch)
 	httpMetricsPrometheus(ch)
@@ -92,6 +104,26 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 	healingMetricsPrometheus(ch)
 }
 
+func nodeHealthMetricsPrometheus(ch chan<- prometheus.Metric) {
+	nodesUp, nodesDown := GetPeerOnlineCount()
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(minioNamespace, "nodes", "online"),
+			"Total number of MinIO nodes online",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(nodesUp),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(minioNamespace, "nodes", "offline"),
+			"Total number of MinIO nodes offline",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(nodesDown),
+	)
+}
+
 // collects healing specific metrics for MinIO instance in Prometheus specific format
 // and sends to given channel
 func healingMetricsPrometheus(ch chan<- prometheus.Metric) {
@@ -102,7 +134,6 @@ func healingMetricsPrometheus(ch chan<- prometheus.Metric) {
 	if !exists {
 		return
 	}
-	healMetricsNamespace := "self_heal"
 
 	var dur time.Duration
 	if !bgSeq.lastHealActivity.IsZero() {
@@ -172,7 +203,7 @@ func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
 
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("gateway", globalGatewayName, "bytes_received"),
+			prometheus.BuildFQName(gatewayNamespace, globalGatewayName, "bytes_received"),
 			"Total number of bytes received by current MinIO Gateway "+globalGatewayName+" backend",
 			nil, nil),
 		prometheus.CounterValue,
@@ -180,7 +211,7 @@ func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
 	)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("gateway", globalGatewayName, "bytes_sent"),
+			prometheus.BuildFQName(gatewayNamespace, globalGatewayName, "bytes_sent"),
 			"Total number of bytes sent by current MinIO Gateway to "+globalGatewayName+" backend",
 			nil, nil),
 		prometheus.CounterValue,
@@ -189,7 +220,7 @@ func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
 	s := m.GetRequests()
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("gateway", globalGatewayName, "requests"),
+			prometheus.BuildFQName(gatewayNamespace, globalGatewayName, "requests"),
 			"Total number of requests made to "+globalGatewayName+" by current MinIO Gateway",
 			[]string{"method"}, nil),
 		prometheus.CounterValue,
@@ -198,7 +229,7 @@ func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
 	)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("gateway", globalGatewayName, "requests"),
+			prometheus.BuildFQName(gatewayNamespace, globalGatewayName, "requests"),
 			"Total number of requests made to "+globalGatewayName+" by current MinIO Gateway",
 			[]string{"method"}, nil),
 		prometheus.CounterValue,
@@ -207,7 +238,7 @@ func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
 	)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("gateway", globalGatewayName, "requests"),
+			prometheus.BuildFQName(gatewayNamespace, globalGatewayName, "requests"),
 			"Total number of requests made to "+globalGatewayName+" by current MinIO Gateway",
 			[]string{"method"}, nil),
 		prometheus.CounterValue,
@@ -216,7 +247,7 @@ func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
 	)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("gateway", globalGatewayName, "requests"),
+			prometheus.BuildFQName(gatewayNamespace, globalGatewayName, "requests"),
 			"Total number of requests made to "+globalGatewayName+" by current MinIO Gateway",
 			[]string{"method"}, nil),
 		prometheus.CounterValue,
@@ -236,7 +267,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("cache", "hits", "total"),
+			prometheus.BuildFQName(cacheNamespace, "hits", "total"),
 			"Total number of disk cache hits in current MinIO instance",
 			nil, nil),
 		prometheus.CounterValue,
@@ -244,7 +275,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 	)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("cache", "misses", "total"),
+			prometheus.BuildFQName(cacheNamespace, "misses", "total"),
 			"Total number of disk cache misses in current MinIO instance",
 			nil, nil),
 		prometheus.CounterValue,
@@ -252,7 +283,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 	)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("cache", "data", "served"),
+			prometheus.BuildFQName(cacheNamespace, "data", "served"),
 			"Total number of bytes served from cache of current MinIO instance",
 			nil, nil),
 		prometheus.CounterValue,
@@ -262,7 +293,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 		// Cache disk usage percentage
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("cache", "usage", "percent"),
+				prometheus.BuildFQName(cacheNamespace, "usage", "percent"),
 				"Total percentage cache usage",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
@@ -271,7 +302,7 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 		)
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("cache", "usage", "high"),
+				prometheus.BuildFQName(cacheNamespace, "usage", "high"),
 				"Indicates cache usage is high or low, relative to current cache 'quota' settings",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
@@ -309,7 +340,7 @@ func httpMetricsPrometheus(ch chan<- prometheus.Metric) {
 	for api, value := range httpStats.CurrentS3Requests.APIStats {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("s3", "requests", "current"),
+				prometheus.BuildFQName(s3Namespace, "requests", "current"),
 				"Total number of running s3 requests in current MinIO server instance",
 				[]string{"api"}, nil),
 			prometheus.CounterValue,
@@ -321,7 +352,7 @@ func httpMetricsPrometheus(ch chan<- prometheus.Metric) {
 	for api, value := range httpStats.TotalS3Requests.APIStats {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("s3", "requests", "total"),
+				prometheus.BuildFQName(s3Namespace, "requests", "total"),
 				"Total number of s3 requests in current MinIO server instance",
 				[]string{"api"}, nil),
 			prometheus.CounterValue,
@@ -333,7 +364,7 @@ func httpMetricsPrometheus(ch chan<- prometheus.Metric) {
 	for api, value := range httpStats.TotalS3Errors.APIStats {
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("s3", "errors", "total"),
+				prometheus.BuildFQName(s3Namespace, "errors", "total"),
 				"Total number of s3 errors in current MinIO server instance",
 				[]string{"api"}, nil),
 			prometheus.CounterValue,
@@ -351,7 +382,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 	// Network Sent/Received Bytes (internode)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("internode", "tx", "bytes_total"),
+			prometheus.BuildFQName(interNodeNamespace, "tx", "bytes_total"),
 			"Total number of bytes sent to the other peer nodes by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
@@ -360,7 +391,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("internode", "rx", "bytes_total"),
+			prometheus.BuildFQName(interNodeNamespace, "rx", "bytes_total"),
 			"Total number of internode bytes received by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
@@ -370,7 +401,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 	// Network Sent/Received Bytes (Outbound)
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("s3", "tx", "bytes_total"),
+			prometheus.BuildFQName(s3Namespace, "tx", "bytes_total"),
 			"Total number of s3 bytes sent by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
@@ -379,7 +410,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("s3", "rx", "bytes_total"),
+			prometheus.BuildFQName(s3Namespace, "rx", "bytes_total"),
 			"Total number of s3 bytes received by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
@@ -414,7 +445,7 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		// Total space used by bucket
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("bucket", "usage", "size"),
+				prometheus.BuildFQName(bucketNamespace, "usage", "size"),
 				"Total bucket size",
 				[]string{"bucket"}, nil),
 			prometheus.GaugeValue,
@@ -423,7 +454,7 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		)
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("bucket", "objects", "count"),
+				prometheus.BuildFQName(bucketNamespace, "objects", "count"),
 				"Total number of objects in a bucket",
 				[]string{"bucket"}, nil),
 			prometheus.GaugeValue,
@@ -469,7 +500,7 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		for k, v := range usageInfo.ObjectSizesHistogram {
 			ch <- prometheus.MustNewConstMetric(
 				prometheus.NewDesc(
-					prometheus.BuildFQName("bucket", "objects", "histogram"),
+					prometheus.BuildFQName(bucketNamespace, "objects", "histogram"),
 					"Total number of objects of different sizes in a bucket",
 					[]string{"bucket", "object_size"}, nil),
 				prometheus.GaugeValue,
@@ -497,10 +528,50 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 	onlineDisks, offlineDisks := getOnlineOfflineDisksStats(server.Disks)
 	totalDisks := offlineDisks.Merge(onlineDisks)
 
+	// Report total capacity
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(minioNamespace, "capacity_raw", "total"),
+			"Total capacity online in the cluster",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(GetTotalCapacity(GlobalContext)),
+	)
+
+	// Report total capacity free
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(minioNamespace, "capacity_raw_free", "total"),
+			"Total free capacity online in the cluster",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(GetTotalCapacityFree(GlobalContext)),
+	)
+
+	s, _ := objLayer.StorageInfo(GlobalContext)
+	// Report total usable capacity
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(minioNamespace, "capacity_usable", "total"),
+			"Total usable capacity online in the cluster",
+			nil, nil),
+		prometheus.GaugeValue,
+		GetTotalUsableCapacity(GlobalContext, s),
+	)
+	// Report total usable capacity free
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(minioNamespace, "capacity_usable_free", "total"),
+			"Total free usable capacity online in the cluster",
+			nil, nil),
+		prometheus.GaugeValue,
+		GetTotalUsableCapacityFree(GlobalContext, s),
+	)
+
 	// MinIO Offline Disks per node
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "disks", "offline"),
+			prometheus.BuildFQName(minioNamespace, "disks", "offline"),
 			"Total number of offline disks in current MinIO server instance",
 			nil, nil),
 		prometheus.GaugeValue,
@@ -510,7 +581,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 	// MinIO Total Disks per node
 	ch <- prometheus.MustNewConstMetric(
 		prometheus.NewDesc(
-			prometheus.BuildFQName("minio", "disks", "total"),
+			prometheus.BuildFQName(minioNamespace, "disks", "total"),
 			"Total number of disks for current MinIO server instance",
 			nil, nil),
 		prometheus.GaugeValue,
@@ -521,7 +592,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		// Total disk usage by the disk
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("disk", "storage", "used"),
+				prometheus.BuildFQName(diskNamespace, "storage", "used"),
 				"Total disk storage used on the disk",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
@@ -532,7 +603,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		// Total available space in the disk
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("disk", "storage", "available"),
+				prometheus.BuildFQName(diskNamespace, "storage", "available"),
 				"Total available space left on the disk",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,
@@ -543,7 +614,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		// Total storage space of the disk
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(
-				prometheus.BuildFQName("disk", "storage", "total"),
+				prometheus.BuildFQName(diskNamespace, "storage", "total"),
 				"Total space on the disk",
 				[]string{"disk"}, nil),
 			prometheus.GaugeValue,

cmd/namespace-lock_test.go

@@ -41,9 +41,7 @@ func TestGetSource(t *testing.T) {
 
 // Test lock race
 func TestNSLockRace(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping test in short mode.")
-	}
+	t.Skip("long test skip it")
 
 	ctx := context.Background()