update console to v0.9.1

This free-version id may be used to track tiered object contents of the
object (version) being deleted.

CONTRIBUTING.md

@@ -56,7 +56,7 @@ $ git push origin my-new-feature
 Pull requests can be created via GitHub. Refer to [this document](https://help.github.com/articles/creating-a-pull-request/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged.
 
 ## FAQs
-### How does ``MinIO`` manages dependencies?
+### How does ``MinIO`` manage dependencies?
 ``MinIO`` uses `go mod` to manage its dependencies.
 - Run `go get foo/bar` in the source folder to add the dependency to `go.mod` file.
 

README.md

@@ -259,4 +259,5 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 # License
-Use of MinIO is governed by the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
+MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file. 
+MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).

cmd/admin-bucket-handlers.go

@@ -85,7 +85,7 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -122,7 +122,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
-	update := r.URL.Query().Get("update") == "true"
+	update := r.Form.Get("update") == "true"
 
 	if !globalIsErasure {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -130,7 +130,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -169,7 +169,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	target.SourceBucket = bucket
 	var ops []madmin.TargetUpdateType
 	if update {
-		ops = madmin.GetTargetUpdateOps(r.URL.Query())
+		ops = madmin.GetTargetUpdateOps(r.Form)
 	} else {
 		target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
 	}
@@ -258,7 +258,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 		return
 	}
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketTargetAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -298,7 +298,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 		return
 	}
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.SetBucketTargetAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return

cmd/admin-handler-utils.go

@@ -0,0 +1,172 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/minio/kes"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return nil, auth.Credentials{}
+	}
+
+	// Validate request signature.
+	cred, adminAPIErr := checkAdminRequestAuth(ctx, r, action, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return nil, cred
+	}
+
+	return objectAPI, cred
+}
+
+// AdminError - is a generic error for all admin APIs.
+type AdminError struct {
+	Code       string
+	Message    string
+	StatusCode int
+}
+
+func (ae AdminError) Error() string {
+	return ae.Message
+}
+
+func toAdminAPIErr(ctx context.Context, err error) APIError {
+	if err == nil {
+		return noError
+	}
+
+	var apiErr APIError
+	switch e := err.(type) {
+	case iampolicy.Error:
+		apiErr = APIError{
+			Code:           "XMinioMalformedIAMPolicy",
+			Description:    e.Error(),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	case config.Error:
+		apiErr = APIError{
+			Code:           "XMinioConfigError",
+			Description:    e.Error(),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	case AdminError:
+		apiErr = APIError{
+			Code:           e.Code,
+			Description:    e.Message,
+			HTTPStatusCode: e.StatusCode,
+		}
+	default:
+		switch {
+		case errors.Is(err, errConfigNotFound):
+			apiErr = APIError{
+				Code:           "XMinioConfigError",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusNotFound,
+			}
+		case errors.Is(err, errIAMActionNotAllowed):
+			apiErr = APIError{
+				Code:           "XMinioIAMActionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusForbidden,
+			}
+		case errors.Is(err, errIAMNotInitialized):
+			apiErr = APIError{
+				Code:           "XMinioIAMNotInitialized",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusServiceUnavailable,
+			}
+		case errors.Is(err, kes.ErrKeyExists):
+			apiErr = APIError{
+				Code:           "XMinioKMSKeyExists",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusConflict,
+			}
+
+		// Tier admin API errors
+		case errors.Is(err, madmin.ErrTierNameEmpty):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierNameEmpty",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, madmin.ErrTierInvalidConfig):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierInvalidConfig",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, madmin.ErrTierInvalidConfigVersion):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierInvalidConfigVersion",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, madmin.ErrTierTypeUnsupported):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierTypeUnsupported",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errors.Is(err, errTierBackendInUse):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierBackendInUse",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusConflict,
+			}
+		case errors.Is(err, errTierInsufficientCreds):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierInsufficientCreds",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		case errIsTierPermError(err):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierInsufficientPermissions",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		default:
+			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
+		}
+	}
+	return apiErr
+}
+
+// toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
+// specific error.
+func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
+	switch err {
+	case errErasureWriteQuorum:
+		return ErrAdminConfigNoQuorum
+	default:
+		return toAPIErrorCode(ctx, err)
+	}
+}

cmd/admin-handlers-config-kv.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -28,7 +27,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/etcd"
@@ -40,31 +38,13 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
-func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return auth.Credentials{}, nil
-	}
-
-	// Validate request signature.
-	cred, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
-		return cred, nil
-	}
-
-	return cred, objectAPI
-}
-
 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteConfigKV")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -106,7 +86,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -173,7 +153,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -202,7 +182,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -239,7 +219,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -287,7 +267,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -327,7 +307,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -337,7 +317,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 	subSys := vars["subSys"]
 	key := vars["key"]
 
-	_, envOnly := r.URL.Query()["env"]
+	_, envOnly := r.Form["env"]
 
 	rd, err := GetHelp(subSys, key, envOnly)
 	if err != nil {
@@ -355,7 +335,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -407,7 +387,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers-users.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"errors"
 	"io"
@@ -29,40 +28,18 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
-func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
-	var cred auth.Credentials
-	var adminAPIErr APIErrorCode
-
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return nil, cred
-	}
-
-	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
-		return nil, cred
-	}
-
-	return objectAPI, cred
-}
-
 // RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -100,7 +77,7 @@ func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -136,7 +113,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -234,7 +211,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -279,7 +256,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -308,7 +285,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -334,7 +311,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -371,7 +348,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -562,10 +539,12 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
+	var ldapUsername string
 	if globalLDAPConfig.Enabled && targetUser != "" {
 		// If LDAP enabled, service accounts need
 		// to be created only for LDAP users.
 		var err error
+		ldapUsername = targetUser
 		targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -604,6 +583,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		secretKey:     createReq.SecretKey,
 		sessionPolicy: sp,
 	}
+	if ldapUsername != "" {
+		opts.ldapUsername = ldapUsername
+	}
 	newCred, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -846,7 +828,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 
 	var targetAccount string
 
-	user := r.URL.Query().Get("user")
+	user := r.Form.Get("user")
 	if user != "" {
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
@@ -992,7 +974,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	r.Header.Set("delimiter", SlashSeparator)
 
 	// Check if we are asked to return prefix usage
-	enablePrefixUsage := r.URL.Query().Get("prefix-usage") == "true"
+	enablePrefixUsage := r.Form.Get("prefix-usage") == "true"
 
 	isAllowedAccess := func(bucketName string) (rd, wr bool) {
 		if globalIAMSys.IsAllowed(iampolicy.Args{
@@ -1139,7 +1121,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1164,7 +1146,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1200,7 +1182,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1234,7 +1216,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1262,7 +1244,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1314,7 +1296,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers.go

@@ -23,7 +23,6 @@ import (
 	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -39,12 +38,10 @@ import (
 	"strings"
 	"time"
 
+	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
@@ -449,7 +446,7 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 	}
 
 	count := 10 // by default list only top 10 entries
-	if countStr := r.URL.Query().Get("count"); countStr != "" {
+	if countStr := r.Form.Get("count"); countStr != "" {
 		var err error
 		count, err = strconv.Atoi(countStr)
 		if err != nil {
@@ -457,7 +454,7 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 			return
 		}
 	}
-	stale := r.URL.Query().Get("stale") == "true" // list also stale locks
+	stale := r.Form.Get("stale") == "true" // list also stale locks
 
 	peerLocks := globalNotificationSys.GetLocks(ctx, r)
 
@@ -712,7 +709,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	hip, errCode := extractHealInitParams(mux.Vars(r), r.URL.Query(), r.Body)
+	hip, errCode := extractHealInitParams(mux.Vars(r), r.Form, r.Body)
 	if errCode != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
 		return
@@ -910,35 +907,50 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 }
 
-func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
-	var cred auth.Credentials
-	var adminAPIErr APIErrorCode
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return nil, cred
+func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SpeedtestHandler")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	if objectAPI == nil {
+		return
 	}
 
-	// Validate request signature.
-	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
-		return nil, cred
+	if !globalIsErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
 	}
 
-	return objectAPI, cred
-}
+	sizeStr := r.Form.Get(peerRESTSize)
+	durationStr := r.Form.Get(peerRESTDuration)
+	concurrentStr := r.Form.Get(peerRESTConcurrent)
 
-// AdminError - is a generic error for all admin APIs.
-type AdminError struct {
-	Code       string
-	Message    string
-	StatusCode int
-}
+	size, err := strconv.Atoi(sizeStr)
+	if err != nil {
+		size = 64 * humanize.MiByte
+	}
 
-func (ae AdminError) Error() string {
-	return ae.Message
+	concurrent, err := strconv.Atoi(concurrentStr)
+	if err != nil {
+		concurrent = 32
+	}
+
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		duration = time.Second * 10
+	}
+
+	results := globalNotificationSys.Speedtest(ctx, size, concurrent, duration)
+
+	if err := json.NewEncoder(w).Encode(results); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), true)
+
+	w.(http.Flusher).Flush()
 }
 
 // Admin API errors
@@ -948,119 +960,6 @@ const (
 	AdminUpdateApplyFailure      = "XMinioAdminUpdateApplyFailure"
 )
 
-// toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
-// specific error.
-func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
-	switch err {
-	case errErasureWriteQuorum:
-		return ErrAdminConfigNoQuorum
-	default:
-		return toAPIErrorCode(ctx, err)
-	}
-}
-
-func toAdminAPIErr(ctx context.Context, err error) APIError {
-	if err == nil {
-		return noError
-	}
-
-	var apiErr APIError
-	switch e := err.(type) {
-	case iampolicy.Error:
-		apiErr = APIError{
-			Code:           "XMinioMalformedIAMPolicy",
-			Description:    e.Error(),
-			HTTPStatusCode: http.StatusBadRequest,
-		}
-	case config.Error:
-		apiErr = APIError{
-			Code:           "XMinioConfigError",
-			Description:    e.Error(),
-			HTTPStatusCode: http.StatusBadRequest,
-		}
-	case AdminError:
-		apiErr = APIError{
-			Code:           e.Code,
-			Description:    e.Message,
-			HTTPStatusCode: e.StatusCode,
-		}
-	default:
-		switch {
-		case errors.Is(err, errConfigNotFound):
-			apiErr = APIError{
-				Code:           "XMinioConfigError",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusNotFound,
-			}
-		case errors.Is(err, errIAMActionNotAllowed):
-			apiErr = APIError{
-				Code:           "XMinioIAMActionNotAllowed",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusForbidden,
-			}
-		case errors.Is(err, errIAMNotInitialized):
-			apiErr = APIError{
-				Code:           "XMinioIAMNotInitialized",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusServiceUnavailable,
-			}
-		case errors.Is(err, kes.ErrKeyExists):
-			apiErr = APIError{
-				Code:           "XMinioKMSKeyExists",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusConflict,
-			}
-
-		// Tier admin API errors
-		case errors.Is(err, madmin.ErrTierNameEmpty):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierNameEmpty",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, madmin.ErrTierInvalidConfig):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierInvalidConfig",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, madmin.ErrTierInvalidConfigVersion):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierInvalidConfigVersion",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, madmin.ErrTierTypeUnsupported):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierTypeUnsupported",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, errTierBackendInUse):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierBackendInUse",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusConflict,
-			}
-		case errors.Is(err, errTierInsufficientCreds):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierInsufficientCreds",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errIsTierPermError(err):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierInsufficientPermissions",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		default:
-			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
-		}
-	}
-	return apiErr
-}
-
 // Returns true if the madmin.TraceInfo should be traced,
 // false if certain conditions are not met.
 // - input entry is not of the type *madmin.TraceInfo*
@@ -1110,7 +1009,7 @@ func mustTrace(entry interface{}, opts madmin.ServiceTraceOpts) (shouldTrace boo
 }
 
 func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err error) {
-	q := r.URL.Query()
+	q := r.Form
 
 	opts.OnlyErrors = q.Get("err") == "true"
 	opts.S3 = q.Get("s3") == "true"
@@ -1212,15 +1111,15 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	if objectAPI == nil {
 		return
 	}
-	node := r.URL.Query().Get("node")
+	node := r.Form.Get("node")
 	// limit buffered console entries if client requested it.
-	limitStr := r.URL.Query().Get("limit")
+	limitStr := r.Form.Get("limit")
 	limitLines, err := strconv.Atoi(limitStr)
 	if err != nil {
 		limitLines = 10
 	}
 
-	logKind := r.URL.Query().Get("logType")
+	logKind := r.Form.Get("logType")
 	if logKind == "" {
 		logKind = string(logger.All)
 	}
@@ -1295,7 +1194,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if err := GlobalKMS.CreateKey(r.URL.Query().Get("key-id")); err != nil {
+	if err := GlobalKMS.CreateKey(r.Form.Get("key-id")); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1362,7 +1261,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	keyID := r.URL.Query().Get("key-id")
+	keyID := r.Form.Get("key-id")
 	if keyID == "" {
 		keyID = stat.DefaultKey
 	}
@@ -1529,7 +1428,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	query := r.URL.Query()
+	query := r.Form
 	healthInfo := madmin.HealthInfo{Version: madmin.HealthInfoVersion}
 	healthInfoCh := make(chan madmin.HealthInfo)
 
@@ -1553,7 +1452,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	deadline := 1 * time.Hour
-	if dstr := r.URL.Query().Get("deadline"); dstr != "" {
+	if dstr := r.Form.Get("deadline"); dstr != "" {
 		var err error
 		deadline, err = time.ParseDuration(dstr)
 		if err != nil {
@@ -1652,6 +1551,38 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
+	getAndWriteSysErrors := func() {
+		if query.Get(string(madmin.HealthDataTypeSysErrors)) == "true" {
+			localSysErrors := madmin.GetSysErrors(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localSysErrors)
+			healthInfo.Sys.SysErrs = append(healthInfo.Sys.SysErrs, localSysErrors)
+			partialWrite(healthInfo)
+
+			peerSysErrs := globalNotificationSys.GetSysErrors(deadlinedCtx)
+			for _, se := range peerSysErrs {
+				anonymizeAddr(&se)
+				healthInfo.Sys.SysErrs = append(healthInfo.Sys.SysErrs, se)
+			}
+			partialWrite(healthInfo)
+		}
+	}
+
+	getAndWriteSysServices := func() {
+		if query.Get(string(madmin.HealthDataTypeSysServices)) == "true" {
+			localSysServices := madmin.GetSysServices(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localSysServices)
+			healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, localSysServices)
+			partialWrite(healthInfo)
+
+			peerSysServices := globalNotificationSys.GetSysServices(deadlinedCtx)
+			for _, sli := range peerSysServices {
+				anonymizeAddr(&sli)
+				healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, sli)
+			}
+			partialWrite(healthInfo)
+		}
+	}
+
 	anonymizeCmdLine := func(cmdLine string) string {
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
@@ -1825,6 +1756,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		getAndWriteMinioConfig()
 		getAndWriteDrivePerfInfo()
 		getAndWriteNetPerfInfo()
+		getAndWriteSysErrors()
+		getAndWriteSysServices()
 
 		if query.Get("minioinfo") == "true" {
 			infoMessage := getServerInfo(ctx, r)
@@ -1911,7 +1844,7 @@ func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http
 	reportCh := make(chan madmin.BucketBandwidthReport)
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
-	bucketsRequestedString := r.URL.Query().Get("buckets")
+	bucketsRequestedString := r.Form.Get("buckets")
 	bucketsRequested := strings.Split(bucketsRequestedString, ",")
 	go func() {
 		defer close(reportCh)
@@ -2169,8 +2102,8 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	volume := r.URL.Query().Get("volume")
-	file := r.URL.Query().Get("file")
+	volume := r.Form.Get("volume")
+	file := r.Form.Get("file")
 	if len(volume) == 0 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
 		return
@@ -2252,12 +2185,13 @@ func createHostAnonymizerForFSMode() map[string]string {
 
 	apiEndpoints := getAPIEndpoints()
 	for _, ep := range apiEndpoints {
-		if len(ep) > 0 {
-			if url, err := xnet.ParseHTTPURL(ep); err == nil {
-				// In FS mode the drive names don't include the host.
-				// So mapping just the host should be sufficient.
-				hostAnonymizer[url.Host] = "server1"
-			}
+		if len(ep) == 0 {
+			continue
+		}
+		if url, err := xnet.ParseHTTPURL(ep); err == nil {
+			// In FS mode the drive names don't include the host.
+			// So mapping just the host should be sufficient.
+			hostAnonymizer[url.Host] = "server1"
 		}
 	}
 	return hostAnonymizer

cmd/admin-handlers_test.go

@@ -182,7 +182,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 
 	adminTestBed, err := prepareAdminErasureTestBed(ctx)
 	if err != nil {
-		t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.")
+		t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.", err)
 	}
 	defer adminTestBed.TearDown()
 
@@ -253,7 +253,7 @@ func TestAdminServerInfo(t *testing.T) {
 
 	adminTestBed, err := prepareAdminErasureTestBed(ctx)
 	if err != nil {
-		t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.")
+		t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.", err)
 	}
 
 	defer adminTestBed.TearDown()

cmd/admin-router.go

@@ -24,6 +24,7 @@ import (
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/logger"
 )
 
 const (
@@ -48,16 +49,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminAPIVersionPrefix,
 	}
 
-	gz := func(h http.HandlerFunc) http.HandlerFunc {
-		return h
+	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
+	if err != nil {
+		// Static params, so this is very unlikely.
+		logger.Fatal(err, "Unable to initialize server")
 	}
 
-	wrapper, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
-	if err == nil {
-		gz = func(h http.HandlerFunc) http.HandlerFunc {
-			return wrapper(h).(http.HandlerFunc)
-		}
-	}
 	for _, adminVersion := range adminVersions {
 		// Restart and stop MinIO service.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(gz(httpTraceAll(adminAPI.ServiceHandler))).Queries("action", "{action:.*}")
@@ -204,8 +201,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 				Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
 		}
 
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
+
 		// HTTP Trace
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(adminAPI.TraceHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(http.HandlerFunc(adminAPI.TraceHandler)))
 
 		// Console Logs
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(gz(httpTraceAll(adminAPI.ConsoleLogHandler)))

cmd/api-errors.go

@@ -231,7 +231,6 @@ const (
 	// MinIO extended errors.
 	ErrReadQuorum
 	ErrWriteQuorum
-	ErrParentIsObject
 	ErrStorageFull
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
@@ -1122,11 +1121,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
-	ErrParentIsObject: {
-		Code:           "XMinioParentIsObject",
-		Description:    "Object-prefix is already an object, please choose a different object-prefix name.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrRequestBodyParse: {
 		Code:           "XMinioRequestBodyParse",
 		Description:    "The request body failed to parse.",
@@ -1899,8 +1893,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectExistsAsDirectory
 	case PrefixAccessDenied:
 		apiErr = ErrAccessDenied
-	case ParentIsObject:
-		apiErr = ErrParentIsObject
 	case BucketNameInvalid:
 		apiErr = ErrInvalidBucketName
 	case BucketNotFound:

cmd/api-headers.go

@@ -184,12 +184,13 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.AmzBucketReplicationStatus] = []string{objInfo.ReplicationStatus.String()}
 	}
 
+	if objInfo.IsRemote() {
+		// Check if object is being restored. For more information on x-amz-restore header see
+		// https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax
+		w.Header()[xhttp.AmzStorageClass] = []string{objInfo.TransitionedObject.Tier}
+	}
+
 	if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
-		if objInfo.IsRemote() {
-			// Check if object is being restored. For more information on x-amz-restore header see
-			// https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax
-			w.Header()[xhttp.AmzStorageClass] = []string{objInfo.TransitionTier}
-		}
 		lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
 	}
 

cmd/api-response.go

@@ -36,11 +36,11 @@ import (
 
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	iso8601TimeFormat = "2006-01-02T15:04:05.000Z"                     // Reply date format with nanosecond precision.
-	maxObjectList     = metacacheBlockSize - (metacacheBlockSize / 10) // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                                          // Limit number of objects deleted in a delete call.
-	maxUploadsList    = 10000                                          // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 10000                                          // Limit number of parts in a listPartsResponse.
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
+	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
+	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
+	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.

cmd/api-router.go

@@ -25,6 +25,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/gzhttp"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/wildcard"
 	"github.com/rs/cors"
 )
@@ -208,15 +209,10 @@ func registerAPIRouter(router *mux.Router) {
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	gz := func(h http.HandlerFunc) http.HandlerFunc {
-		return h
-	}
-
-	wrapper, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
-	if err == nil {
-		gz = func(h http.HandlerFunc) http.HandlerFunc {
-			return wrapper(h).(http.HandlerFunc)
-		}
+	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
+	if err != nil {
+		// Static params, so this is very unlikely.
+		logger.Fatal(err, "Unable to initialize server")
 	}
 
 	for _, router := range routers {

cmd/auth-handler.go

@@ -27,6 +27,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -61,13 +62,13 @@ func isRequestSignatureV2(r *http.Request) bool {
 
 // Verify if request has AWS PreSign Version '4'.
 func isRequestPresignedSignatureV4(r *http.Request) bool {
-	_, ok := r.URL.Query()[xhttp.AmzCredential]
+	_, ok := r.Form[xhttp.AmzCredential]
 	return ok
 }
 
 // Verify request has AWS PreSign Version '2'.
 func isRequestPresignedSignatureV2(r *http.Request) bool {
-	_, ok := r.URL.Query()[xhttp.AmzAccessKeyID]
+	_, ok := r.Form[xhttp.AmzAccessKeyID]
 	return ok
 }
 
@@ -102,6 +103,14 @@ const (
 
 // Get request authentication type.
 func getRequestAuthType(r *http.Request) authType {
+	if r.URL != nil {
+		var err error
+		r.Form, err = url.ParseQuery(r.URL.RawQuery)
+		if err != nil {
+			logger.LogIf(r.Context(), err)
+			return authTypeUnknown
+		}
+	}
 	if isRequestSignatureV2(r) {
 		return authTypeSignedV2
 	} else if isRequestPresignedSignatureV2(r) {
@@ -116,7 +125,7 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypeJWT
 	} else if isRequestPostPolicySignatureV4(r) {
 		return authTypePostPolicy
-	} else if _, ok := r.URL.Query()[xhttp.Action]; ok {
+	} else if _, ok := r.Form[xhttp.Action]; ok {
 		return authTypeSTS
 	} else if _, ok := r.Header[xhttp.Authorization]; !ok {
 		return authTypeAnonymous
@@ -146,12 +155,7 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 		return cred, nil, owner, s3Err
 	}
 
-	claims, s3Err := checkClaimsFromToken(r, cred)
-	if s3Err != ErrNone {
-		return cred, nil, owner, s3Err
-	}
-
-	return cred, claims, owner, ErrNone
+	return cred, cred.Claims, owner, ErrNone
 }
 
 // checkAdminRequestAuth checks for authentication and authorization for the incoming
@@ -183,7 +187,7 @@ func getSessionToken(r *http.Request) (token string) {
 	if token != "" {
 		return token
 	}
-	return r.URL.Query().Get(xhttp.AmzSecurityToken)
+	return r.Form.Get(xhttp.AmzSecurityToken)
 }
 
 // Fetch claims in the security token returned by the client, doesn't return
@@ -213,27 +217,17 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 		return nil, errAuthentication
 	}
 
-	if globalPolicyOPA == nil {
-		// If OPA is not set and if ldap claim key is set, allow the claim.
-		if _, ok := claims.MapClaims[ldapUser]; ok {
-			return claims.Map(), nil
-		}
+	// If OPA is set, return without any further checks.
+	if globalPolicyOPA != nil {
+		return claims.Map(), nil
+	}
 
-		// If OPA is not set, session token should
-		// have a policy and its mandatory, reject
-		// requests without policy claim.
-		_, pokOpenID := claims.MapClaims[iamPolicyClaimNameOpenID()]
-		_, pokSA := claims.MapClaims[iamPolicyClaimNameSA()]
-		if !pokOpenID && !pokSA {
-			return nil, errAuthentication
-		}
-
-		sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
-		if !spok {
-			return claims.Map(), nil
-		}
+	// Check if a session policy is set. If so, decode it here.
+	sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
+	if spok {
 		// Looks like subpolicy is set and is a string, if set then its
-		// base64 encoded, decode it. Decoding fails reject such requests.
+		// base64 encoded, decode it. Decoding fails reject such
+		// requests.
 		spBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
@@ -244,6 +238,19 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
 
+	// If LDAP claim key is set, return here.
+	if _, ok := claims.MapClaims[ldapUser]; ok {
+		return claims.Map(), nil
+	}
+
+	// Session token must have a policy, reject requests without policy
+	// claim.
+	_, pokOpenID := claims.MapClaims[iamPolicyClaimNameOpenID()]
+	_, pokSA := claims.MapClaims[iamPolicyClaimNameSA()]
+	if !pokOpenID && !pokSA {
+		return nil, errAuthentication
+	}
+
 	return claims.Map(), nil
 }
 
@@ -306,12 +313,6 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 		return cred, owner, s3Err
 	}
 
-	var claims map[string]interface{}
-	claims, s3Err = checkClaimsFromToken(r, cred)
-	if s3Err != ErrNone {
-		return cred, owner, s3Err
-	}
-
 	// LocationConstraint is valid only for CreateBucketAction.
 	var locationConstraint string
 	if action == policy.CreateBucketAction {
@@ -376,10 +377,10 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 		Groups:          cred.Groups,
 		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
 		ObjectName:      objectName,
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		// Request is allowed return the appropriate access key.
 		return cred, owner, ErrNone
@@ -393,10 +394,10 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 			Groups:          cred.Groups,
 			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
 			ObjectName:      objectName,
 			IsOwner:         owner,
-			Claims:          claims,
+			Claims:          cred.Claims,
 		}) {
 			// Request is allowed return the appropriate access key.
 			return cred, owner, ErrNone
@@ -441,7 +442,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 	// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
 	var contentSHA256 []byte
 	if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
-		if sha256Sum, ok := r.URL.Query()[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
+		if sha256Sum, ok := r.Form[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
 			contentSHA256, err = hex.DecodeString(sha256Sum[0])
 			if err != nil {
 				return ErrContentSHA256Mismatch
@@ -508,75 +509,39 @@ func setAuthHandler(h http.Handler) http.Handler {
 	})
 }
 
-func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, map[string]interface{}, APIErrorCode) {
+func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	var s3Err APIErrorCode
 	switch atype {
 	case authTypeUnknown, authTypeStreamingSigned:
-		return cred, owner, nil, ErrSignatureVersionNotSupported
+		return cred, owner, ErrSignatureVersionNotSupported
 	case authTypeSignedV2, authTypePresignedV2:
 		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
-			return cred, owner, nil, s3Err
+			return cred, owner, s3Err
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypePresigned, authTypeSigned:
 		region := globalServerRegion
 		if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
-			return cred, owner, nil, s3Err
+			return cred, owner, s3Err
 		}
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
-		return cred, owner, nil, s3Err
+		return cred, owner, s3Err
 	}
 
-	claims, s3Err := checkClaimsFromToken(r, cred)
-	if s3Err != ErrNone {
-		return cred, owner, nil, s3Err
-	}
-
-	return cred, owner, claims, ErrNone
+	return cred, owner, ErrNone
 }
 
-func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool, claims map[string]interface{}) (s3Err APIErrorCode) {
+func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) {
 	var retSet bool
 	if cred.AccessKey == "" {
-		conditions := getConditionValues(r, "", "", nil)
-		conditions["object-lock-mode"] = []string{string(retMode)}
-		conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
-		if retDays > 0 {
-			conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
-		}
-		if retMode == objectlock.RetGovernance && byPassSet {
-			byPassSet = globalPolicySys.IsAllowed(policy.Args{
-				AccountName:     cred.AccessKey,
-				Groups:          cred.Groups,
-				Action:          policy.BypassGovernanceRetentionAction,
-				BucketName:      bucketName,
-				ConditionValues: conditions,
-				IsOwner:         false,
-				ObjectName:      objectName,
-			})
-		}
-		if globalPolicySys.IsAllowed(policy.Args{
-			AccountName:     cred.AccessKey,
-			Groups:          cred.Groups,
-			Action:          policy.PutObjectRetentionAction,
-			BucketName:      bucketName,
-			ConditionValues: conditions,
-			IsOwner:         false,
-			ObjectName:      objectName,
-		}) {
-			retSet = true
-		}
-		if byPassSet || retSet {
-			return ErrNone
-		}
 		return ErrAccessDenied
 	}
 
-	conditions := getConditionValues(r, "", cred.AccessKey, claims)
+	conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
 	conditions["object-lock-mode"] = []string{string(retMode)}
 	conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
 	if retDays > 0 {
@@ -591,7 +556,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 			ObjectName:      objectName,
 			ConditionValues: conditions,
 			IsOwner:         owner,
-			Claims:          claims,
+			Claims:          cred.Claims,
 		})
 	}
 	if globalIAMSys.IsAllowed(iampolicy.Args{
@@ -602,7 +567,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		ConditionValues: conditions,
 		ObjectName:      objectName,
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		retSet = true
 	}
@@ -631,11 +596,6 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 		return s3Err
 	}
 
-	claims, s3Err := checkClaimsFromToken(r, cred)
-	if s3Err != ErrNone {
-		return s3Err
-	}
-
 	if cred.AccessKey != "" {
 		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
 	}
@@ -669,10 +629,10 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 		Groups:          cred.Groups,
 		Action:          action,
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
 		ObjectName:      objectName,
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		return ErrNone
 	}

cmd/auth-handler_test.go

@@ -38,6 +38,7 @@ func TestGetRequestAuthType(t *testing.T) {
 		req   *http.Request
 		authT authType
 	}
+	nopCloser := ioutil.NopCloser(io.LimitReader(&nullReader{}, 1024))
 	testCases := []testCase{
 		// Test case - 1
 		// Check for generic signature v4 header.
@@ -54,6 +55,7 @@ func TestGetRequestAuthType(t *testing.T) {
 					"Content-Encoding":     []string{streamingContentEncoding},
 				},
 				Method: http.MethodPut,
+				Body:   nopCloser,
 			},
 			authT: authTypeStreamingSigned,
 		},
@@ -113,6 +115,7 @@ func TestGetRequestAuthType(t *testing.T) {
 					"Content-Type": []string{"multipart/form-data"},
 				},
 				Method: http.MethodPost,
+				Body:   nopCloser,
 			},
 			authT: authTypePostPolicy,
 		},
@@ -220,6 +223,7 @@ func TestIsRequestPresignedSignatureV2(t *testing.T) {
 		q := inputReq.URL.Query()
 		q.Add(testCase.inputQueryKey, testCase.inputQueryValue)
 		inputReq.URL.RawQuery = q.Encode()
+		inputReq.ParseForm()
 
 		actualResult := isRequestPresignedSignatureV2(inputReq)
 		if testCase.expectedResult != actualResult {
@@ -254,6 +258,7 @@ func TestIsRequestPresignedSignatureV4(t *testing.T) {
 		q := inputReq.URL.Query()
 		q.Add(testCase.inputQueryKey, testCase.inputQueryValue)
 		inputReq.URL.RawQuery = q.Encode()
+		inputReq.ParseForm()
 
 		actualResult := isRequestPresignedSignatureV4(inputReq)
 		if testCase.expectedResult != actualResult {

cmd/bucket-handlers.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"bytes"
-	"crypto/subtle"
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
@@ -246,7 +245,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.URL.Query())
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.Form)
 	if errCode != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
 		return
@@ -345,9 +344,6 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		// Set delimiter value for "s3:delimiter" policy conditionals.
 		r.Header.Set("delimiter", SlashSeparator)
 
-		// err will be nil here as we already called this function
-		// earlier in this request.
-		claims, _ := getClaimsFromToken(getSessionToken(r))
 		n := 0
 		// Use the following trick to filter in place
 		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
@@ -357,10 +353,10 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				Groups:          cred.Groups,
 				Action:          iampolicy.ListBucketAction,
 				BucketName:      bucketInfo.Name,
-				ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
 				IsOwner:         owner,
 				ObjectName:      "",
-				Claims:          claims,
+				Claims:          cred.Claims,
 			}) {
 				bucketsInfo[n] = bucketInfo
 				n++
@@ -454,6 +450,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if api.CacheAPI() != nil {
 		getObjectInfoFn = api.CacheAPI().GetObjectInfo
 	}
+
 	var (
 		hasLockEnabled, replicateSync bool
 		goi                           ObjectInfo
@@ -464,6 +461,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		hasLockEnabled = true
 	}
 
+	versioned := globalBucketVersioningSys.Enabled(bucket)
+	suspended := globalBucketVersioningSys.Suspended(bucket)
+
 	dErrs := make([]DeleteError, len(deleteObjects.Objects))
 	oss := make([]*objSweeper, len(deleteObjects.Objects))
 	for index, object := range deleteObjects.Objects {
@@ -495,16 +495,24 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 		}
 
-		oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(multiDelete(object))
-		// Mutations of objects on versioning suspended buckets
-		// affect its null version. Through opts below we select
-		// the null version's remote object to delete if
-		// transitioned.
-		opts := oss[index].GetOpts()
-		goi, gerr = getObjectInfoFn(ctx, bucket, object.ObjectName, opts)
-		if gerr == nil {
-			oss[index].SetTransitionState(goi)
+		opts := ObjectOptions{
+			VersionID:        object.VersionID,
+			Versioned:        versioned,
+			VersionSuspended: suspended,
 		}
+
+		if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() {
+			if !globalTierConfigMgr.Empty() && object.VersionID == "" && opts.VersionSuspended {
+				opts.VersionID = nullVersionID
+			}
+			goi, gerr = getObjectInfoFn(ctx, bucket, object.ObjectName, opts)
+		}
+
+		if !globalTierConfigMgr.Empty() {
+			oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(opts.VersionID).WithVersioning(versioned, suspended)
+			oss[index].SetTransitionState(goi.TransitionedObject)
+		}
+
 		if replicateDeletes {
 			replicate, repsync := checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectName: object.ObjectName,
@@ -526,18 +534,16 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				}
 			}
 		}
-		if object.VersionID != "" {
-			if hasLockEnabled {
-				if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
-					apiErr := errorCodes.ToAPIErr(apiErrCode)
-					dErrs[index] = DeleteError{
-						Code:      apiErr.Code,
-						Message:   apiErr.Description,
-						Key:       object.ObjectName,
-						VersionID: object.VersionID,
-					}
-					continue
+		if object.VersionID != "" && hasLockEnabled {
+			if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
+				apiErr := errorCodes.ToAPIErr(apiErrCode)
+				dErrs[index] = DeleteError{
+					Code:      apiErr.Code,
+					Message:   apiErr.Description,
+					Key:       object.ObjectName,
+					VersionID: object.VersionID,
 				}
+				continue
 			}
 		}
 
@@ -559,8 +565,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
-		Versioned:        globalBucketVersioningSys.Enabled(bucket),
-		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
+		Versioned:        versioned,
+		VersionSuspended: suspended,
 	})
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
@@ -622,16 +628,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	}
 
-	// Clean up transitioned objects from remote tier
-	for _, os := range oss {
-		if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
-			continue
-		}
-		logger.LogIf(ctx, os.Sweep())
-	}
-
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
+		if dobj.ObjectName == "" {
+			continue
+		}
+
 		eventName := event.ObjectRemovedDelete
 		objInfo := ObjectInfo{
 			Name:         dobj.ObjectName,
@@ -654,6 +656,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			Host:         handlers.GetSourceIP(r),
 		})
 	}
+
+	// Clean up transitioned objects from remote tier
+	for _, os := range oss {
+		if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
+			continue
+		}
+		logger.LogIf(ctx, os.Sweep())
+	}
 }
 
 // PutBucketHandler - PUT Bucket
@@ -899,41 +909,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Once signature is validated, check if the user has
 	// explicit permissions for the user.
-	{
-		token := formValues.Get(xhttp.AmzSecurityToken)
-		if token != "" && cred.AccessKey == "" {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoAccessKey), r.URL)
-			return
-		}
-
-		if cred.IsServiceAccount() && token == "" {
-			token = cred.SessionToken
-		}
-
-		if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidToken), r.URL)
-			return
-		}
-
-		// Extract claims if any.
-		claims, err := getClaimsFromToken(token)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
-		}
-
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     cred.AccessKey,
-			Action:          iampolicy.PutObjectAction,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
-			BucketName:      bucket,
-			ObjectName:      object,
-			IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
-			Claims:          claims,
-		}) {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+	if !globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          iampolicy.PutObjectAction,
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+		BucketName:      bucket,
+		ObjectName:      object,
+		IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
+		Claims:          cred.Claims,
+	}) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
@@ -1699,7 +1685,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseW
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
-	durationStr := r.URL.Query().Get("older-than")
+	durationStr := r.Form.Get("older-than")
 	var (
 		days time.Duration
 		err  error

cmd/bucket-lifecycle.go

@@ -24,9 +24,9 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
@@ -84,6 +84,11 @@ type expiryState struct {
 	expiryCh chan expiryTask
 }
 
+// PendingTasks returns the number of pending ILM expiry tasks.
+func (es *expiryState) PendingTasks() int {
+	return len(es.expiryCh)
+}
+
 func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) {
 	select {
 	case <-GlobalContext.Done():
@@ -115,9 +120,16 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 }
 
 type transitionState struct {
-	once sync.Once
-	// add future metrics here
+	once         sync.Once
 	transitionCh chan ObjectInfo
+
+	ctx        context.Context
+	objAPI     ObjectLayer
+	mu         sync.Mutex
+	numWorkers int
+	killCh     chan struct{}
+
+	activeTasks int32
 }
 
 func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
@@ -132,50 +144,71 @@ func (t *transitionState) queueTransitionTask(oi ObjectInfo) {
 }
 
 var (
-	globalTransitionState      *transitionState
-	globalTransitionConcurrent = runtime.GOMAXPROCS(0) / 2
+	globalTransitionState *transitionState
 )
 
-func newTransitionState() *transitionState {
-	// fix minimum concurrent transition to 1 for single CPU setup
-	if globalTransitionConcurrent == 0 {
-		globalTransitionConcurrent = 1
-	}
+func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
 	return &transitionState{
 		transitionCh: make(chan ObjectInfo, 10000),
+		ctx:          ctx,
+		objAPI:       objAPI,
+		killCh:       make(chan struct{}),
 	}
 }
 
-// addWorker creates a new worker to process tasks
-func (t *transitionState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
-	// Add a new worker.
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case oi, ok := <-t.transitionCh:
-				if !ok {
-					return
-				}
+// PendingTasks returns the number of ILM transition tasks waiting for a worker
+// goroutine.
+func (t *transitionState) PendingTasks() int {
+	return len(globalTransitionState.transitionCh)
+}
 
-				if err := transitionObject(ctx, objectAPI, oi); err != nil {
-					logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
-				}
+// ActiveTasks returns the number of active (ongoing) ILM transition tasks.
+func (t *transitionState) ActiveTasks() int {
+	return int(atomic.LoadInt32(&t.activeTasks))
+}
+
+// worker waits for transition tasks
+func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
+	for {
+		select {
+		case <-t.killCh:
+			return
+		case <-ctx.Done():
+			return
+		case oi, ok := <-t.transitionCh:
+			if !ok {
+				return
 			}
+			atomic.AddInt32(&t.activeTasks, 1)
+			if err := transitionObject(ctx, objectAPI, oi); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
+			}
+			atomic.AddInt32(&t.activeTasks, -1)
 		}
-	}()
+	}
+}
+
+// UpdateWorkers at the end of this function leaves n goroutines waiting for
+// transition tasks
+func (t *transitionState) UpdateWorkers(n int) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	for t.numWorkers < n {
+		go t.worker(t.ctx, t.objAPI)
+		t.numWorkers++
+	}
+
+	for t.numWorkers > n {
+		go func() { t.killCh <- struct{}{} }()
+		t.numWorkers--
+	}
 }
 
 func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
-	if globalTransitionState == nil {
-		return
-	}
-
-	// Start with globalTransitionConcurrent.
-	for i := 0; i < globalTransitionConcurrent; i++ {
-		globalTransitionState.addWorker(ctx, objectAPI)
-	}
+	globalTransitionState = newTransitionState(ctx, objectAPI)
+	n := globalAPIConfig.getTransitionWorkers()
+	globalTransitionState.UpdateWorkers(n)
 }
 
 var errInvalidStorageClass = errors.New("invalid storage class")
@@ -219,9 +252,9 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		// When an object is past expiry or when a transitioned object is being
 		// deleted, 'mark' the data in the remote tier for delete.
 		entry := jentry{
-			ObjName:   oi.transitionedObjName,
-			VersionID: oi.transitionVersionID,
-			TierName:  oi.TransitionTier,
+			ObjName:   oi.TransitionedObject.Name,
+			VersionID: oi.TransitionedObject.VersionID,
+			TierName:  oi.TransitionedObject.Tier,
 		}
 		if err := globalTierJournal.AddEntry(entry); err != nil {
 			logger.LogIf(ctx, err)
@@ -303,7 +336,7 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 
 // getTransitionedObjectReader returns a reader from the transitioned tier.
 func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	tgtClient, err := globalTierConfigMgr.getDriver(oi.TransitionTier)
+	tgtClient, err := globalTierConfigMgr.getDriver(oi.TransitionedObject.Tier)
 	if err != nil {
 		return nil, fmt.Errorf("transition storage class not configured")
 	}
@@ -320,7 +353,7 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
 		gopts.length = length
 	}
 
-	reader, err := tgtClient.Get(ctx, oi.transitionedObjName, remoteVersionID(oi.transitionVersionID), gopts)
+	reader, err := tgtClient.Get(ctx, oi.TransitionedObject.Name, remoteVersionID(oi.TransitionedObject.VersionID), gopts)
 	if err != nil {
 		return nil, err
 	}
@@ -458,7 +491,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
 	versioned := globalBucketVersioningSys.Enabled(bucket)
 	versionSuspended := globalBucketVersioningSys.Suspended(bucket)
-	vid := strings.TrimSpace(r.URL.Query().Get(xhttp.VersionID))
+	vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
 		if err != nil {
@@ -544,7 +577,7 @@ func (fi FileInfo) IsRemote() bool {
 // IsRemote returns true if this object version's contents are in its remote
 // tier.
 func (oi ObjectInfo) IsRemote() bool {
-	if oi.TransitionStatus != lifecycle.TransitionComplete {
+	if oi.TransitionedObject.Status != lifecycle.TransitionComplete {
 		return false
 	}
 	return !isRestoredObjectOnDisk(oi.UserDefined)
@@ -672,7 +705,7 @@ func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
 		SuccessorModTime:       oi.SuccessorModTime,
 		RestoreOngoing:         oi.RestoreOngoing,
 		RestoreExpires:         oi.RestoreExpires,
-		TransitionStatus:       oi.TransitionStatus,
+		TransitionStatus:       oi.TransitionedObject.Status,
 		RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
 	}
 }

cmd/bucket-listobjects-handlers.go

@@ -92,7 +92,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	urlValues := r.URL.Query()
+	urlValues := r.Form
 
 	// Extract all the listBucketVersions query params to their native values.
 	prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode := getListBucketObjectVersionsArgs(urlValues)
@@ -153,7 +153,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	urlValues := r.URL.Query()
+	urlValues := r.Form
 
 	// Extract all the listObjectsV2 query params to their native values.
 	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
@@ -220,7 +220,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	urlValues := r.URL.Query()
+	urlValues := r.Form
 
 	// Extract all the listObjectsV2 query params to their native values.
 	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
@@ -329,7 +329,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query())
+	prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.Form)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return

cmd/bucket-object-lock.go

@@ -175,7 +175,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted.
-func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool, claims map[string]interface{}) (ObjectInfo, APIErrorCode) {
+func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool) (ObjectInfo, APIErrorCode) {
 	byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header)
 	opts, err := getOpts(ctx, r, bucket, object)
 	if err != nil {
@@ -203,7 +203,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 			perm := isPutRetentionAllowed(bucket, object,
 				days, objRetention.RetainUntilDate.Time,
 				objRetention.Mode, byPassSet, r, cred,
-				owner, claims)
+				owner)
 			return oi, perm
 		}
 
@@ -211,7 +211,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 		case objectlock.RetGovernance:
 			govPerm := isPutRetentionAllowed(bucket, object, days,
 				objRetention.RetainUntilDate.Time, objRetention.Mode,
-				byPassSet, r, cred, owner, claims)
+				byPassSet, r, cred, owner)
 			// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
 			if !byPassSet {
 				if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
@@ -227,7 +227,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 			}
 			compliancePerm := isPutRetentionAllowed(bucket, object,
 				days, objRetention.RetainUntilDate.Time, objRetention.Mode,
-				false, r, cred, owner, claims)
+				false, r, cred, owner)
 			return oi, compliancePerm
 		}
 		return oi, ErrNone
@@ -235,7 +235,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 
 	perm := isPutRetentionAllowed(bucket, object,
 		days, objRetention.RetainUntilDate.Time,
-		objRetention.Mode, byPassSet, r, cred, owner, claims)
+		objRetention.Mode, byPassSet, r, cred, owner)
 	return oi, perm
 }
 

cmd/bucket-policy.go

@@ -77,10 +77,10 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		}
 	}
 
-	vid := r.URL.Query().Get("versionId")
+	vid := r.Form.Get(xhttp.VersionID)
 	if vid == "" {
 		if u, err := url.Parse(r.Header.Get(xhttp.AmzCopySource)); err == nil {
-			vid = u.Query().Get("versionId")
+			vid = u.Query().Get(xhttp.VersionID)
 		}
 	}
 
@@ -143,8 +143,8 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		}
 	}
 
-	var cloneURLValues = url.Values{}
-	for k, v := range r.URL.Query() {
+	cloneURLValues := make(url.Values, len(r.Form))
+	for k, v := range r.Form {
 		cloneURLValues[k] = v
 	}
 

cmd/bucket-replication.go

@@ -43,6 +43,8 @@ import (
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
+const throttleDeadline = 1 * time.Hour
+
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	if globalIsGateway {
@@ -792,15 +794,18 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			Bucket:     objInfo.Bucket,
 			HeaderSize: headerSize,
 		}
-		newCtx, cancel := context.WithTimeout(ctx, globalOperationTimeout.Timeout())
-		defer cancel()
+		newCtx := ctx
+		if globalBucketMonitor.IsThrottled(bucket) {
+			var cancel context.CancelFunc
+			newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
+			defer cancel()
+		}
 		r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
-		if len(objInfo.Parts) > 1 {
-			if uploadID, err := replicateObjectWithMultipart(ctx, c, dest.Bucket, object, r, objInfo, putOpts); err != nil {
+		if objInfo.Multipart {
+			if err := replicateObjectWithMultipart(ctx, c, dest.Bucket, object,
+				r, objInfo, putOpts); err != nil {
 				replicationStatus = replication.Failed
 				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
-				// block and abort remote upload upon failure.
-				c.AbortMultipartUpload(ctx, dest.Bucket, object, uploadID)
 			}
 		} else {
 			if _, err = c.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts); err != nil {
@@ -868,38 +873,52 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	}
 }
 
-func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket, object string, r io.Reader, objInfo ObjectInfo, opts miniogo.PutObjectOptions) (uploadID string, err error) {
+func replicateObjectWithMultipart(ctx context.Context, c *miniogo.Core, bucket, object string, r io.Reader, objInfo ObjectInfo, opts miniogo.PutObjectOptions) (err error) {
 	var uploadedParts []miniogo.CompletePart
-	uploadID, err = c.NewMultipartUpload(context.Background(), bucket, object, opts)
+	uploadID, err := c.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err != nil {
-		return
+		return err
 	}
+
+	defer func() {
+		if err != nil {
+			// block and abort remote upload upon failure.
+			if aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID); aerr != nil {
+				aerr = fmt.Errorf("Unable to cleanup failed multipart replication %s on remote %s/%s: %w", uploadID, bucket, object, aerr)
+				logger.LogIf(ctx, aerr)
+			}
+		}
+	}()
+
 	var (
 		hr    *hash.Reader
 		pInfo miniogo.ObjectPart
 	)
+
 	for _, partInfo := range objInfo.Parts {
 		hr, err = hash.NewReader(r, partInfo.Size, "", "", partInfo.Size)
 		if err != nil {
-			return
+			return err
 		}
 		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, "", "", opts.ServerSideEncryption)
 		if err != nil {
-			return
+			return err
 		}
 		if pInfo.Size != partInfo.Size {
-			return uploadID, fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.Size)
+			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.Size)
 		}
 		uploadedParts = append(uploadedParts, miniogo.CompletePart{
 			PartNumber: pInfo.PartNumber,
 			ETag:       pInfo.ETag,
 		})
 	}
-	_, err = c.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{
-		SourceMTime:        objInfo.ModTime,
-		ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
-	}})
-	return
+	_, err = c.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, miniogo.PutObjectOptions{
+		Internal: miniogo.AdvancedPutOptions{
+			SourceMTime: objInfo.ModTime,
+			// always set this to distinguish between `mc mirror` replication and serverside
+			ReplicationRequest: true,
+		}})
+	return err
 }
 
 // filterReplicationStatusMetadata filters replication status metadata for COPY

cmd/bucket-targets.go

@@ -346,6 +346,7 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	getRemoteTargetInstanceTransportOnce.Do(func() {
 		getRemoteTargetInstanceTransport = NewRemoteTargetHTTPTransport()
 	})
+
 	api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,

cmd/common-main.go

@@ -44,6 +44,7 @@ import (
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
 	"github.com/minio/kes"
+	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
@@ -70,7 +71,7 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsKubernetesReplicaSet() || IsPCFTile() {
+	if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
 		// 30 seconds matches the orchestrator DNS TTLs, have
 		// a 5 second timeout to lookup from DNS servers.
 		globalDNSCache = xhttp.NewDNSCache(30*time.Second, 5*time.Second, logger.LogOnceIf)
@@ -91,8 +92,6 @@ func init() {
 		},
 	})
 
-	globalTransitionState = newTransitionState()
-
 	console.SetColor("Debug", fcolor.New())
 
 	gob.Register(StorageErr(""))
@@ -104,6 +103,10 @@ func init() {
 			},
 		},
 	}
+
+	// All minio-go API operations shall be performed only once,
+	// another way to look at this is we are turning off retries.
+	minio.MaxRetry = 1
 }
 
 const consolePrefix = "CONSOLE_"
@@ -111,10 +114,11 @@ const consolePrefix = "CONSOLE_"
 func minioConfigToConsoleFeatures() {
 	os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID)
 	os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID)
-	if globalMinioEndpoint == "" {
-		logger.Fatal(errInvalidArgument, "Unable to start console service MinIO Endpoint is empty")
+	if globalMinioEndpoint != "" {
+		os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
+	} else {
+		os.Setenv("CONSOLE_MINIO_SERVER", getAPIEndpoints()[0])
 	}
-	os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
 	if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {
 		os.Setenv("CONSOLE_LOG_QUERY_URL", value)
 		if value := env.Get("MINIO_LOG_QUERY_AUTH_TOKEN", ""); value != "" {
@@ -148,6 +152,9 @@ func minioConfigToConsoleFeatures() {
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalServerRegion)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
+	if globalSubnetLicense != "" {
+		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetLicense)
+	}
 }
 
 func initConsoleServer() (*restapi.Server, error) {
@@ -172,14 +179,16 @@ func initConsoleServer() (*restapi.Server, error) {
 		return nil, err
 	}
 
+	noLog := func(string, ...interface{}) {
+		// nothing to log
+	}
+
 	// Initialize MinIO loggers
-	restapi.LogInfo = logger.Info
-	restapi.LogError = logger.Error
+	restapi.LogInfo = noLog
+	restapi.LogError = noLog
 
 	api := operations.NewConsoleAPI(swaggerSpec)
-	api.Logger = func(_ string, _ ...interface{}) {
-		// nothing to log.
-	}
+	api.Logger = noLog
 
 	server := restapi.NewServer(api)
 	// register all APIs
@@ -207,11 +216,6 @@ func initConsoleServer() (*restapi.Server, error) {
 }
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
-	if (GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
-		logger.Fatal(errInvalidArgument,
-			"Encryption support is requested but '%s' does not support encryption", name)
-	}
-
 	if strings.HasPrefix(name, "gateway") {
 		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
 			uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
@@ -412,7 +416,7 @@ func handleCommonEnvVars() {
 		}
 	}
 
-	if serverURL := env.Get(config.EnvMinIOServerURL, globalEndpoints.Localhost()); serverURL != "" {
+	if serverURL := env.Get(config.EnvMinIOServerURL, ""); serverURL != "" {
 		u, err := xnet.ParseHTTPURL(serverURL)
 		if err != nil {
 			logger.Fatal(err, "Invalid MINIO_SERVER_URL value in environment variable")
@@ -595,6 +599,8 @@ func handleCommonEnvVars() {
 	if tiers := env.Get("_MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY", ""); tiers != "" {
 		globalDebugRemoteTiersImmediately = strings.Split(tiers, ",")
 	}
+
+	globalSubnetLicense = env.Get(config.EnvMinIOSubnetLicense, "")
 }
 
 func logStartupMessage(msg string) {

cmd/crossdomain-xml-handler_test.go

@@ -28,7 +28,7 @@ import (
 // Test cross domain xml handler.
 func TestCrossXMLHandler(t *testing.T) {
 	// Server initialization.
-	router := mux.NewRouter().SkipClean(true)
+	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
 	handler := setCrossDomainPolicy(router)
 	srv := httptest.NewServer(handler)
 

cmd/data-scanner.go

@@ -525,12 +525,22 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				return
 			}
 			if !into.Compacted {
-				into.addChild(dataUsageHash(folder.name))
+				h := dataUsageHash(folder.name)
+				into.addChild(h)
+				// We scanned a folder, optionally send update.
+				f.updateCache.deleteRecursive(h)
+				f.updateCache.copyWithChildren(&f.newCache, h, folder.parent)
+				f.sendUpdate()
 			}
-			// We scanned a folder, optionally send update.
-			f.sendUpdate()
 		}
 
+		// Transfer existing
+		if !into.Compacted {
+			for _, folder := range existingFolders {
+				h := hashPath(folder.name)
+				f.updateCache.copyWithChildren(&f.oldCache, h, folder.parent)
+			}
+		}
 		// Scan new...
 		for _, folder := range newFolders {
 			h := hashPath(folder.name)
@@ -903,7 +913,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta ac
 			SuccessorModTime:       meta.oi.SuccessorModTime,
 			RestoreOngoing:         meta.oi.RestoreOngoing,
 			RestoreExpires:         meta.oi.RestoreExpires,
-			TransitionStatus:       meta.oi.TransitionStatus,
+			TransitionStatus:       meta.oi.TransitionedObject.Status,
 			RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
 		})
 	if i.debug {
@@ -966,7 +976,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, meta ac
 // applyTierObjSweep removes remote object pending deletion and the free-version
 // tracking this information.
 func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta actionMeta) {
-	if !meta.oi.tierFreeVersion {
+	if !meta.oi.TransitionedObject.FreeVersion {
 		// nothing to be done
 		return
 	}
@@ -979,7 +989,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, meta
 		return err
 	}
 	// Remove the remote object
-	err := deleteObjectFromRemoteTier(ctx, meta.oi.transitionedObjName, meta.oi.transitionVersionID, meta.oi.TransitionTier)
+	err := deleteObjectFromRemoteTier(ctx, meta.oi.TransitionedObject.Name, meta.oi.TransitionedObject.VersionID, meta.oi.TransitionedObject.Tier)
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
 		return
@@ -1055,7 +1065,7 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, obj Ob
 
 func applyTransitionAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) bool {
 	srcOpts := ObjectOptions{}
-	if obj.TransitionStatus == "" {
+	if obj.TransitionedObject.Status == "" {
 		srcOpts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
 		srcOpts.VersionID = obj.VersionID
 		// mark transition as pending
@@ -1127,7 +1137,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 // Apply object, object version, restored object or restored object version action on the given object
 func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
-	if obj.TransitionStatus != "" {
+	if obj.TransitionedObject.Status != "" {
 		return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject)
 	}
 	return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj, applyOnVersion)
@@ -1354,6 +1364,8 @@ const (
 	ILMExpiry = "ilm:expiry"
 	// ILMFreeVersionDelete - audit trail for ILM free-version delete
 	ILMFreeVersionDelete = "ilm:free-version-delete"
+	// ILMTransition - audit trail for ILM transitioning.
+	ILMTransition = " ilm:transition"
 )
 
 func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
@@ -1363,6 +1375,8 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, trigger string) {
 		apiName = "ILMExpiry"
 	case ILMFreeVersionDelete:
 		apiName = "ILMFreeVersionDelete"
+	case ILMTransition:
+		apiName = "ILMTransition"
 	}
 	auditLogInternal(ctx, oi.Bucket, oi.Name, AuditLogOptions{
 		Trigger:   trigger,

cmd/data-update-tracker.go

@@ -46,7 +46,7 @@ const (
 	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 5
+	dataUpdateTrackerVersion      = 6
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -397,7 +397,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2, 3, 4:
+	case 1, 2, 3, 4, 5:
 		if intDataUpdateTracker.debug {
 			console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		}

cmd/data-usage-cache.go

@@ -513,7 +513,11 @@ func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compact
 
 // StringAll returns a detailed string representation of all entries in the cache.
 func (d *dataUsageCache) StringAll() string {
+	// Remove bloom filter from print.
+	bf := d.Info.BloomFilter
+	d.Info.BloomFilter = nil
 	s := fmt.Sprintf("info:%+v\n", d.Info)
+	d.Info.BloomFilter = bf
 	for k, v := range d.Cache {
 		s += fmt.Sprintf("\t%v: %+v\n", k, v)
 	}

cmd/endpoint.go

@@ -377,27 +377,6 @@ func (endpoints Endpoints) GetAllStrings() (all []string) {
 	return
 }
 
-func hostResolveToLocalhost(endpoint Endpoint) bool {
-	hostIPs, err := getHostIP(endpoint.Hostname())
-	if err != nil {
-		// Log the message to console about the host resolving
-		reqInfo := (&logger.ReqInfo{}).AppendTags(
-			"host",
-			endpoint.Hostname(),
-		)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogIf(ctx, err, logger.Application)
-		return false
-	}
-	var loopback int
-	for _, hostIP := range hostIPs.ToSlice() {
-		if net.ParseIP(hostIP).IsLoopback() {
-			loopback++
-		}
-	}
-	return loopback == len(hostIPs)
-}
-
 func (endpoints Endpoints) atleastOneEndpointLocal() bool {
 	for _, endpoint := range endpoints {
 		if endpoint.IsLocal {
@@ -410,7 +389,6 @@ func (endpoints Endpoints) atleastOneEndpointLocal() bool {
 // UpdateIsLocal - resolves the host and discovers the local host.
 func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	orchestrated := IsDocker() || IsKubernetes()
-	k8sReplicaSet := IsKubernetesReplicaSet()
 
 	var epsResolved int
 	var foundLocal bool
@@ -443,24 +421,6 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 					endpoints[i].Hostname(),
 				)
 
-				if k8sReplicaSet && hostResolveToLocalhost(endpoints[i]) {
-					err := fmt.Errorf("host %s resolves to 127.*, DNS incorrectly configured retrying",
-						endpoints[i])
-					// time elapsed
-					timeElapsed := time.Since(startTime)
-					// log error only if more than 1s elapsed
-					if timeElapsed > time.Second {
-						reqInfo.AppendTags("elapsedTime",
-							humanize.RelTime(startTime,
-								startTime.Add(timeElapsed),
-								"elapsed",
-								""))
-						ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-						logger.LogIf(ctx, err, logger.Application)
-					}
-					continue
-				}
-
 				// return err if not Docker or Kubernetes
 				// We use IsDocker() to check for Docker environment
 				// We use IsKubernetes() to check for Kubernetes environment
@@ -489,36 +449,6 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 				} else {
 					resolvedList[i] = true
 					endpoints[i].IsLocal = isLocal
-					if k8sReplicaSet && !endpoints.atleastOneEndpointLocal() && !foundPrevLocal {
-						// In replicated set in k8s deployment, IPs might
-						// get resolved for older IPs, add this code
-						// to ensure that we wait for this server to
-						// participate atleast one disk and be local.
-						//
-						// In special cases for replica set with expanded
-						// pool setups we need to make sure to provide
-						// value of foundPrevLocal from pool1 if we already
-						// found a local setup. Only if we haven't found
-						// previous local we continue to wait to look for
-						// atleast one local.
-						resolvedList[i] = false
-						// time elapsed
-						err := fmt.Errorf("no endpoint is local to this host: %s", endpoints[i])
-						timeElapsed := time.Since(startTime)
-						// log error only if more than 1s elapsed
-						if timeElapsed > time.Second {
-							reqInfo.AppendTags("elapsedTime",
-								humanize.RelTime(startTime,
-									startTime.Add(timeElapsed),
-									"elapsed",
-									"",
-								))
-							ctx := logger.SetReqInfo(GlobalContext,
-								reqInfo)
-							logger.LogIf(ctx, err, logger.Application)
-						}
-						continue
-					}
 					epsResolved++
 					if !foundLocal {
 						foundLocal = isLocal

cmd/erasure-common.go

@@ -20,8 +20,6 @@ package cmd
 import (
 	"context"
 	"sync"
-
-	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 func (er erasureObjects) getLocalDisks() (localDisks []StorageAPI) {
@@ -47,38 +45,6 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
-func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
-	disks := er.getDisks()
-	var wg sync.WaitGroup
-	var mu sync.Mutex
-	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
-		i := i
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if disks[i-1] == nil {
-				return
-			}
-			di, err := disks[i-1].DiskInfo(context.Background())
-			if err != nil || di.Healing {
-				// - Do not consume disks which are not reachable
-				//   unformatted or simply not accessible for some reason.
-				//
-				// - Do not consume disks which are being healed
-				//
-				// - Future: skip busy disks
-				return
-			}
-
-			mu.Lock()
-			newDisks = append(newDisks, disks[i-1])
-			mu.Unlock()
-		}()
-	}
-	wg.Wait()
-	return newDisks
-}
-
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
 // ensures to skip disks if they are not healing and online.
 func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
@@ -133,34 +99,3 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 	// Return disks which have maximum disk usage common.
 	return newDisks[max]
 }
-
-// This function does the following check, suppose
-// object is "a/b/c/d", stat makes sure that objects
-// - "a/b/c"
-// - "a/b"
-// - "a"
-// do not exist on the namespace.
-func (er erasureObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] == nil {
-				return errDiskNotFound
-			}
-			// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
-			return storageDisks[index].CheckFile(ctx, bucket, parent)
-		}, index)
-	}
-
-	// NOTE: Observe we are not trying to read `xl.meta` and figure out the actual
-	// quorum intentionally, but rely on the default case scenario. Actual quorum
-	// verification will happen by top layer by using getObjectInfo() and will be
-	// ignored if necessary.
-	readQuorum := getReadQuorum(len(storageDisks))
-
-	return reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum) == nil
-}

cmd/erasure-common_test.go

@@ -1,83 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"os"
-	"testing"
-)
-
-// Tests for if parent directory is object
-func TestErasureParentDirIsObject(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	obj, fsDisks, err := prepareErasureSets32(ctx)
-	if err != nil {
-		t.Fatalf("Unable to initialize 'Erasure' object layer.")
-	}
-	defer obj.Shutdown(context.Background())
-
-	// Remove all disks.
-	for _, disk := range fsDisks {
-		defer os.RemoveAll(disk)
-	}
-
-	bucketName := "testbucket"
-	objectName := "object"
-
-	if err = obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
-		t.Fatal(err)
-	}
-
-	objectContent := "12345"
-	_, err = obj.PutObject(GlobalContext, bucketName, objectName,
-		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	testCases := []struct {
-		expectedErr bool
-		objectName  string
-	}{
-		{
-			expectedErr: true,
-			objectName:  pathJoin(objectName, "parent-is-object"),
-		},
-		{
-			expectedErr: false,
-			objectName:  pathJoin("no-parent", "object"),
-		},
-	}
-
-	for _, testCase := range testCases {
-		t.Run("", func(t *testing.T) {
-			_, err = obj.PutObject(GlobalContext, bucketName, testCase.objectName,
-				mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
-			if testCase.expectedErr && err == nil {
-				t.Error("Expected error but got nil")
-			}
-			if !testCase.expectedErr && err != nil {
-				t.Errorf("Expected nil but got %v", err)
-			}
-		})
-	}
-}

cmd/erasure-healing-common.go

@@ -189,35 +189,6 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 	return latestFileInfo, nil
 }
 
-// fileInfoConsistent whether all fileinfos are consistent with each other.
-// Will return false if any fileinfo mismatches.
-func fileInfoConsistent(ctx context.Context, partsMetadata []FileInfo, errs []error) bool {
-	// There should be atleast half correct entries, if not return failure
-	if reducedErr := reduceReadQuorumErrs(ctx, errs, nil, len(partsMetadata)/2); reducedErr != nil {
-		return false
-	}
-	if len(partsMetadata) == 1 {
-		return true
-	}
-	// Reference
-	ref := partsMetadata[0]
-	if !ref.IsValid() {
-		return false
-	}
-	for _, meta := range partsMetadata[1:] {
-		if !meta.IsValid() {
-			return false
-		}
-		if !meta.ModTime.Equal(ref.ModTime) {
-			return false
-		}
-		if meta.DataDir != ref.DataDir {
-			return false
-		}
-	}
-	return true
-}
-
 // disksWithAllParts - This function needs to be called with
 // []StorageAPI returned by listOnlineDisks. Returns,
 //
@@ -225,8 +196,9 @@ func fileInfoConsistent(ctx context.Context, partsMetadata []FileInfo, errs []er
 //
 // - slice of errors about the state of data files on disk - can have
 //   a not-found error or a hash-mismatch error.
-func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo, errs []error, bucket,
-	object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
+func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
+	errs []error, bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
+
 	// List of disks having latest version of the object er.meta  (by modtime)
 	_, modTime, dataDir := listOnlineDisks(onlineDisks, partsMetadata, errs)
 
@@ -239,15 +211,17 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// consider the offline disks as consistent.
 			continue
 		}
-		if len(meta.Erasure.Distribution) != len(onlineDisks) {
-			// Erasure distribution seems to have lesser
-			// number of items than number of online disks.
-			inconsistent++
-			continue
-		}
-		if meta.Erasure.Distribution[i] != meta.Erasure.Index {
-			// Mismatch indexes with distribution order
-			inconsistent++
+		if !meta.Deleted {
+			if len(meta.Erasure.Distribution) != len(onlineDisks) {
+				// Erasure distribution seems to have lesser
+				// number of items than number of online disks.
+				inconsistent++
+				continue
+			}
+			if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+				// Mismatch indexes with distribution order
+				inconsistent++
+			}
 		}
 	}
 
@@ -267,8 +241,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			dataErrs[i] = errDiskNotFound
 			continue
 		}
-		meta := partsMetadata[i]
 
+		meta := partsMetadata[i]
 		if !meta.ModTime.Equal(modTime) || meta.DataDir != dataDir {
 			dataErrs[i] = errFileCorrupt
 			partsMetadata[i] = FileInfo{}
@@ -280,20 +254,22 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 				continue
 			}
 
-			if len(meta.Erasure.Distribution) != len(onlineDisks) {
-				// Erasure distribution is not the same as onlineDisks
-				// attempt a fix if possible, assuming other entries
-				// might have the right erasure distribution.
-				partsMetadata[i] = FileInfo{}
-				dataErrs[i] = errFileCorrupt
-				continue
-			}
+			if !meta.Deleted {
+				if len(meta.Erasure.Distribution) != len(onlineDisks) {
+					// Erasure distribution is not the same as onlineDisks
+					// attempt a fix if possible, assuming other entries
+					// might have the right erasure distribution.
+					partsMetadata[i] = FileInfo{}
+					dataErrs[i] = errFileCorrupt
+					continue
+				}
 
-			// Since erasure.Distribution is trustable we can fix the mismatching erasure.Index
-			if meta.Erasure.Distribution[i] != meta.Erasure.Index {
-				partsMetadata[i] = FileInfo{}
-				dataErrs[i] = errFileCorrupt
-				continue
+				// Since erasure.Distribution is trustable we can fix the mismatching erasure.Index
+				if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+					partsMetadata[i] = FileInfo{}
+					dataErrs[i] = errFileCorrupt
+					continue
+				}
 			}
 		}
 
@@ -320,11 +296,11 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// disk has a valid xl.meta but may not have all the
 			// parts. This is considered an outdated disk, since
 			// it needs healing too.
-			if !partsMetadata[i].IsRemote() {
+			if !partsMetadata[i].Deleted && !partsMetadata[i].IsRemote() {
 				dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, partsMetadata[i])
 			}
 		case madmin.HealNormalScan:
-			if !partsMetadata[i].IsRemote() {
+			if !partsMetadata[i].Deleted && !partsMetadata[i].IsRemote() {
 				dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, partsMetadata[i])
 			}
 		}

cmd/erasure-healing.go

@@ -212,7 +212,7 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, quorumModTime t
 		return true
 	}
 	if erErr == nil {
-		if !meta.IsRemote() {
+		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
 				errFileNotFound,
@@ -267,10 +267,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
 
-	_, err = getLatestFileInfo(ctx, partsMetadata, errs)
-	if err != nil {
+	if _, err = getLatestFileInfo(ctx, partsMetadata, errs); err != nil {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
 	}
+
 	// List of disks having latest version of the object er.meta
 	// (by modtime).
 	_, modTime, dataDir := listOnlineDisks(storageDisks, partsMetadata, errs)
@@ -292,7 +292,8 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// used here for reconstruction. This is done to ensure that
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
-	availableDisks, dataErrs := disksWithAllParts(ctx, storageDisks, partsMetadata, errs, bucket, object, scanMode)
+	availableDisks, dataErrs := disksWithAllParts(ctx, storageDisks, partsMetadata,
+		errs, bucket, object, scanMode)
 
 	// Loop to find number of disks with valid data, per-drive
 	// data state and a list of outdated disks on which data needs
@@ -357,7 +358,8 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
 		}
 		// File is fully gone, fileInfo is empty.
-		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
+		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
 	}
 
 	// If less than read quorum number of disks have all the parts
@@ -642,30 +644,25 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 
 // Populates default heal result item entries with possible values when we are returning prematurely.
 // This is to ensure that in any circumstance we are not returning empty arrays with wrong values.
-func defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []string, errs []error, bucket, object, versionID string, defaultParityCount int) madmin.HealResultItem {
+func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []string, errs []error, bucket, object, versionID string) madmin.HealResultItem {
 	// Initialize heal result object
 	result := madmin.HealResultItem{
-		Type:      madmin.HealItemObject,
-		Bucket:    bucket,
-		Object:    object,
-		VersionID: versionID,
-		DiskCount: len(storageDisks),
+		Type:       madmin.HealItemObject,
+		Bucket:     bucket,
+		Object:     object,
+		ObjectSize: lfi.Size,
+		VersionID:  versionID,
+		DiskCount:  len(storageDisks),
 	}
 
 	if lfi.IsValid() {
-		result.ObjectSize = lfi.Size
 		result.ParityBlocks = lfi.Erasure.ParityBlocks
 	} else {
 		// Default to most common configuration for erasure blocks.
-		result.ParityBlocks = defaultParityCount
+		result.ParityBlocks = er.defaultParityCount
 	}
 	result.DataBlocks = len(storageDisks) - result.ParityBlocks
 
-	if errs == nil {
-		// No disks related errors are provided, quit
-		return result
-	}
-
 	for index, disk := range storageDisks {
 		if disk == nil {
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
@@ -684,6 +681,8 @@ func defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints
 		switch errs[index] {
 		case errFileNotFound, errVolumeNotFound:
 			driveState = madmin.DriveStateMissing
+		case nil:
+			driveState = madmin.DriveStateOk
 		}
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
@@ -697,15 +696,6 @@ func defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints
 		})
 	}
 
-	if !lfi.IsValid() {
-		// Default to most common configuration for erasure blocks.
-		result.ParityBlocks = defaultParityCount
-		result.DataBlocks = len(storageDisks) - defaultParityCount
-	} else {
-		result.ParityBlocks = lfi.Erasure.ParityBlocks
-		result.DataBlocks = lfi.Erasure.DataBlocks
-	}
-
 	return result
 }
 
@@ -793,11 +783,9 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 		var err error
 		var returnNotFound bool
 		if !opts.DryRun && opts.Remove {
-			if versionID == "" {
-				err = er.deleteObject(ctx, bucket, object, writeQuorum)
-			} else {
-				err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{VersionID: versionID}, false)
-			}
+			err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{
+				VersionID: versionID,
+			}, false)
 
 			// If Delete was successful, make sure to return the appropriate error
 			// and heal result appropriate with delete's error messages
@@ -820,15 +808,18 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 			if versionID != "" {
 				err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
 			}
-			return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
+			return er.defaultHealResult(m, storageDisks, storageEndpoints,
+				errs, bucket, object, versionID), err
 		}
-		return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), toObjectErr(err, bucket, object, versionID)
+		return er.defaultHealResult(m, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID)
 	}
 
 	readQuorum := len(storageDisks) - er.defaultParityCount
 
-	err := toObjectErr(reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum), bucket, object, versionID)
-	return defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
+	err := toObjectErr(reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum),
+		bucket, object, versionID)
+	return er.defaultHealResult(m, storageDisks, storageEndpoints, errs, bucket, object, versionID), err
 }
 
 // Object is considered dangling/corrupted if any only
@@ -914,22 +905,15 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 
 	// Perform quick read without lock.
 	// This allows to quickly check if all is ok or all are missing.
-	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
+	_, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
 	if isAllNotFound(errs) {
 		err = toObjectErr(errFileNotFound, bucket, object)
 		if versionID != "" {
 			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
 		}
 		// Nothing to do, file is already gone.
-		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
-	}
-
-	// Return early if all ok and not deep scanning.
-	if opts.ScanMode == madmin.HealNormalScan && fileInfoConsistent(ctx, partsMetadata, errs) {
-		fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
-		if err == nil && fi.VersionID == versionID {
-			return defaultHealResult(fi, storageDisks, storageEndpoints, errs, bucket, object, versionID, fi.Erasure.ParityBlocks), nil
-		}
+		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), err
 	}
 
 	// Heal the object.

cmd/erasure-healing_test.go

@@ -221,7 +221,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 
@@ -365,7 +365,7 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 

cmd/erasure-metadata.go

@@ -27,6 +27,8 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/etag"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -141,6 +143,13 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 	// Extract etag from metadata.
 	objInfo.ETag = extractETag(fi.Metadata)
 
+	// Verify if Etag is parseable, if yes
+	// then check if its multipart etag.
+	et, e := etag.Parse(objInfo.ETag)
+	if e == nil {
+		objInfo.Multipart = et.IsMultipart()
+	}
+
 	// Add user tags to the object info
 	tags := fi.Metadata[xhttp.AmzObjectTagging]
 	if len(tags) != 0 {
@@ -153,11 +162,13 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 		objInfo.ReplicationStatus = replication.StatusType(fi.DeleteMarkerReplicationStatus)
 	}
 
-	objInfo.TransitionStatus = fi.TransitionStatus
-	objInfo.transitionedObjName = fi.TransitionedObjName
-	objInfo.transitionVersionID = fi.TransitionVersionID
-	objInfo.tierFreeVersion = fi.TierFreeVersion()
-	objInfo.TransitionTier = fi.TransitionTier
+	objInfo.TransitionedObject = TransitionedObject{
+		Name:        fi.TransitionedObjName,
+		VersionID:   fi.TransitionVersionID,
+		Status:      fi.TransitionStatus,
+		FreeVersion: fi.TierFreeVersion(),
+		Tier:        fi.TransitionTier,
+	}
 
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
@@ -165,6 +176,14 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
 	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(fi.Metadata)
 
+	// Set multipart for encryption properly if
+	// not set already.
+	if !objInfo.Multipart {
+		if _, ok := crypto.IsEncrypted(objInfo.UserDefined); ok {
+			objInfo.Multipart = crypto.IsMultiPart(objInfo.UserDefined)
+		}
+	}
+
 	// All the parts per object.
 	objInfo.Parts = fi.Parts
 

cmd/erasure-multipart.go

@@ -347,8 +347,9 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	// Fill all the necessary metadata.
 	// Update `xl.meta` content on each disks.
 	for index := range partsMetadata {
-		partsMetadata[index].Metadata = opts.UserDefined
+		partsMetadata[index].Fresh = true
 		partsMetadata[index].ModTime = modTime
+		partsMetadata[index].Metadata = opts.UserDefined
 	}
 
 	uploadID := mustGetUUID()
@@ -782,12 +783,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	// Check if an object is present as one of the parent dir.
-	// -- FIXME. (needs a new kind of lock).
-	if opts.ParentIsObject != nil && opts.ParentIsObject(rctx, bucket, path.Dir(object)) {
-		return oi, toObjectErr(errFileParentIsFile, bucket, object)
-	}
-
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := getCompleteMultipartMD5(parts)
 

cmd/erasure-object.go

@@ -256,26 +256,6 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 		return toObjectErr(err, bucket, object)
 	}
 
-	// This hack is needed to avoid a bug found when overwriting
-	// the inline data object with a un-inlined version, for the
-	// time being we need this as we have released inline-data
-	// version already and this bug is already present in newer
-	// releases.
-	//
-	// This mainly happens with objects < smallFileThreshold when
-	// they are overwritten with un-inlined objects >= smallFileThreshold,
-	// due to a bug in RenameData() the fi.Data is not niled leading to
-	// GetObject thinking that fi.Data is valid while fi.Size has
-	// changed already.
-	if fi.InlineData() {
-		shardFileSize := erasure.ShardFileSize(fi.Size)
-		if shardFileSize >= 0 && shardFileSize >= smallFileThreshold {
-			for i := range metaArr {
-				metaArr[i].Data = nil
-			}
-		}
-	}
-
 	var healOnce sync.Once
 
 	// once we have obtained a common FileInfo i.e latest, we should stick
@@ -442,7 +422,9 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}
 
 	// if missing metadata can be reconstructed, attempt to reconstruct.
-	if missingBlocks > 0 && missingBlocks < readQuorum {
+	// additionally do not heal delete markers inline, let them be
+	// healed upon regular heal process.
+	if !fi.Deleted && missingBlocks > 0 && missingBlocks < readQuorum {
 		if _, healing := er.getOnlineDisksWithHealing(); !healing {
 			go healObject(bucket, object, fi.VersionID, madmin.HealNormalScan)
 		}
@@ -580,6 +562,142 @@ func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBuc
 	return evalDisks(disks, errs), err
 }
 
+func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	data := r.Reader
+
+	// No metadata is set, allocate a new one.
+	if opts.UserDefined == nil {
+		opts.UserDefined = make(map[string]string)
+	}
+
+	storageDisks := er.getDisks()
+	// Get parity and data drive count based on storage class metadata
+	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
+	if parityDrives <= 0 {
+		parityDrives = er.defaultParityCount
+	}
+	dataDrives := len(storageDisks) - parityDrives
+
+	// we now know the number of blocks this object needs for data and parity.
+	// writeQuorum is dataBlocks + 1
+	writeQuorum := dataDrives
+	if dataDrives == parityDrives {
+		writeQuorum++
+	}
+
+	// Validate input data size and it can never be less than zero.
+	if data.Size() < -1 {
+		logger.LogIf(ctx, errInvalidArgument, logger.Application)
+		return ObjectInfo{}, toObjectErr(errInvalidArgument)
+	}
+
+	// Initialize parts metadata
+	partsMetadata := make([]FileInfo, len(storageDisks))
+
+	fi := newFileInfo(pathJoin(minioMetaBucket, key), dataDrives, parityDrives)
+	fi.DataDir = mustGetUUID()
+
+	// Initialize erasure metadata.
+	for index := range partsMetadata {
+		partsMetadata[index] = fi
+	}
+
+	// Order disks according to erasure distribution
+	var onlineDisks []StorageAPI
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi)
+
+	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
+	if err != nil {
+		return ObjectInfo{}, toObjectErr(err, minioMetaBucket, key)
+	}
+
+	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
+	var buffer []byte
+	switch size := data.Size(); {
+	case size == 0:
+		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+	case size >= fi.Erasure.BlockSize:
+		buffer = er.bp.Get()
+		defer er.bp.Put(buffer)
+	case size < fi.Erasure.BlockSize:
+		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
+		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
+	}
+
+	if len(buffer) > int(fi.Erasure.BlockSize) {
+		buffer = buffer[:fi.Erasure.BlockSize]
+	}
+
+	shardFileSize := erasure.ShardFileSize(data.Size())
+	writers := make([]io.Writer, len(onlineDisks))
+	inlineBuffers := make([]*bytes.Buffer, len(onlineDisks))
+	for i, disk := range onlineDisks {
+		if disk == nil {
+			continue
+		}
+
+		inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
+		writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
+	}
+
+	n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum)
+	closeBitrotWriters(writers)
+	if erasureErr != nil {
+		return ObjectInfo{}, toObjectErr(erasureErr, minioMetaBucket, key)
+	}
+
+	// Should return IncompleteBody{} error when reader has fewer bytes
+	// than specified in request header.
+	if n < data.Size() {
+		return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key}
+	}
+
+	for i, w := range writers {
+		if w == nil {
+			onlineDisks[i] = nil
+			continue
+		}
+		partsMetadata[i].Data = inlineBuffers[i].Bytes()
+		partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize())
+		partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{
+			PartNumber: 1,
+			Algorithm:  DefaultBitrotAlgorithm,
+			Hash:       bitrotWriterSum(w),
+		})
+	}
+
+	modTime := UTCNow()
+
+	// Fill all the necessary metadata.
+	// Update `xl.meta` content on each disks.
+	for index := range partsMetadata {
+		partsMetadata[index].Size = n
+		partsMetadata[index].Fresh = true
+		partsMetadata[index].ModTime = modTime
+		partsMetadata[index].Metadata = opts.UserDefined
+	}
+
+	// Set an additional header when data is inlined.
+	for index := range partsMetadata {
+		partsMetadata[index].SetInlineData()
+	}
+
+	for i := 0; i < len(onlineDisks); i++ {
+		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
+			// Object info is the same in all disks, so we can pick
+			// the first meta from online disk
+			fi = partsMetadata[i]
+			break
+		}
+	}
+
+	if _, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaBucket, key, partsMetadata, writeQuorum); err != nil {
+		return ObjectInfo{}, toObjectErr(err, minioMetaBucket, key)
+	}
+
+	return fi.ToObjectInfo(minioMetaBucket, key), nil
+}
+
 // PutObject - creates an object upon reading from the input stream
 // until EOF, erasure codes the data across all disk and additionally
 // writes `xl.meta` which carries the necessary metadata for future
@@ -658,13 +776,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
-	// Check if an object is present as one of the parent dir.
-	// -- FIXME. (needs a new kind of lock).
-	// -- FIXME (this also causes performance issue when disks are down).
-	if opts.ParentIsObject != nil && opts.ParentIsObject(ctx, bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
-	}
-
 	// Initialize parts metadata
 	partsMetadata := make([]FileInfo, len(storageDisks))
 
@@ -738,6 +849,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		} else if shardFileSize < smallFileThreshold/8 {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 		}
+	} else {
+		// If compressed, use actual size to determine.
+		if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 {
+			if !opts.Versioned && sz < smallFileThreshold {
+				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
+			} else if sz < smallFileThreshold/8 {
+				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
+			}
+		}
 	}
 	for i, disk := range onlineDisks {
 		if disk == nil {
@@ -745,7 +865,11 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		}
 
 		if len(inlineBuffers) > 0 {
-			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
+			sz := shardFileSize
+			if sz < 0 {
+				sz = data.ActualSize()
+			}
+			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, sz))
 			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 			continue
 		}
@@ -941,6 +1065,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 					DeleteMarkerReplicationStatus: objects[i].DeleteMarkerReplicationStatus,
 					VersionPurgeStatus:            objects[i].VersionPurgeStatus,
 				}
+				versions[i].SetTierFreeVersionID(mustGetUUID())
 				if opts.Versioned {
 					versions[i].VersionID = uuid
 				}
@@ -1394,11 +1519,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		}
 	}
 
-	// object content is small enough that it's inlined, skipping transition
-	if fi.InlineData() {
-		return nil
-	}
-
 	destObj, err := genTransitionObjName(bucket)
 	if err != nil {
 		return err
@@ -1435,16 +1555,15 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		er.addPartial(bucket, object, opts.VersionID, -1)
 		break
 	}
-	// Notify object deleted event.
+
+	objInfo := fi.ToObjectInfo(bucket, object)
 	sendEvent(eventArgs{
 		EventName:  eventName,
 		BucketName: bucket,
-		Object: ObjectInfo{
-			Name:      object,
-			VersionID: opts.VersionID,
-		},
-		Host: "Internal: [ILM-Transition]",
+		Object:     objInfo,
+		Host:       "Internal: [ILM-Transition]",
 	})
+	auditLogLifecycle(ctx, objInfo, ILMTransition)
 	return err
 }
 

cmd/erasure-server-pool.go

@@ -273,12 +273,7 @@ type poolObjInfo struct {
 	Err       error
 }
 
-// getPoolIdxExisting returns the (first) found object pool index containing an object.
-// If the object exists, but the latest version is a delete marker, the index with it is still returned.
-// If the object does not exist ObjectNotFound error is returned.
-// If any other error is found, it is returned.
-// The check is skipped if there is only one zone, and 0, nil is always returned in that case.
-func (z *erasureServerPools) getPoolIdxExisting(ctx context.Context, bucket, object string) (idx int, err error) {
+func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (idx int, err error) {
 	if z.SinglePool() {
 		return 0, nil
 	}
@@ -294,7 +289,7 @@ func (z *erasureServerPools) getPoolIdxExisting(ctx context.Context, bucket, obj
 			pinfo := poolObjInfo{
 				PoolIndex: i,
 			}
-			pinfo.ObjInfo, pinfo.Err = pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+			pinfo.ObjInfo, pinfo.Err = pool.GetObjectInfo(ctx, bucket, object, opts)
 			poolObjInfos[i] = pinfo
 		}(i, pool)
 	}
@@ -331,6 +326,19 @@ func (z *erasureServerPools) getPoolIdxExisting(ctx context.Context, bucket, obj
 	return -1, toObjectErr(errFileNotFound, bucket, object)
 }
 
+func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) {
+	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{NoLock: true})
+}
+
+// getPoolIdxExisting returns the (first) found object pool index containing an object.
+// If the object exists, but the latest version is a delete marker, the index with it is still returned.
+// If the object does not exist ObjectNotFound error is returned.
+// If any other error is found, it is returned.
+// The check is skipped if there is only one zone, and 0, nil is always returned in that case.
+func (z *erasureServerPools) getPoolIdxExisting(ctx context.Context, bucket, object string) (idx int, err error) {
+	return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{})
+}
+
 // getPoolIdx returns the found previous object and its corresponding pool idx,
 // if none are found falls back to most available space pool.
 func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
@@ -837,27 +845,6 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		objSets.Add(objects[i].ObjectName)
 	}
 
-	poolObjIdxMap := map[int][]ObjectToDelete{}
-	origIndexMap := map[int][]int{}
-	if !z.SinglePool() {
-		for j, obj := range objects {
-			idx, err := z.getPoolIdxExisting(ctx, bucket, obj.ObjectName)
-			if isErrObjectNotFound(err) {
-				derrs[j] = err
-				continue
-			}
-			if err != nil {
-				// Unhandled errors return right here.
-				for i := range derrs {
-					derrs[i] = err
-				}
-				return dobjects, derrs
-			}
-			poolObjIdxMap[idx] = append(poolObjIdxMap[idx], obj)
-			origIndexMap[idx] = append(origIndexMap[idx], j)
-		}
-	}
-
 	// Acquire a bulk write lock across 'objects'
 	multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...)
 	lkctx, err := multiDeleteLock.GetLock(ctx, globalOperationTimeout)
@@ -874,17 +861,65 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		return z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
 	}
 
-	for idx, pool := range z.serverPools {
-		objs := poolObjIdxMap[idx]
-		orgIndexes := origIndexMap[idx]
-		deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
-		for i, derr := range errs {
-			if derr != nil {
-				derrs[orgIndexes[i]] = derr
+	// Fetch location of up to 10 objects concurrently.
+	poolObjIdxMap := map[int][]ObjectToDelete{}
+	origIndexMap := map[int][]int{}
+
+	var mu sync.Mutex
+	eg := errgroup.WithNErrs(len(objects)).WithConcurrency(10)
+	cctx, cancel := eg.WithCancelOnError(ctx)
+	defer cancel()
+	for j, obj := range objects {
+		j := j
+		obj := obj
+		eg.Go(func() error {
+			idx, err := z.getPoolIdxExistingNoLock(cctx, bucket, obj.ObjectName)
+			if isErrObjectNotFound(err) {
+				derrs[j] = err
+				return nil
 			}
-			dobjects[orgIndexes[i]] = deletedObjects[i]
-		}
+			if err != nil {
+				// unhandled errors return right here.
+				return err
+			}
+			mu.Lock()
+			poolObjIdxMap[idx] = append(poolObjIdxMap[idx], obj)
+			origIndexMap[idx] = append(origIndexMap[idx], j)
+			mu.Unlock()
+			return nil
+		}, j)
 	}
+
+	if err := eg.WaitErr(); err != nil {
+		for i := range derrs {
+			derrs[i] = err
+		}
+		return dobjects, derrs
+	}
+
+	// Delete concurrently in all server pools.
+	var wg sync.WaitGroup
+	wg.Add(len(z.serverPools))
+	for idx, pool := range z.serverPools {
+		go func(idx int, pool *erasureSets) {
+			defer wg.Done()
+			objs := poolObjIdxMap[idx]
+			if len(objs) > 0 {
+				orgIndexes := origIndexMap[idx]
+				deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
+				mu.Lock()
+				for i, derr := range errs {
+					if derr != nil {
+						derrs[orgIndexes[i]] = derr
+					}
+					dobjects[orgIndexes[i]] = deletedObjects[i]
+				}
+				mu.Unlock()
+			}
+		}(idx, pool)
+	}
+	wg.Wait()
+
 	return dobjects, derrs
 }
 
@@ -984,6 +1019,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	if err != nil && err != io.EOF {
 		return loi, err
 	}
+	defer merged.truncate(0) // Release when returning
 	if versionMarker == "" {
 		o := listPathOptions{Marker: marker}
 		// If we are not looking for a specific version skip it.
@@ -1040,6 +1076,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 
 	merged.forwardPast(opts.Marker)
+	defer merged.truncate(0) // Release when returning
 
 	// Default is recursive, if delimiter is set then list non recursive.
 	objects := merged.fileInfos(bucket, prefix, delimiter)
@@ -1644,7 +1681,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 						path = prefix
 					}
 
-					if err := listPathRaw(ctx, listPathRawOptions{
+					lopts := listPathRawOptions{
 						disks:          disks,
 						bucket:         bucket,
 						path:           path,
@@ -1660,7 +1697,9 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 							}
 						},
 						finished: nil,
-					}); err != nil {
+					}
+					if err := listPathRaw(ctx, lopts); err != nil {
+						errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
 						cancel()
 						return
 					}

cmd/erasure-sets.go

@@ -878,18 +878,10 @@ func (s *erasureSets) GetObjectNInfo(ctx context.Context, bucket, object string,
 	return set.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 }
 
-func (s *erasureSets) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
-	if parent == "." {
-		return false
-	}
-	return s.getHashedSet(parent).parentDirIsObject(ctx, bucket, parent)
-}
-
 // PutObject - writes an object to hashedSet based on the object name.
 func (s *erasureSets) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	set := s.getHashedSet(object)
 	auditObjectErasureSet(ctx, object, set)
-	opts.ParentIsObject = s.parentDirIsObject
 	return set.PutObject(ctx, bucket, object, data, opts)
 }
 
@@ -1074,7 +1066,6 @@ func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object,
 func (s *erasureSets) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	set := s.getHashedSet(object)
 	auditObjectErasureSet(ctx, object, set)
-	opts.ParentIsObject = s.parentDirIsObject
 	return set.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 }
 

cmd/erasure.go

@@ -482,11 +482,10 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 							Entry:  update,
 						}
 						if intDataUpdateTracker.debug {
-							console.Debugln("bucket", bucket.Name, "got update", update)
+							console.Debugln("z:", er.poolIndex, "s:", er.setIndex, "bucket", name, "got update", update)
 						}
 					}
 				}(cache.Info.Name)
-
 				// Calc usage
 				before := cache.Info.LastUpdate
 				var err error

cmd/format-erasure.go

@@ -705,8 +705,10 @@ func saveUnformattedFormat(ctx context.Context, storageDisks []StorageAPI, forma
 		if format == nil {
 			continue
 		}
-		if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
-			return err
+		if storageDisks[index] != nil && storageDisks[index].IsOnline() {
+			if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+				return err
+			}
 		}
 	}
 	return nil

cmd/fs-v1-helpers.go

@@ -24,6 +24,7 @@ import (
 	pathutil "path"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/minio/minio/internal/lock"
 	"github.com/minio/minio/internal/logger"
@@ -168,6 +169,25 @@ func fsStat(ctx context.Context, statLoc string) (os.FileInfo, error) {
 	return fi, nil
 }
 
+// fsTouch updates a file access & modtime with current time
+func fsTouch(ctx context.Context, statLoc string) error {
+	if statLoc == "" {
+		logger.LogIf(ctx, errInvalidArgument)
+		return errInvalidArgument
+	}
+	if err := checkPathLength(statLoc); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	now := time.Now()
+	if err := os.Chtimes(statLoc, now, now); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+
+	return nil
+}
+
 // Lookup if volume exists, returns volume attributes upon success.
 func fsStatVolume(ctx context.Context, volume string) (os.FileInfo, error) {
 	fi, err := fsStat(ctx, volume)

cmd/fs-v1-metadata.go

@@ -29,6 +29,8 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/etag"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/lock"
 	"github.com/minio/minio/internal/logger"
@@ -167,6 +169,12 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	}
 
 	objInfo.ETag = extractETag(m.Meta)
+
+	et, e := etag.Parse(objInfo.ETag)
+	if e == nil {
+		objInfo.Multipart = et.IsMultipart()
+	}
+
 	objInfo.ContentType = m.Meta["content-type"]
 	objInfo.ContentEncoding = m.Meta["content-encoding"]
 	if storageClass, ok := m.Meta[xhttp.AmzStorageClass]; ok {
@@ -176,7 +184,6 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	}
 	var (
 		t time.Time
-		e error
 	)
 	if exp, ok := m.Meta["expires"]; ok {
 		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
@@ -193,6 +200,14 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(m.Meta)
 
+	// Set multipart for encryption properly if
+	// not set already.
+	if !objInfo.Multipart {
+		if _, ok := crypto.IsEncrypted(objInfo.UserDefined); ok {
+			objInfo.Multipart = crypto.IsMultiPart(objInfo.UserDefined)
+		}
+	}
+
 	// All the parts per object.
 	objInfo.Parts = m.Parts
 

cmd/fs-v1-multipart.go

@@ -24,7 +24,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	pathutil "path"
 	"sort"
 	"strconv"
 	"strings"
@@ -550,11 +549,6 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, toObjectErr(err)
 	}
 
-	// Check if an object is present as one of the parent dir.
-	if fs.parentDirIsObject(ctx, bucket, pathutil.Dir(object)) {
-		return oi, toObjectErr(errFileParentIsFile, bucket, object)
-	}
-
 	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
 		return oi, toObjectErr(err, bucket)
 	}

cmd/fs-v1.go

@@ -682,8 +682,15 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 			return oi, toObjectErr(err, srcBucket, srcObject)
 		}
 
-		// Stat the file to get file size.
-		fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, srcBucket, srcObject))
+		fsObjectPath := pathJoin(fs.fsPath, srcBucket, srcObject)
+
+		// Update object modtime
+		err = fsTouch(ctx, fsObjectPath)
+		if err != nil {
+			return oi, toObjectErr(err, srcBucket, srcObject)
+		}
+		// Stat the file to get object info
+		fi, err := fsStatFile(ctx, fsObjectPath)
 		if err != nil {
 			return oi, toObjectErr(err, srcBucket, srcObject)
 		}
@@ -999,26 +1006,6 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 	return oi, toObjectErr(err, bucket, object)
 }
 
-// This function does the following check, suppose
-// object is "a/b/c/d", stat makes sure that objects ""a/b/c""
-// "a/b" and "a" do not exist.
-func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
-	var isParentDirObject func(string) bool
-	isParentDirObject = func(p string) bool {
-		if p == "." || p == SlashSeparator {
-			return false
-		}
-		if fsIsFile(ctx, pathJoin(fs.fsPath, bucket, p)) {
-			// If there is already a file at prefix "p", return true.
-			return true
-		}
-
-		// Check if there is a file as one of the parent paths.
-		return isParentDirObject(path.Dir(p))
-	}
-	return isParentDirObject(parent)
-}
-
 // PutObject - creates an object upon reading from the input stream
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
@@ -1072,10 +1059,6 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	// with a slash separator, we treat it like a valid operation
 	// and return success.
 	if isObjectDir(object, data.Size()) {
-		// Check if an object is present as one of the parent dir.
-		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
-		}
 		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			logger.LogIf(ctx, err)
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
@@ -1087,11 +1070,6 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 		return fsMeta.ToObjectInfo(bucket, object, fi), nil
 	}
 
-	// Check if an object is present as one of the parent dir.
-	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
-	}
-
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument, logger.Application)

cmd/fs-v1_test.go

@@ -26,71 +26,6 @@ import (
 	"github.com/minio/madmin-go"
 )
 
-// Tests for if parent directory is object
-func TestFSParentDirIsObject(t *testing.T) {
-	obj, disk, err := prepareFS()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(disk)
-
-	bucketName := "testbucket"
-	objectName := "object"
-
-	if err = obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
-		t.Fatal(err)
-	}
-	objectContent := "12345"
-	objInfo, err := obj.PutObject(GlobalContext, bucketName, objectName,
-		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if objInfo.Name != objectName {
-		t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName)
-	}
-
-	fs := obj.(*FSObjects)
-	testCases := []struct {
-		parentIsObject bool
-		objectName     string
-	}{
-		// parentIsObject is true if object is available.
-		{
-			parentIsObject: true,
-			objectName:     objectName,
-		},
-		{
-			parentIsObject: false,
-			objectName:     "",
-		},
-		{
-			parentIsObject: false,
-			objectName:     ".",
-		},
-		// Should not cause infinite loop.
-		{
-			parentIsObject: false,
-			objectName:     SlashSeparator,
-		},
-		{
-			parentIsObject: false,
-			objectName:     "\\",
-		},
-		// Should not cause infinite loop with double forward slash.
-		{
-			parentIsObject: false,
-			objectName:     "//",
-		},
-	}
-	for i, testCase := range testCases {
-		gotValue := fs.parentDirIsObject(GlobalContext, bucketName, testCase.objectName)
-		if testCase.parentIsObject != gotValue {
-			t.Errorf("Test %d: Unexpected value returned got %t, expected %t", i+1, gotValue, testCase.parentIsObject)
-		}
-	}
-}
-
 // TestNewFS - tests initialization of all input disks
 // and constructs a valid `FS` object layer.
 func TestNewFS(t *testing.T) {
@@ -226,35 +161,6 @@ func TestFSPutObject(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(GlobalContext, bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
-	if err == nil {
-		t.Fatal("Unexpected should fail here, backend corruption occurred")
-	}
-	if nerr, ok := err.(ParentIsObject); !ok {
-		t.Fatalf("Expected ParentIsObject, got %#v", err)
-	} else {
-		if nerr.Bucket != "bucket" {
-			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
-		}
-		if nerr.Object != "1/2/3/4/object/1" {
-			t.Fatalf("Expected '1/2/3/4/object/1', got %s", nerr.Object)
-		}
-	}
-
-	_, err = obj.PutObject(GlobalContext, bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
-	if err == nil {
-		t.Fatal("Unexpected should fail here, backned corruption occurred")
-	}
-	if nerr, ok := err.(ParentIsObject); !ok {
-		t.Fatalf("Expected ParentIsObject, got %#v", err)
-	} else {
-		if nerr.Bucket != "bucket" {
-			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
-		}
-		if nerr.Object != "1/2/3/4/object/1/" {
-			t.Fatalf("Expected '1/2/3/4/object/1/', got %s", nerr.Object)
-		}
-	}
 }
 
 // TestFSDeleteObject - test fs.DeleteObject() with healthy and corrupted disks

cmd/generic-handlers.go

@@ -406,7 +406,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 			return
 		}
 		// Check for bad components in URL query values.
-		for _, vv := range r.URL.Query() {
+		for _, vv := range r.Form {
 			for _, v := range vv {
 				if hasBadPathComponent(v) {
 					writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
@@ -436,55 +436,6 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 			return
 		}
 
-		// For browser requests, when federation is setup we need to
-		// specifically handle download and upload for browser requests.
-		if guessIsBrowserReq(r) {
-			var bucket, _ string
-			switch r.Method {
-			case http.MethodPut:
-				if getRequestAuthType(r) == authTypeJWT {
-					bucket, _ = path2BucketObjectWithBasePath(minioReservedBucketPath+"/upload", r.URL.Path)
-				}
-			case http.MethodGet:
-				if t := r.URL.Query().Get("token"); t != "" {
-					bucket, _ = path2BucketObjectWithBasePath(minioReservedBucketPath+"/download", r.URL.Path)
-				} else if getRequestAuthType(r) != authTypeJWT && !strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
-					bucket, _ = request2BucketObjectName(r)
-				}
-			}
-			if bucket == "" {
-				h.ServeHTTP(w, r)
-				return
-			}
-			sr, err := globalDNSConfig.Get(bucket)
-			if err != nil {
-				if err == dns.ErrNoEntriesFound {
-					writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNoSuchBucket),
-						r.URL)
-				} else {
-					writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err),
-						r.URL)
-				}
-				return
-			}
-			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
-				r.URL.Scheme = "http"
-				if globalIsTLS {
-					r.URL.Scheme = "https"
-				}
-				r.URL.Host = getHostFromSrv(sr)
-				// Make sure we remove any existing headers before
-				// proxying the request to another node.
-				for k := range w.Header() {
-					w.Header().Del(k)
-				}
-				globalForwarder.ServeHTTP(w, r)
-				return
-			}
-			h.ServeHTTP(w, r)
-			return
-		}
-
 		bucket, object := request2BucketObjectName(r)
 
 		// Requests in federated setups for STS type calls which are
@@ -560,7 +511,6 @@ func addSecurityHeaders(h http.Handler) http.Handler {
 		header := w.Header()
 		header.Set("X-XSS-Protection", "1; mode=block")                                // Prevents against XSS attacks
 		header.Set("Content-Security-Policy", "block-all-mixed-content")               // prevent mixed (HTTP / HTTPS content)
-		header.Set("X-Frame-Options", "deny")                                          // Prevent against Clickjack attack
 		header.Set("X-Content-Type-Options", "nosniff")                                // Prevent mime-sniff
 		header.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains") // HSTS mitigates variants of MITM attacks
 		h.ServeHTTP(w, r)

cmd/global-heal.go

@@ -68,17 +68,6 @@ func newBgHealSequence() *healSequence {
 	}
 }
 
-func getCurrentMRFStatus() madmin.MRFStatus {
-	mrfInfo := globalMRFState.getCurrentMRFRoundInfo()
-	return madmin.MRFStatus{
-		BytesHealed: mrfInfo.bytesHealed,
-		ItemsHealed: mrfInfo.itemsHealed,
-		TotalItems:  mrfInfo.itemsHealed + mrfInfo.pendingItems,
-		TotalBytes:  mrfInfo.bytesHealed + mrfInfo.pendingBytes,
-		Started:     mrfInfo.triggeredAt,
-	}
-}
-
 // getBackgroundHealStatus will return the
 func getBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealState, bool) {
 	if globalBackgroundHealState == nil {
@@ -94,9 +83,9 @@ func getBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealS
 		ScannedItemsCount: bgSeq.getScannedItemsCount(),
 	}
 
-	if globalMRFState != nil {
+	if globalMRFState.initialized() {
 		status.MRF = map[string]madmin.MRFStatus{
-			globalLocalNodeName: getCurrentMRFStatus(),
+			globalLocalNodeName: globalMRFState.getCurrentMRFRoundInfo(),
 		}
 	}
 
@@ -283,6 +272,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketIn
 			finished: nil,
 		})
 
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return err
+		}
+
 		select {
 		// If context is canceled don't mark as done...
 		case <-ctx.Done():

cmd/globals.go

@@ -107,7 +107,7 @@ const (
 	maxBucketSSEConfigSize = 1 * humanize.MiByte
 
 	// diskFillFraction is the fraction of a disk we allow to be filled.
-	diskFillFraction = 0.95
+	diskFillFraction = 0.99
 
 	// diskAssumeUnknownSize is the size to assume when an unknown size upload is requested.
 	diskAssumeUnknownSize = 1 << 30
@@ -216,6 +216,9 @@ var (
 	// The name of this local node, fetched from arguments
 	globalLocalNodeName string
 
+	// The global subnet license
+	globalSubnetLicense string
+
 	globalRemoteEndpoints map[string]Endpoint
 
 	// Global server's network statistics
@@ -293,7 +296,7 @@ var (
 	globalBackgroundHealRoutine *healRoutine
 	globalBackgroundHealState   *allHealState
 
-	globalMRFState *mrfState
+	globalMRFState mrfState
 
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool

cmd/handler-api.go

@@ -39,6 +39,7 @@ type apiConfig struct {
 	totalDriveCount          int
 	replicationWorkers       int
 	replicationFailedWorkers int
+	transitionWorkers        int
 }
 
 func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
@@ -87,6 +88,10 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	}
 	t.replicationFailedWorkers = cfg.ReplicationFailedWorkers
 	t.replicationWorkers = cfg.ReplicationWorkers
+	if globalTransitionState != nil && cfg.TransitionWorkers != t.transitionWorkers {
+		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
+	}
+	t.transitionWorkers = cfg.TransitionWorkers
 }
 
 func (t *apiConfig) getListQuorum() int {
@@ -173,3 +178,10 @@ func (t *apiConfig) getReplicationWorkers() int {
 
 	return t.replicationWorkers
 }
+
+func (t *apiConfig) getTransitionWorkers() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.transitionWorkers
+}

cmd/handler-utils.go

@@ -112,7 +112,7 @@ var userMetadataKeyPrefixes = []string{
 
 // extractMetadata extracts metadata from HTTP header and HTTP queryString.
 func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]string, err error) {
-	query := r.URL.Query()
+	query := r.Form
 	header := r.Header
 	metadata = make(map[string]string)
 	// Extract all query values.

cmd/healthcheck-handler.go

@@ -42,7 +42,7 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
-	opts := HealthOptions{Maintenance: r.URL.Query().Get("maintenance") == "true"}
+	opts := HealthOptions{Maintenance: r.Form.Get("maintenance") == "true"}
 	result := objLayer.Health(ctx, opts)
 	if result.WriteQuorum > 0 {
 		w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))

cmd/http-stats.go

@@ -38,12 +38,12 @@ type ConnStats struct {
 }
 
 // Increase total input bytes
-func (s *ConnStats) incInputBytes(n int) {
+func (s *ConnStats) incInputBytes(n int64) {
 	atomic.AddUint64(&s.totalInputBytes, uint64(n))
 }
 
 // Increase total output bytes
-func (s *ConnStats) incOutputBytes(n int) {
+func (s *ConnStats) incOutputBytes(n int64) {
 	atomic.AddUint64(&s.totalOutputBytes, uint64(n))
 }
 
@@ -58,12 +58,12 @@ func (s *ConnStats) getTotalOutputBytes() uint64 {
 }
 
 // Increase outbound input bytes
-func (s *ConnStats) incS3InputBytes(n int) {
+func (s *ConnStats) incS3InputBytes(n int64) {
 	atomic.AddUint64(&s.s3InputBytes, uint64(n))
 }
 
 // Increase outbound output bytes
-func (s *ConnStats) incS3OutputBytes(n int) {
+func (s *ConnStats) incS3OutputBytes(n int64) {
 	atomic.AddUint64(&s.s3OutputBytes, uint64(n))
 }
 

cmd/http-tracer.go

@@ -30,11 +30,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/logger"
-	jsonrpc "github.com/minio/rpc"
 )
 
 // recordRequest - records the first recLen bytes
@@ -110,75 +108,6 @@ func getOpName(name string) (op string) {
 	return op
 }
 
-// WebTrace gets trace of web request
-func WebTrace(ri *jsonrpc.RequestInfo) madmin.TraceInfo {
-	r := ri.Request
-	w := ri.ResponseWriter
-
-	name := ri.Method
-	// Setup a http request body recorder
-	reqHeaders := r.Header.Clone()
-	reqHeaders.Set("Host", r.Host)
-	if len(r.TransferEncoding) == 0 {
-		reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
-	} else {
-		reqHeaders.Set("Transfer-Encoding", strings.Join(r.TransferEncoding, ","))
-	}
-
-	now := time.Now().UTC()
-	t := madmin.TraceInfo{TraceType: madmin.TraceHTTP, FuncName: name, Time: now}
-	t.NodeName = r.Host
-	if globalIsDistErasure {
-		t.NodeName = globalLocalNodeName
-	}
-	if t.NodeName == "" {
-		t.NodeName = globalLocalNodeName
-	}
-
-	// strip only standard port from the host address
-	if host, port, err := net.SplitHostPort(t.NodeName); err == nil {
-		if port == "443" || port == "80" {
-			t.NodeName = host
-		}
-	}
-
-	vars := mux.Vars(r)
-	rq := madmin.TraceRequestInfo{
-		Time:     now,
-		Proto:    r.Proto,
-		Method:   r.Method,
-		Path:     SlashSeparator + pathJoin(vars["bucket"], vars["object"]),
-		RawQuery: redactLDAPPwd(r.URL.RawQuery),
-		Client:   handlers.GetSourceIP(r),
-		Headers:  reqHeaders,
-	}
-
-	rw, ok := w.(*logger.ResponseWriter)
-	if ok {
-		rs := madmin.TraceResponseInfo{
-			Time:       time.Now().UTC(),
-			Headers:    rw.Header().Clone(),
-			StatusCode: rw.StatusCode,
-			Body:       logger.BodyPlaceHolder,
-		}
-
-		if rs.StatusCode == 0 {
-			rs.StatusCode = http.StatusOK
-		}
-
-		t.RespInfo = rs
-		t.CallStats = madmin.TraceCallStats{
-			Latency:         rs.Time.Sub(rw.StartTime),
-			InputBytes:      int(r.ContentLength),
-			OutputBytes:     rw.Size(),
-			TimeToFirstByte: rw.TimeToFirstByte,
-		}
-	}
-
-	t.ReqInfo = rq
-	return t
-}
-
 // Trace gets trace of http request
 func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Request) madmin.TraceInfo {
 	name := getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())

cmd/iam.go

@@ -664,6 +664,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 			for {
 				time.Sleep(globalRefreshIAMInterval)
 				sys.purgeExpiredCredentialsForLDAP(ctx)
+				sys.updateGroupMembershipsForLDAP(ctx)
 			}
 		}()
 	}
@@ -1165,6 +1166,9 @@ type newServiceAccountOpts struct {
 	sessionPolicy *iampolicy.Policy
 	accessKey     string
 	secretKey     string
+
+	// LDAP username
+	ldapUsername string
 }
 
 // NewServiceAccount - create a new service account
@@ -1222,6 +1226,11 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		m[iamPolicyClaimNameSA()] = "inherited-policy"
 	}
 
+	// For LDAP service account, save the ldap username in the metadata.
+	if opts.ldapUsername != "" {
+		m[ldapUserN] = opts.ldapUsername
+	}
+
 	var (
 		cred auth.Credentials
 	)
@@ -1597,7 +1606,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	}
 	sys.store.unlock()
 
-	expiredUsers, err := globalLDAPConfig.GetNonExistentUserDNS(parentUsers)
+	expiredUsers, err := globalLDAPConfig.GetNonEligibleUserDistNames(parentUsers)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		logger.LogIf(GlobalContext, err)
@@ -1627,6 +1636,92 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	sys.store.unlock()
 }
 
+// updateGroupMembershipsForLDAP - updates the list of groups associated with the credential.
+func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
+	// 1. Collect all LDAP users with active creds.
+	sys.store.lock()
+	// List of unique LDAP (parent) user DNs that have active creds
+	parentUsers := make([]string, 0, len(sys.iamUsersMap))
+	// Map of LDAP user to list of active credential objects
+	parentUserToCredsMap := make(map[string][]auth.Credentials, len(sys.iamUsersMap))
+	// DN to ldap username mapping for each LDAP user
+	parentUserToLDAPUsernameMap := make(map[string]string, len(sys.iamUsersMap))
+	for _, cred := range sys.iamUsersMap {
+		if cred.IsServiceAccount() || cred.IsTemp() {
+			if globalLDAPConfig.IsLDAPUserDN(cred.ParentUser) {
+				// Check if this is the first time we are
+				// encountering this LDAP user.
+				if _, ok := parentUserToCredsMap[cred.ParentUser]; !ok {
+					// Try to find the ldapUsername for this
+					// parentUser by extracting JWT claims
+					jwtClaims, err := auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+					if err != nil {
+						// skip this cred - session token seems
+						// invalid
+						continue
+					}
+					ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
+					if !ok {
+						// skip this cred - we dont have the
+						// username info needed
+						continue
+					}
+
+					// Collect each new cred.ParentUser into parentUsers
+					parentUsers = append(parentUsers, cred.ParentUser)
+
+					// Update the ldapUsernameMap
+					parentUserToLDAPUsernameMap[cred.ParentUser] = ldapUsername
+				}
+				parentUserToCredsMap[cred.ParentUser] = append(parentUserToCredsMap[cred.ParentUser], cred)
+			}
+		}
+	}
+	sys.store.unlock()
+
+	// 2. Query LDAP server for groups of the LDAP users collected.
+	updatedGroups, err := globalLDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
+	if err != nil {
+		// Log and return on error - perhaps it'll work the next time.
+		logger.LogIf(GlobalContext, err)
+		return
+	}
+
+	// 3. Update creds for those users whose groups are changed
+	sys.store.lock()
+	defer sys.store.unlock()
+	for _, parentUser := range parentUsers {
+		currGroupsSet := updatedGroups[parentUser]
+		currGroups := currGroupsSet.ToSlice()
+		for _, cred := range parentUserToCredsMap[parentUser] {
+			gSet := set.CreateStringSet(cred.Groups...)
+			if gSet.Equals(currGroupsSet) {
+				// No change to groups memberships for this
+				// credential.
+				continue
+			}
+
+			cred.Groups = currGroups
+			userType := regUser
+			if cred.IsServiceAccount() {
+				userType = svcUser
+			} else if cred.IsTemp() {
+				userType = stsUser
+			}
+			// Overwrite the user identity here. As store should be
+			// atomic, it shouldn't cause any corruption.
+			if err := sys.store.saveUserIdentity(ctx, cred.AccessKey, userType, newUserIdentity(cred)); err != nil {
+				// Log and continue error - perhaps it'll work the next time.
+				logger.LogIf(GlobalContext, err)
+				continue
+			}
+			// If we wrote the updated creds to IAM storage, we can
+			// update the in memory map.
+			sys.iamUsersMap[cred.AccessKey] = cred
+		}
+	}
+}
+
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	if !sys.Initialized() {
@@ -2205,20 +2300,15 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 
 // IsAllowedLDAPSTS - checks for LDAP specific claims and values
 func (sys *IAMSys) IsAllowedLDAPSTS(args iampolicy.Args, parentUser string) bool {
-	parentInClaimIface, ok := args.Claims[ldapUser]
-	if ok {
-		parentInClaim, ok := parentInClaimIface.(string)
-		if !ok {
-			// ldap parentInClaim name is not a string reject it.
-			return false
-		}
-
-		if parentInClaim != parentUser {
-			// ldap claim has been modified maliciously reject it.
-			return false
-		}
-	} else {
-		// no ldap parentInClaim claim present reject it.
+	// parentUser value must match the ldap user in the claim.
+	if parentInClaimIface, ok := args.Claims[ldapUser]; !ok {
+		// no ldapUser claim present reject it.
+		return false
+	} else if parentInClaim, ok := parentInClaimIface.(string); !ok {
+		// not the right type, reject it.
+		return false
+	} else if parentInClaim != parentUser {
+		// ldap claim has been modified maliciously reject it.
 		return false
 	}
 
@@ -2353,13 +2443,6 @@ func isAllowedBySessionPolicy(args iampolicy.Args) (hasSessionPolicy bool, isAll
 		return
 	}
 
-	policyBytes, err := base64.StdEncoding.DecodeString(spolicyStr)
-	if err != nil {
-		// Got a malformed base64 string
-		return
-	}
-	spolicyStr = string(policyBytes)
-
 	// Check if policy is parseable.
 	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 	if err != nil {

cmd/listen-notification-handlers.go

@@ -65,7 +65,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 		}
 	}
 
-	values := r.URL.Query()
+	values := r.Form
 
 	var prefix string
 	if len(values[peerRESTListenPrefix]) > 1 {

cmd/lock-rest-server.go

@@ -63,15 +63,16 @@ func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
-	quorum, err := strconv.Atoi(r.URL.Query().Get(lockRESTQuorum))
+	values := r.Form
+	quorum, err := strconv.Atoi(values.Get(lockRESTQuorum))
 	if err != nil {
 		return args, err
 	}
 
 	args = dsync.LockArgs{
-		Owner:  r.URL.Query().Get(lockRESTOwner),
-		UID:    r.URL.Query().Get(lockRESTUID),
-		Source: r.URL.Query().Get(lockRESTSource),
+		Owner:  values.Get(lockRESTOwner),
+		UID:    values.Get(lockRESTUID),
+		Source: values.Get(lockRESTSource),
 		Quorum: quorum,
 	}
 

cmd/metacache-entries.go

@@ -37,6 +37,9 @@ type metaCacheEntry struct {
 
 	// cached contains the metadata if decoded.
 	cached *FileInfo
+
+	// Indicates the entry can be reused and only one reference to metadata is expected.
+	reusable bool
 }
 
 // isDir returns if the entry is representing a prefix directory.
@@ -345,6 +348,8 @@ type metaCacheEntriesSorted struct {
 	o metaCacheEntries
 	// list id is not serialized
 	listID string
+	// Reuse buffers
+	reuse bool
 }
 
 // shallowClone will create a shallow clone of the array objects,
@@ -490,6 +495,15 @@ func (m *metaCacheEntriesSorted) forwardTo(s string) {
 	idx := sort.Search(len(m.o), func(i int) bool {
 		return m.o[i].name >= s
 	})
+	if m.reuse {
+		for i, entry := range m.o[:idx] {
+			if len(entry.metadata) >= metaDataReadDefault && len(entry.metadata) < metaDataReadDefault*4 {
+				metaDataPool.Put(entry.metadata)
+			}
+			m.o[i].metadata = nil
+		}
+	}
+
 	m.o = m.o[idx:]
 }
 
@@ -501,6 +515,14 @@ func (m *metaCacheEntriesSorted) forwardPast(s string) {
 	idx := sort.Search(len(m.o), func(i int) bool {
 		return m.o[i].name > s
 	})
+	if m.reuse {
+		for i, entry := range m.o[:idx] {
+			if len(entry.metadata) >= metaDataReadDefault && len(entry.metadata) < metaDataReadDefault*4 {
+				metaDataPool.Put(entry.metadata)
+			}
+			m.o[i].metadata = nil
+		}
+	}
 	m.o = m.o[idx:]
 }
 
@@ -715,6 +737,14 @@ func (m *metaCacheEntriesSorted) truncate(n int) {
 		return
 	}
 	if len(m.o) > n {
+		if m.reuse {
+			for i, entry := range m.o[n:] {
+				if len(entry.metadata) >= metaDataReadDefault && len(entry.metadata) < metaDataReadDefault*4 {
+					metaDataPool.Put(entry.metadata)
+				}
+				m.o[n+i].metadata = nil
+			}
+		}
 		m.o = m.o[:n]
 	}
 }

cmd/metacache-server-pool.go

@@ -97,6 +97,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	o.parseMarker()
 	o.BaseDir = baseDirFromPrefix(o.Prefix)
 	o.Transient = o.Transient || isReservedOrInvalidBucket(o.Bucket, false)
+	o.SetFilter()
 	if o.Transient {
 		o.Create = false
 	}
@@ -162,6 +163,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 			if o.pool < len(z.serverPools) && o.set < len(z.serverPools[o.pool].sets) {
 				o.debugln("Resuming", o)
 				entries, err = z.serverPools[o.pool].sets[o.set].streamMetadataParts(ctx, *o)
+				entries.reuse = true // We read from stream and are not sharing results.
 				if err == nil {
 					return entries, nil
 				}
@@ -185,9 +187,11 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 			rpc := globalNotificationSys.restClientFromHash(o.Bucket)
 			ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
 			defer cancel()
-			if c, err := rpc.GetMetacacheListing(ctx, *o); err == nil {
+			c, err := rpc.GetMetacacheListing(ctx, *o)
+			if err == nil {
 				c.error = "no longer used"
 				c.status = scanStateError
+				rpc.UpdateMetacacheListing(ctx, *c)
 			}
 		}()
 		o.ID = ""
@@ -198,8 +202,8 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	// Create filter for results.
 	o.debugln("Raw List", o)
 	filterCh := make(chan metaCacheEntry, o.Limit)
-	filteredResults := o.gatherResults(filterCh)
 	listCtx, cancelList := context.WithCancel(ctx)
+	filteredResults := o.gatherResults(listCtx, filterCh)
 	var wg sync.WaitGroup
 	wg.Add(1)
 	var listErr error
@@ -217,6 +221,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	if listErr != nil && !errors.Is(listErr, context.Canceled) {
 		return entries, listErr
 	}
+	entries.reuse = true
 	truncated := entries.len() > o.Limit || err == nil
 	entries.truncate(o.Limit)
 	if !o.Transient && truncated {
@@ -341,7 +346,7 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
 	inCh := make(chan metaCacheEntry, metacacheBlockSize)
 	outCh := make(chan metaCacheEntry, o.Limit)
 
-	filteredResults := o.gatherResults(outCh)
+	filteredResults := o.gatherResults(ctx, outCh)
 
 	mc := o.newMetacache()
 	meta := metaCacheRPC{meta: &mc, cancel: cancel, rpc: globalNotificationSys.restClientFromHash(o.Bucket), o: *o}
@@ -363,13 +368,33 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
 		o.debugln("listAndSave: listing", o.ID, "finished with ", err)
 	}(*o)
 
+	// Keep track of when we return since we no longer have to send entries to output.
+	var funcReturned bool
+	var funcReturnedMu sync.Mutex
+	defer func() {
+		funcReturnedMu.Lock()
+		funcReturned = true
+		funcReturnedMu.Unlock()
+	}()
 	// Write listing to results and saver.
 	go func() {
+		var returned bool
 		for entry := range inCh {
-			outCh <- entry
+			if !returned {
+				funcReturnedMu.Lock()
+				returned = funcReturned
+				funcReturnedMu.Unlock()
+				outCh <- entry
+				if returned {
+					close(outCh)
+				}
+			}
+			entry.reusable = returned
 			saveCh <- entry
 		}
-		close(outCh)
+		if !returned {
+			close(outCh)
+		}
 		close(saveCh)
 	}()
 

cmd/metacache-set.go

@@ -131,20 +131,31 @@ func (o *listPathOptions) debugln(data ...interface{}) {
 
 // gatherResults will collect all results on the input channel and filter results according to the options.
 // Caller should close the channel when done.
-// The returned function will return the results once there is enough or input is closed.
-func (o *listPathOptions) gatherResults(in <-chan metaCacheEntry) func() (metaCacheEntriesSorted, error) {
+// The returned function will return the results once there is enough or input is closed,
+// or the context is canceled.
+func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCacheEntry) func() (metaCacheEntriesSorted, error) {
 	var resultsDone = make(chan metaCacheEntriesSorted)
 	// Copy so we can mutate
 	resCh := resultsDone
+	var done bool
+	var mu sync.Mutex
 	resErr := io.EOF
 
 	go func() {
 		var results metaCacheEntriesSorted
+		var returned bool
 		for entry := range in {
-			if resCh == nil {
+			if returned {
 				// past limit
 				continue
 			}
+			mu.Lock()
+			returned = done
+			mu.Unlock()
+			if returned {
+				resCh = nil
+				continue
+			}
 			if !o.IncludeDirectories && entry.isDir() {
 				continue
 			}
@@ -167,6 +178,7 @@ func (o *listPathOptions) gatherResults(in <-chan metaCacheEntry) func() (metaCa
 					resErr = nil
 					resCh <- results
 					resCh = nil
+					returned = true
 				}
 				continue
 			}
@@ -178,7 +190,15 @@ func (o *listPathOptions) gatherResults(in <-chan metaCacheEntry) func() (metaCa
 		}
 	}()
 	return func() (metaCacheEntriesSorted, error) {
-		return <-resultsDone, resErr
+		select {
+		case <-ctx.Done():
+			mu.Lock()
+			done = true
+			mu.Unlock()
+			return metaCacheEntriesSorted{}, ctx.Err()
+		case r := <-resultsDone:
+			return r, resErr
+		}
 	}
 }
 
@@ -351,18 +371,18 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 		// All operations are performed without locks, so we must be careful and allow for failures.
 		// Read metadata associated with the object from a disk.
 		if retries > 0 {
-			disks := er.getOnlineDisks()
-			if len(disks) == 0 {
-				time.Sleep(retryDelay)
-				retries++
-				continue
-			}
-
-			_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(0), "", false)
-			if err != nil {
-				time.Sleep(retryDelay)
-				retries++
-				continue
+			for _, disk := range er.getDisks() {
+				if disk == nil {
+					continue
+				}
+				_, err := disk.ReadVersion(ctx, minioMetaBucket,
+					o.objectPath(0), "", false)
+				if err != nil {
+					time.Sleep(retryDelay)
+					retries++
+					continue
+				}
+				break
 			}
 		}
 
@@ -421,20 +441,21 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 
 				if retries > 0 {
 					// Load from one disk only
-					disks := er.getOnlineDisks()
-					if len(disks) == 0 {
-						time.Sleep(retryDelay)
-						retries++
-						continue
-					}
-
-					_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(partN), "", false)
-					if err != nil {
-						time.Sleep(retryDelay)
-						retries++
-						continue
+					for _, disk := range er.getDisks() {
+						if disk == nil {
+							continue
+						}
+						_, err := disk.ReadVersion(ctx, minioMetaBucket,
+							o.objectPath(partN), "", false)
+						if err != nil {
+							time.Sleep(retryDelay)
+							retries++
+							continue
+						}
+						break
 					}
 				}
+
 				// Load first part metadata...
 				fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true)
 				if err != nil {
@@ -512,7 +533,8 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 
 	askDisks := o.AskDisks
 	listingQuorum := o.AskDisks - 1
-	disks := er.getOnlineDisks()
+	disks := er.getDisks()
+	var fallbackDisks []StorageAPI
 
 	// Special case: ask all disks if the drive count is 4
 	if askDisks == -1 || er.setDriveCount == 4 {
@@ -527,6 +549,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 		rand.Shuffle(len(disks), func(i, j int) {
 			disks[i], disks[j] = disks[j], disks[i]
 		})
+		fallbackDisks = disks[askDisks:]
 		disks = disks[:askDisks]
 	}
 
@@ -539,13 +562,14 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 
 	ctxDone := ctx.Done()
 	return listPathRaw(ctx, listPathRawOptions{
-		disks:        disks,
-		bucket:       o.Bucket,
-		path:         o.BaseDir,
-		recursive:    o.Recursive,
-		filterPrefix: o.FilterPrefix,
-		minDisks:     listingQuorum,
-		forwardTo:    o.Marker,
+		disks:         disks,
+		fallbackDisks: fallbackDisks,
+		bucket:        o.Bucket,
+		path:          o.BaseDir,
+		recursive:     o.Recursive,
+		filterPrefix:  o.FilterPrefix,
+		minDisks:      listingQuorum,
+		forwardTo:     o.Marker,
 		agreed: func(entry metaCacheEntry) {
 			select {
 			case <-ctxDone:
@@ -644,10 +668,8 @@ func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCache
 		r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
 		logger.LogIf(ctx, err)
 		custom := b.headerKV()
-		_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
-			UserDefined:    custom,
-			NoLock:         true, // No need to hold namespace lock, each prefix caches uniquely.
-			ParentIsObject: nil,
+		_, err = er.putMetacacheObject(ctx, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
+			UserDefined: custom,
 		})
 		if err != nil {
 			mc.setErr(err.Error())
@@ -710,9 +732,10 @@ func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCache
 }
 
 type listPathRawOptions struct {
-	disks        []StorageAPI
-	bucket, path string
-	recursive    bool
+	disks         []StorageAPI
+	fallbackDisks []StorageAPI
+	bucket, path  string
+	recursive     bool
 
 	// Only return results with this prefix.
 	filterPrefix string
@@ -752,10 +775,18 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	if len(disks) == 0 {
 		return fmt.Errorf("listPathRaw: 0 drives provided")
 	}
+
 	// Cancel upstream if we finish before we expect.
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	fallback := func(err error) bool {
+		if err == nil {
+			return false
+		}
+		return err.Error() == errUnformattedDisk.Error() ||
+			err.Error() == errVolumeNotFound.Error()
+	}
 	askDisks := len(disks)
 	readers := make([]*metacacheReader, askDisks)
 	for i := range disks {
@@ -768,15 +799,44 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 
 		// Send request to each disk.
 		go func() {
-			werr := d.WalkDir(ctx, WalkDirOptions{
-				Bucket:         opts.bucket,
-				BaseDir:        opts.path,
-				Recursive:      opts.recursive,
-				ReportNotFound: opts.reportNotFound,
-				FilterPrefix:   opts.filterPrefix,
-				ForwardTo:      opts.forwardTo,
-			}, w)
+			var werr error
+			if d == nil {
+				werr = errDiskNotFound
+			} else {
+				werr = d.WalkDir(ctx, WalkDirOptions{
+					Bucket:         opts.bucket,
+					BaseDir:        opts.path,
+					Recursive:      opts.recursive,
+					ReportNotFound: opts.reportNotFound,
+					FilterPrefix:   opts.filterPrefix,
+					ForwardTo:      opts.forwardTo,
+				}, w)
+			}
+
+			// fallback only when set.
+			if len(opts.fallbackDisks) > 0 && fallback(werr) {
+				// This fallback is only set when
+				// askDisks is less than total
+				// number of disks per set.
+				for _, fd := range opts.fallbackDisks {
+					if fd == nil {
+						continue
+					}
+					werr = fd.WalkDir(ctx, WalkDirOptions{
+						Bucket:         opts.bucket,
+						BaseDir:        opts.path,
+						Recursive:      opts.recursive,
+						ReportNotFound: opts.reportNotFound,
+						FilterPrefix:   opts.filterPrefix,
+						ForwardTo:      opts.forwardTo,
+					}, w)
+					if werr == nil {
+						break
+					}
+				}
+			}
 			w.CloseWithError(werr)
+
 			if werr != io.EOF && werr != nil &&
 				werr.Error() != errFileNotFound.Error() &&
 				werr.Error() != errVolumeNotFound.Error() &&
@@ -795,10 +855,8 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 		for i := range topEntries {
 			topEntries[i] = metaCacheEntry{}
 		}
-		select {
-		case <-ctx.Done():
+		if contextCanceled(ctx) {
 			return ctx.Err()
-		default:
 		}
 		for i, r := range readers {
 			if errs[i] != nil {
@@ -866,8 +924,12 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			var combinedErr []string
 			for i, err := range errs {
 				if err != nil {
-					combinedErr = append(combinedErr,
-						fmt.Sprintf("disk %s returned: %s", disks[i], err))
+					if disks[i] != nil {
+						combinedErr = append(combinedErr,
+							fmt.Sprintf("disk %s returned: %s", disks[i], err))
+					} else {
+						combinedErr = append(combinedErr, err.Error())
+					}
 				}
 			}
 			return errors.New(strings.Join(combinedErr, ", "))

cmd/metacache-stream.go

@@ -56,10 +56,11 @@ const metacacheStreamVersion = 2
 
 // metacacheWriter provides a serializer of metacache objects.
 type metacacheWriter struct {
-	mw        *msgp.Writer
-	creator   func() error
-	closer    func() error
-	blockSize int
+	mw          *msgp.Writer
+	creator     func() error
+	closer      func() error
+	blockSize   int
+	reuseBlocks bool
 
 	streamErr error
 	streamWg  sync.WaitGroup
@@ -141,6 +142,11 @@ func (w *metacacheWriter) write(objs ...metaCacheEntry) error {
 		if err != nil {
 			return err
 		}
+		if w.reuseBlocks || o.reusable {
+			if cap(o.metadata) >= metaDataReadDefault && cap(o.metadata) < metaDataReadDefault*4 {
+				metaDataPool.Put(o.metadata)
+			}
+		}
 	}
 
 	return nil
@@ -354,10 +360,14 @@ func (r *metacacheReader) next() (metaCacheEntry, error) {
 		r.err = err
 		return m, err
 	}
-	m.metadata, err = r.mr.ReadBytes(nil)
+	m.metadata, err = r.mr.ReadBytes(metaDataPool.Get().([]byte)[:0])
 	if err == io.EOF {
 		err = io.ErrUnexpectedEOF
 	}
+	if len(m.metadata) == 0 && cap(m.metadata) >= metaDataReadDefault {
+		metaDataPool.Put(m.metadata)
+		m.metadata = nil
+	}
 	r.err = err
 	return m, err
 }
@@ -510,13 +520,17 @@ func (r *metacacheReader) readN(n int, inclDeleted, inclDirs bool, prefix string
 			r.mr.R.Skip(1)
 			return metaCacheEntriesSorted{o: res}, io.EOF
 		}
-		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
+		if meta.metadata, err = r.mr.ReadBytes(metaDataPool.Get().([]byte)[:0]); err != nil {
 			if err == io.EOF {
 				err = io.ErrUnexpectedEOF
 			}
 			r.err = err
 			return metaCacheEntriesSorted{o: res}, err
 		}
+		if len(meta.metadata) == 0 && cap(meta.metadata) >= metaDataReadDefault {
+			metaDataPool.Put(meta.metadata)
+			meta.metadata = nil
+		}
 		if !inclDirs && meta.isDir() {
 			continue
 		}
@@ -565,13 +579,17 @@ func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry
 			r.err = err
 			return err
 		}
-		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
+		if meta.metadata, err = r.mr.ReadBytes(metaDataPool.Get().([]byte)[:0]); err != nil {
 			if err == io.EOF {
 				err = io.ErrUnexpectedEOF
 			}
 			r.err = err
 			return err
 		}
+		if len(meta.metadata) == 0 && cap(meta.metadata) >= metaDataReadDefault {
+			metaDataPool.Put(meta.metadata)
+			meta.metadata = nil
+		}
 		select {
 		case <-ctx.Done():
 			r.err = ctx.Err()

cmd/metacache-walk.go

@@ -22,7 +22,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -66,8 +65,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	}
 
 	// Stat a volume entry.
-	_, err = os.Lstat(volumeDir)
-	if err != nil {
+	if err = Access(volumeDir); err != nil {
 		if osIsNotExist(err) {
 			return errVolumeNotFound
 		} else if isSysErrIO(err) {
@@ -78,6 +76,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 
 	// Use a small block size to start sending quickly
 	w := newMetacacheWriter(wr, 16<<10)
+	w.reuseBlocks = true // We are not sharing results, so reuse buffers.
 	defer w.Close()
 	out, err := w.stream()
 	if err != nil {
@@ -100,7 +99,8 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				metadata: metadata,
 			}
 		} else {
-			if st, err := Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() {
+			st, sterr := Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile))
+			if sterr == nil && st.Mode().IsRegular() {
 				return errFileNotFound
 			}
 		}
@@ -110,6 +110,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	var scanDir func(path string) error
 
 	scanDir = func(current string) error {
+		// always skip the directory that doesn't match the prefix
+		if len(current) > 0 && !strings.HasPrefix(current, prefix) {
+			return nil
+		}
+
 		// Skip forward, if requested...
 		forward := ""
 		if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, current) {
@@ -118,7 +123,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				forward = forward[:idx]
 			}
 		}
-
 		if contextCanceled(ctx) {
 			return ctx.Err()
 		}
@@ -134,11 +138,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// Forward some errors?
 			return nil
 		}
+		if len(entries) == 0 {
+			return nil
+		}
 		dirObjects := make(map[string]struct{})
 		for i, entry := range entries {
-			if len(prefix) > 0 && !strings.HasPrefix(entry, prefix) {
-				continue
-			}
 			if len(forward) > 0 && entry < forward {
 				continue
 			}
@@ -195,7 +199,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		// Process in sort order.
 		sort.Strings(entries)
 		dirStack := make([]string, 0, 5)
-		prefix = "" // Remove prefix after first level.
 		if len(forward) > 0 {
 			idx := sort.SearchStrings(entries, forward)
 			if idx > 0 {
@@ -245,8 +248,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			case osIsNotExist(err):
 				meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1))
 				if err == nil {
-					// Maybe rename? Would make it inconsistent across disks though.
-					// os.Rename(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1), pathJoin(volumeDir, meta.name, xlStorageFormatFile))
 					// It was an object
 					out <- meta
 					continue
@@ -265,6 +266,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				logger.LogIf(ctx, err)
 			}
 		}
+
 		// If directory entry left on stack, pop it now.
 		for len(dirStack) > 0 {
 			pop := dirStack[len(dirStack)-1]
@@ -332,8 +334,8 @@ func (s *storageRESTServer) WalkDirHandler(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
-	prefix := r.URL.Query().Get(storageRESTPrefixFilter)
-	forward := r.URL.Query().Get(storageRESTForwardFilter)
+	prefix := r.Form.Get(storageRESTPrefixFilter)
+	forward := r.Form.Get(storageRESTForwardFilter)
 	writer := streamHTTPResponse(w)
 	writer.CloseWithError(s.storage.WalkDir(r.Context(), WalkDirOptions{
 		Bucket:         volume,

cmd/metrics-v2.go

@@ -68,6 +68,7 @@ const (
 	softwareSubsystem         MetricSubsystem = "software"
 	sysCallSubsystem          MetricSubsystem = "syscall"
 	usageSubsystem            MetricSubsystem = "usage"
+	ilmSubsystem              MetricSubsystem = "ilm"
 )
 
 // MetricName are the individual names for the metric.
@@ -121,6 +122,10 @@ const (
 	upTime           = "uptime_seconds"
 	memory           = "resident_memory_bytes"
 	cpu              = "cpu_total_seconds"
+
+	expiryPendingTasks     MetricName = "expiry_pending_tasks"
+	transitionPendingTasks MetricName = "transition_pending_tasks"
+	transitionActiveTasks  MetricName = "transition_active_tasks"
 )
 
 const (
@@ -249,6 +254,7 @@ func GetGeneratorsForPeer() []MetricsGenerator {
 		getMinioVersionMetrics,
 		getNetworkMetrics,
 		getS3TTFBMetric,
+		getILMNodeMetrics,
 	}
 	return g
 }
@@ -1000,6 +1006,66 @@ func getS3TTFBMetric() MetricsGroup {
 	}
 }
 
+func getTransitionPendingTasksMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: ilmSubsystem,
+		Name:      transitionPendingTasks,
+		Help:      "Number of pending ILM transition tasks in the queue.",
+		Type:      gaugeMetric,
+	}
+}
+
+func getTransitionActiveTasksMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: ilmSubsystem,
+		Name:      transitionActiveTasks,
+		Help:      "Number of active ILM transition tasks.",
+		Type:      gaugeMetric,
+	}
+}
+
+func getExpiryPendingTasksMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: ilmSubsystem,
+		Name:      expiryPendingTasks,
+		Help:      "Number of pending ILM expiry tasks in the queue.",
+		Type:      gaugeMetric,
+	}
+}
+
+func getILMNodeMetrics() MetricsGroup {
+	return MetricsGroup{
+		id:         "ILMNodeMetrics",
+		cachedRead: cachedRead,
+		read: func(_ context.Context) []Metric {
+			expPendingTasks := Metric{
+				Description: getExpiryPendingTasksMD(),
+			}
+			trPendingTasks := Metric{
+				Description: getTransitionPendingTasksMD(),
+			}
+			trActiveTasks := Metric{
+				Description: getTransitionActiveTasksMD(),
+			}
+			if globalExpiryState != nil {
+				expPendingTasks.Value = float64(globalExpiryState.PendingTasks())
+			}
+			if globalTransitionState != nil {
+				trPendingTasks.Value = float64(globalTransitionState.PendingTasks())
+				trActiveTasks.Value = float64(globalTransitionState.ActiveTasks())
+			}
+			return []Metric{
+				expPendingTasks,
+				trPendingTasks,
+				trActiveTasks,
+			}
+		},
+	}
+}
+
 func getMinioVersionMetrics() MetricsGroup {
 	return MetricsGroup{
 		id:         "MinioVersionMetrics",

cmd/mrf.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/madmin-go"
@@ -48,19 +49,11 @@ type setInfo struct {
 	index, pool int
 }
 
-type mrfStats struct {
-	triggeredAt time.Time
-
-	itemsHealed uint64
-	bytesHealed uint64
-
-	pendingItems uint64
-	pendingBytes uint64
-}
-
 // mrfState sncapsulates all the information
 // related to the global background MRF.
 type mrfState struct {
+	ready int32
+
 	ctx       context.Context
 	objectAPI ObjectLayer
 
@@ -77,9 +70,30 @@ type mrfState struct {
 	triggeredAt time.Time
 }
 
+// Initialize healing MRF subsystem
+func (m *mrfState) init(ctx context.Context, objAPI ObjectLayer) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	m.ctx = ctx
+	m.objectAPI = objAPI
+	m.opCh = make(chan partialOperation, mrfOpsQueueSize)
+	m.pendingOps = make(map[partialOperation]setInfo)
+	m.setReconnectEvent = make(chan setInfo)
+
+	go globalMRFState.maintainMRFList()
+	go globalMRFState.healRoutine()
+
+	atomic.StoreInt32(&m.ready, 1)
+}
+
+func (m *mrfState) initialized() bool {
+	return atomic.LoadInt32(&m.ready) != 0
+}
+
 // Add a partial S3 operation (put/delete) when one or more disks are offline.
 func (m *mrfState) addPartialOp(op partialOperation) {
-	if m == nil {
+	if !m.initialized() {
 		return
 	}
 
@@ -91,7 +105,7 @@ func (m *mrfState) addPartialOp(op partialOperation) {
 
 // Receive the new set (disk) reconnection event
 func (m *mrfState) newSetReconnected(pool, set int) {
-	if m == nil {
+	if !m.initialized() {
 		return
 	}
 
@@ -104,8 +118,8 @@ func (m *mrfState) newSetReconnected(pool, set int) {
 	}
 }
 
-// Get current MRF stats
-func (m *mrfState) getCurrentMRFRoundInfo() mrfStats {
+// Get current MRF stats of the last MRF activity
+func (m *mrfState) getCurrentMRFRoundInfo() madmin.MRFStatus {
 	m.mu.Lock()
 	triggeredAt := m.triggeredAt
 	itemsHealed := m.itemsHealed
@@ -115,15 +129,15 @@ func (m *mrfState) getCurrentMRFRoundInfo() mrfStats {
 	m.mu.Unlock()
 
 	if pendingItems == 0 {
-		return mrfStats{}
+		return madmin.MRFStatus{}
 	}
 
-	return mrfStats{
-		triggeredAt:  triggeredAt,
-		itemsHealed:  itemsHealed,
-		bytesHealed:  bytesHealed,
-		pendingItems: pendingItems,
-		pendingBytes: pendingBytes,
+	return madmin.MRFStatus{
+		Started:     triggeredAt,
+		ItemsHealed: itemsHealed,
+		BytesHealed: bytesHealed,
+		TotalItems:  itemsHealed + pendingItems,
+		TotalBytes:  bytesHealed + pendingBytes,
 	}
 }
 
@@ -230,13 +244,5 @@ func (m *mrfState) healRoutine() {
 
 // Initialize healing MRF
 func initHealMRF(ctx context.Context, obj ObjectLayer) {
-	globalMRFState = &mrfState{
-		ctx:               ctx,
-		objectAPI:         obj,
-		opCh:              make(chan partialOperation, mrfOpsQueueSize),
-		pendingOps:        make(map[partialOperation]setInfo),
-		setReconnectEvent: make(chan setInfo),
-	}
-	go globalMRFState.maintainMRFList()
-	go globalMRFState.healRoutine()
+	globalMRFState.init(ctx, obj)
 }

cmd/naughty-disk_test.go

@@ -218,13 +218,6 @@ func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string
 	return d.disk.CheckParts(ctx, volume, path, fi)
 }
 
-func (d *naughtyDisk) CheckFile(ctx context.Context, volume string, path string) (err error) {
-	if err := d.calcError(); err != nil {
-		return err
-	}
-	return d.disk.CheckFile(ctx, volume, path)
-}
-
 func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
 	if err := d.calcError(); err != nil {
 		return err

cmd/net.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"runtime"
 	"sort"
 	"strings"
 
@@ -46,20 +47,30 @@ func mustSplitHostPort(hostPort string) (host, port string) {
 // mustGetLocalIP4 returns IPv4 addresses of localhost.  It panics on error.
 func mustGetLocalIP4() (ipList set.StringSet) {
 	ipList = set.NewStringSet()
-	addrs, err := net.InterfaceAddrs()
+	ifs, err := net.Interfaces()
 	logger.FatalIf(err, "Unable to get IP addresses of this host")
 
-	for _, addr := range addrs {
-		var ip net.IP
-		switch v := addr.(type) {
-		case *net.IPNet:
-			ip = v.IP
-		case *net.IPAddr:
-			ip = v.IP
+	for _, interf := range ifs {
+		addrs, err := interf.Addrs()
+		if err != nil {
+			continue
+		}
+		if runtime.GOOS == "windows" && interf.Flags&net.FlagUp == 0 {
+			continue
 		}
 
-		if ip.To4() != nil {
-			ipList.Add(ip.String())
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+
+			if ip.To4() != nil {
+				ipList.Add(ip.String())
+			}
 		}
 	}
 
@@ -177,6 +188,9 @@ func getConsoleEndpoints() (consoleEndpoints []string) {
 }
 
 func getAPIEndpoints() (apiEndpoints []string) {
+	if globalMinioEndpoint != "" {
+		return []string{globalMinioEndpoint}
+	}
 	var ipList []string
 	if globalMinioHost == "" {
 		ipList = sortIPs(mustGetLocalIP4().ToSlice())

cmd/notification.go

@@ -995,12 +995,7 @@ func (sys *NotificationSys) GetCPUs(ctx context.Context) []madmin.CPUs {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
 		}
 	}
 	return reply
@@ -1025,12 +1020,7 @@ func (sys *NotificationSys) GetPartitions(ctx context.Context) []madmin.Partitio
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
 		}
 	}
 	return reply
@@ -1055,12 +1045,67 @@ func (sys *NotificationSys) GetOSInfo(ctx context.Context) []madmin.OSInfo {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
+		}
+	}
+	return reply
+}
+
+// GetSysServices - Get information about system services
+// (only the services that are of concern to minio)
+func (sys *NotificationSys) GetSysServices(ctx context.Context) []madmin.SysServices {
+	reply := make([]madmin.SysServices, len(sys.peerClients))
+
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			var err error
+			reply[index], err = sys.peerClients[index].GetSELinuxInfo(ctx)
+			return err
+		}, index)
+	}
+
+	for index, err := range g.Wait() {
+		if err != nil {
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
+		}
+	}
+	return reply
+}
+
+func (sys *NotificationSys) addNodeErr(nodeInfo madmin.NodeInfo, peerClient *peerRESTClient, err error) {
+	addr := peerClient.host.String()
+	reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+	logger.LogIf(ctx, err)
+	nodeInfo.SetAddr(addr)
+	nodeInfo.SetError(err.Error())
+}
+
+// GetSysErrors - Memory information
+func (sys *NotificationSys) GetSysErrors(ctx context.Context) []madmin.SysErrors {
+	reply := make([]madmin.SysErrors, len(sys.peerClients))
+
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			var err error
+			reply[index], err = sys.peerClients[index].GetSysErrors(ctx)
+			return err
+		}, index)
+	}
+
+	for index, err := range g.Wait() {
+		if err != nil {
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
 		}
 	}
 	return reply
@@ -1085,12 +1130,7 @@ func (sys *NotificationSys) GetMemInfo(ctx context.Context) []madmin.MemInfo {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
 		}
 	}
 	return reply
@@ -1115,12 +1155,7 @@ func (sys *NotificationSys) GetProcInfo(ctx context.Context) []madmin.ProcInfo {
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			addr := sys.peerClients[index].host.String()
-			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
-			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
-			reply[index].Addr = addr
-			reply[index].Error = err.Error()
+			sys.addNodeErr(&reply[index], sys.peerClients[index], err)
 		}
 	}
 	return reply
@@ -1427,3 +1462,55 @@ func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) chan Metric {
 	}(&wg, ch)
 	return ch
 }
+
+// Speedtest run GET/PUT tests at input concurrency for requested object size,
+// optionally you can extend the tests longer with time.Duration.
+func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent int, duration time.Duration) []madmin.SpeedtestResult {
+	results := make([]madmin.SpeedtestResult, len(sys.allPeerClients))
+
+	scheme := "http"
+	if globalIsTLS {
+		scheme = "https"
+	}
+
+	var wg sync.WaitGroup
+	for index := range sys.peerClients {
+		if sys.peerClients[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			r, err := sys.peerClients[index].Speedtest(ctx, size, concurrent, duration)
+			u := &url.URL{
+				Scheme: scheme,
+				Host:   sys.peerClients[index].host.String(),
+			}
+			results[index].Endpoint = u.String()
+			results[index].Err = err
+			if err == nil {
+				results[index].Uploads = r.Uploads
+				results[index].Downloads = r.Downloads
+			}
+		}(index)
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r, err := selfSpeedtest(ctx, size, concurrent, duration)
+		u := &url.URL{
+			Scheme: scheme,
+			Host:   globalLocalNodeName,
+		}
+		results[len(results)-1].Endpoint = u.String()
+		results[len(results)-1].Err = err
+		if err == nil {
+			results[len(results)-1].Uploads = r.Uploads
+			results[len(results)-1].Downloads = r.Downloads
+		}
+	}()
+	wg.Wait()
+
+	return results
+}

cmd/object-api-datatypes.go

@@ -113,16 +113,8 @@ type ObjectInfo struct {
 	// to a delete marker on an object.
 	DeleteMarker bool
 
-	// tierFreeVersion is true if this is a free-version
-	tierFreeVersion bool
-	// TransitionStatus indicates if transition is complete/pending
-	TransitionStatus string
-	// Name of transitioned object on remote tier
-	transitionedObjName string
-	// VersionID on the the remote tier
-	transitionVersionID string
-	// Name of remote tier object has transitioned to
-	TransitionTier string
+	// Transitioned object information
+	TransitionedObject TransitionedObject
 
 	// RestoreExpires indicates date a restored object expires
 	RestoreExpires time.Time
@@ -173,6 +165,8 @@ type ObjectInfo struct {
 
 	Legacy bool // indicates object on disk is in legacy data format
 
+	Multipart bool // indicates if object is multipart object.
+
 	// backendType indicates which backend filled this structure
 	backendType BackendType
 
@@ -193,11 +187,12 @@ func (o ObjectInfo) Clone() (cinfo ObjectInfo) {
 		Size:               o.Size,
 		IsDir:              o.IsDir,
 		ETag:               o.ETag,
+		Multipart:          o.Multipart,
 		InnerETag:          o.InnerETag,
 		VersionID:          o.VersionID,
 		IsLatest:           o.IsLatest,
 		DeleteMarker:       o.DeleteMarker,
-		TransitionStatus:   o.TransitionStatus,
+		TransitionedObject: o.TransitionedObject,
 		RestoreExpires:     o.RestoreExpires,
 		RestoreOngoing:     o.RestoreOngoing,
 		ContentType:        o.ContentType,
@@ -351,6 +346,15 @@ type ListMultipartsInfo struct {
 	EncodingType string // Not supported yet.
 }
 
+// TransitionedObject transitioned object tier and status.
+type TransitionedObject struct {
+	Name        string
+	VersionID   string
+	Tier        string
+	FreeVersion bool
+	Status      string
+}
+
 // DeletedObjectInfo - container for list objects versions deleted objects.
 type DeletedObjectInfo struct {
 	// Name of the bucket.

cmd/object-api-errors.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"path"
 )
 
 // Converts underlying storage error. Convenience function written to
@@ -64,15 +63,6 @@ func toObjectErr(err error, params ...string) error {
 			apiErr.Object = decodeDirObject(params[1])
 		}
 		return apiErr
-	case errFileParentIsFile.Error():
-		apiErr := ParentIsObject{}
-		if len(params) >= 1 {
-			apiErr.Bucket = params[0]
-		}
-		if len(params) >= 2 {
-			apiErr.Object = decodeDirObject(params[1])
-		}
-		return apiErr
 	case errIsNotRegular.Error():
 		apiErr := ObjectExistsAsDirectory{}
 		if len(params) >= 1 {
@@ -322,13 +312,6 @@ func (e PrefixAccessDenied) Error() string {
 	return "Prefix access is denied: " + e.Bucket + SlashSeparator + e.Object
 }
 
-// ParentIsObject object access is denied.
-type ParentIsObject GenericError
-
-func (e ParentIsObject) Error() string {
-	return "Parent is object " + e.Bucket + SlashSeparator + path.Dir(e.Object)
-}
-
 // BucketExists bucket exists.
 type BucketExists GenericError
 

cmd/object-api-interface.go

@@ -53,10 +53,9 @@ type ObjectOptions struct {
 	VersionPurgeStatus            VersionPurgeStatusType // Is only set in DELETE operations for delete marker version to be permanently deleted.
 	Transition                    TransitionOptions
 
-	NoLock         bool                                                  // indicates to lower layers if the caller is expecting to hold locks.
-	ProxyRequest   bool                                                  // only set for GET/HEAD in active-active replication scenario
-	ProxyHeaderSet bool                                                  // only set for GET/HEAD in active-active replication scenario
-	ParentIsObject func(ctx context.Context, bucket, parent string) bool // Used to verify if parent is an object.
+	NoLock         bool // indicates to lower layers if the caller is expecting to hold locks.
+	ProxyRequest   bool // only set for GET/HEAD in active-active replication scenario
+	ProxyHeaderSet bool // only set for GET/HEAD in active-active replication scenario
 
 	DeletePrefix bool //  set true to enforce a prefix deletion, only application for DeleteObject API,
 

cmd/object-api-options.go

@@ -83,7 +83,7 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 
 	var partNumber int
 	var err error
-	if pn := r.URL.Query().Get(xhttp.PartNumber); pn != "" {
+	if pn := r.Form.Get(xhttp.PartNumber); pn != "" {
 		partNumber, err = strconv.Atoi(pn)
 		if err != nil {
 			return opts, err
@@ -93,7 +93,7 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 		}
 	}
 
-	vid := strings.TrimSpace(r.URL.Query().Get(xhttp.VersionID))
+	vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
 		if err != nil {
@@ -219,7 +219,8 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
 // get ObjectOptions for PUT calls from encryption headers and metadata
 func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
 	versioned := globalBucketVersioningSys.Enabled(bucket)
-	vid := strings.TrimSpace(r.URL.Query().Get(xhttp.VersionID))
+	versionSuspended := globalBucketVersioningSys.Suspended(bucket)
+	vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID))
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
 		if err != nil {
@@ -266,6 +267,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 			UserDefined:          metadata,
 			VersionID:            vid,
 			Versioned:            versioned,
+			VersionSuspended:     versionSuspended,
 			MTime:                mtime,
 		}, nil
 	}
@@ -273,6 +275,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 		opts, err = getOpts(ctx, r, bucket, object)
 		opts.VersionID = vid
 		opts.Versioned = versioned
+		opts.VersionSuspended = versionSuspended
 		opts.UserDefined = metadata
 		return
 	}
@@ -290,6 +293,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 			UserDefined:          metadata,
 			VersionID:            vid,
 			Versioned:            versioned,
+			VersionSuspended:     versionSuspended,
 			MTime:                mtime,
 		}, nil
 	}
@@ -300,6 +304,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 	}
 	opts.VersionID = vid
 	opts.Versioned = versioned
+	opts.VersionSuspended = versionSuspended
 	opts.MTime = mtime
 	return opts, nil
 }

cmd/object-api-utils.go

@@ -61,7 +61,9 @@ const (
 	// MinIO tmp meta prefix.
 	minioMetaTmpBucket = minioMetaBucket + "/tmp"
 	// MinIO tmp meta prefix for deleted objects.
-	minioMetaTmpDeletedBucket = minioMetaTmpBucket + "/.trash"
+	minioMetaTmpDeletedBucket      = minioMetaTmpBucket + "/.trash"
+	minioMetaSpeedTestBucket       = minioMetaBucket + "/speedtest"
+	minioMetaSpeedTestBucketPrefix = "objects/"
 
 	// DNS separator (period), used for bucket name validation.
 	dnsDelimiter = "."
@@ -882,6 +884,16 @@ func CleanMinioInternalMetadataKeys(metadata map[string]string) map[string]strin
 	return newMeta
 }
 
+// compressOpts are the options for writing compressed data.
+var compressOpts []s2.WriterOption
+
+func init() {
+	if runtime.GOARCH == "amd64" {
+		// On amd64 we have assembly and can use stronger compression.
+		compressOpts = append(compressOpts, s2.WriterBetterCompression())
+	}
+}
+
 // newS2CompressReader will read data from r, compress it and return the compressed data as a Reader.
 // Use Close to ensure resources are released on incomplete streams.
 //
@@ -892,7 +904,7 @@ func newS2CompressReader(r io.Reader, on int64) io.ReadCloser {
 	pr, pw := io.Pipe()
 	// Copy input to compressor
 	go func() {
-		comp := s2.NewWriter(pw)
+		comp := s2.NewWriter(pw, compressOpts...)
 		cn, err := io.Copy(comp, r)
 		if err != nil {
 			comp.Close()

cmd/object-handlers.go

@@ -491,7 +491,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		setPartsCountHeaders(w, objInfo)
 	}
 
-	setHeadGetRespHeaders(w, r.URL.Query())
+	setHeadGetRespHeaders(w, r.Form)
 
 	statusCodeWritten := false
 	httpWriter := ioutil.WriteOnClose(w)
@@ -739,7 +739,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 	}
 
 	// Set any additional requested response headers.
-	setHeadGetRespHeaders(w, r.URL.Query())
+	setHeadGetRespHeaders(w, r.Form)
 
 	// Successful response.
 	if rs != nil || opts.PartNumber > 0 {
@@ -806,7 +806,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 	// to the destination metadata.
 	sc := r.Header.Get(xhttp.AmzStorageClass)
 	if sc == "" {
-		sc = r.URL.Query().Get(xhttp.AmzStorageClass)
+		sc = r.Form.Get(xhttp.AmzStorageClass)
 	}
 
 	// if x-amz-metadata-directive says REPLACE then
@@ -900,18 +900,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
-		if globalIsGateway {
-			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		} else {
-			if !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		}
+	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
 	}
 
 	vars := mux.Vars(r)
@@ -1381,12 +1372,14 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		objInfo.ModTime = remoteObjInfo.LastModified
 	} else {
 
-		os := newObjSweeper(dstBucket, dstObject)
+		os := newObjSweeper(dstBucket, dstObject).WithVersioning(dstOpts.Versioned, dstOpts.VersionSuspended)
 		// Get appropriate object info to identify the remote object to delete
 		if !srcInfo.metadataOnly {
 			goiOpts := os.GetOpts()
-			if goi, gerr := getObjectInfo(ctx, dstBucket, dstObject, goiOpts); gerr == nil {
-				os.SetTransitionState(goi)
+			if !globalTierConfigMgr.Empty() {
+				if goi, gerr := getObjectInfo(ctx, dstBucket, dstObject, goiOpts); gerr == nil {
+					os.SetTransitionState(goi.TransitionedObject)
+				}
 			}
 		}
 
@@ -1404,8 +1397,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 
 		// Remove the transitioned object whose object version is being overwritten.
-		logger.LogIf(ctx, os.Sweep())
+		if !globalTierConfigMgr.Empty() {
+			logger.LogIf(ctx, os.Sweep())
+		}
 	}
+
 	objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
@@ -1455,18 +1451,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
-		if globalIsGateway {
-			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		} else {
-			if !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		}
+	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
 	}
 
 	vars := mux.Vars(r)
@@ -1702,11 +1689,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	// Ensure that metadata does not contain sensitive information
 	crypto.RemoveSensitiveEntries(metadata)
 
-	oc := newObjSweeper(bucket, object)
-	// Get appropriate object info to identify the remote object to delete
-	goiOpts := oc.GetOpts()
-	if goi, gerr := getObjectInfo(ctx, bucket, object, goiOpts); gerr == nil {
-		oc.SetTransitionState(goi)
+	os := newObjSweeper(bucket, object).WithVersioning(opts.Versioned, opts.VersionSuspended)
+	if !globalTierConfigMgr.Empty() {
+		// Get appropriate object info to identify the remote object to delete
+		goiOpts := os.GetOpts()
+		if goi, gerr := getObjectInfo(ctx, bucket, object, goiOpts); gerr == nil {
+			os.SetTransitionState(goi.TransitionedObject)
+		}
 	}
 
 	// Create the object..
@@ -1753,9 +1742,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync, replication.ObjectReplicationType)
 	}
 
-	// Remove the transitioned object whose object version is being overwritten.
-	logger.LogIf(ctx, oc.Sweep())
-
 	setPutObjHeaders(w, objInfo, false)
 
 	writeSuccessResponseHeadersOnly(w)
@@ -1770,6 +1756,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
+
+	// Remove the transitioned object whose object version is being overwritten.
+	if !globalTierConfigMgr.Empty() {
+		logger.LogIf(ctx, os.Sweep())
+	}
 }
 
 // PutObjectExtractHandler - PUT Object extract is an extended API
@@ -1791,18 +1782,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
-		if globalIsGateway {
-			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		} else {
-			if !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		}
+	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
 	}
 
 	vars := mux.Vars(r)
@@ -2077,18 +2059,9 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
-		if globalIsGateway {
-			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		} else {
-			if !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		}
+	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
 	}
 
 	vars := mux.Vars(r)
@@ -2264,8 +2237,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	uploadID := r.URL.Query().Get(xhttp.UploadID)
-	partIDString := r.URL.Query().Get(xhttp.PartNumber)
+	uploadID := r.Form.Get(xhttp.UploadID)
+	partIDString := r.Form.Get(xhttp.PartNumber)
 
 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
@@ -2528,18 +2501,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok {
-		if globalIsGateway {
-			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		} else {
-			if !objectAPI.IsEncryptionSupported() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-				return
-			}
-		}
+	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 	}
 
 	vars := mux.Vars(r)
@@ -2591,8 +2554,8 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	uploadID := r.URL.Query().Get(xhttp.UploadID)
-	partIDString := r.URL.Query().Get(xhttp.PartNumber)
+	uploadID := r.Form.Get(xhttp.UploadID)
+	partIDString := r.Form.Get(xhttp.PartNumber)
 
 	partID, err := strconv.Atoi(partIDString)
 	if err != nil {
@@ -2812,7 +2775,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 		return
 	}
 
-	uploadID, _, _, _, s3Error := getObjectResources(r.URL.Query())
+	uploadID, _, _, _, s3Error := getObjectResources(r.Form)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -2851,7 +2814,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	uploadID, partNumberMarker, maxParts, encodingType, s3Error := getObjectResources(r.URL.Query())
+	uploadID, partNumberMarker, maxParts, encodingType, s3Error := getObjectResources(r.Form)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -2997,7 +2960,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	// Get upload id.
-	uploadID, _, _, _, s3Error := getObjectResources(r.URL.Query())
+	uploadID, _, _, _, s3Error := getObjectResources(r.Form)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -3106,11 +3069,15 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		w.(http.Flusher).Flush()
 	}
 
-	os := newObjSweeper(bucket, object)
-	// Get appropriate object info to identify the remote object to delete
-	goiOpts := os.GetOpts()
-	if goi, gerr := objectAPI.GetObjectInfo(ctx, bucket, object, goiOpts); gerr == nil {
-		os.SetTransitionState(goi)
+	versioned := globalBucketVersioningSys.Enabled(bucket)
+	suspended := globalBucketVersioningSys.Suspended(bucket)
+	os := newObjSweeper(bucket, object).WithVersioning(versioned, suspended)
+	if !globalTierConfigMgr.Empty() {
+		// Get appropriate object info to identify the remote object to delete
+		goiOpts := os.GetOpts()
+		if goi, gerr := objectAPI.GetObjectInfo(ctx, bucket, object, goiOpts); gerr == nil {
+			os.SetTransitionState(goi.TransitionedObject)
+		}
 	}
 
 	setEventStreamHeaders(w)
@@ -3164,9 +3131,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync, replication.ObjectReplicationType)
 	}
 
-	// Remove the transitioned object whose object version is being overwritten.
-	logger.LogIf(ctx, os.Sweep())
-
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
@@ -3180,6 +3144,11 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
+
+	// Remove the transitioned object whose object version is being overwritten.
+	if !globalTierConfigMgr.Empty() {
+		logger.LogIf(ctx, os.Sweep())
+	}
 }
 
 /// Delete objectAPIHandlers
@@ -3209,11 +3178,6 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
-
 	if globalDNSConfig != nil {
 		_, err := globalDNSConfig.Get(bucket)
 		if err != nil && err != dns.ErrNotImplemented {
@@ -3232,16 +3196,20 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		gerr error
 	)
 
-	var goiOpts ObjectOptions
-	os := newObjSweeper(bucket, object).WithVersion(singleDelete(*r))
+	getObjectInfo := objectAPI.GetObjectInfo
+	if api.CacheAPI() != nil {
+		getObjectInfo = api.CacheAPI().GetObjectInfo
+	}
+
+	os := newObjSweeper(bucket, object).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
 	// Mutations of objects on versioning suspended buckets
 	// affect its null version. Through opts below we select
 	// the null version's remote object to delete if
 	// transitioned.
-	goiOpts = os.GetOpts()
+	goiOpts := os.GetOpts()
 	goi, gerr = getObjectInfo(ctx, bucket, object, goiOpts)
 	if gerr == nil {
-		os.SetTransitionState(goi)
+		os.SetTransitionState(goi.TransitionedObject)
 	}
 
 	replicateDel, replicateSync := checkReplicateDelete(ctx, bucket, ObjectToDelete{ObjectName: object, VersionID: opts.VersionID}, goi, gerr)
@@ -3354,8 +3322,9 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Remove the transitioned object whose object version is being overwritten.
-	logger.LogIf(ctx, os.Sweep())
-
+	if !globalTierConfigMgr.Empty() {
+		os.Sweep()
+	}
 }
 
 // PutObjectLegalHoldHandler - set legal hold configuration to object,
@@ -3548,7 +3517,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	cred, owner, claims, s3Err := validateSignature(getRequestAuthType(r), r)
+	cred, owner, s3Err := validateSignature(getRequestAuthType(r), r)
 	if s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -3588,7 +3557,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
 
-	objInfo, s3Err := enforceRetentionBypassForPut(ctx, r, bucket, object, getObjectInfo, objRetention, cred, owner, claims)
+	objInfo, s3Err := enforceRetentionBypassForPut(ctx, r, bucket, object, getObjectInfo, objRetention, cred, owner)
 	if s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -3952,13 +3921,14 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	if objInfo.TransitionStatus != lifecycle.TransitionComplete {
+	if objInfo.TransitionedObject.Status != lifecycle.TransitionComplete {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL)
 		return
 	}

cmd/object-handlers_test.go

@@ -26,6 +26,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"path"
 	"runtime"
 	"strings"
 
@@ -557,12 +558,8 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}
 
-		if actualError.BucketName != testCase.bucketName {
-			t.Fatalf("Test %d: %s: Unexpected bucket name, expected %s, got %s", i+1, instanceType, testCase.bucketName, actualError.BucketName)
-		}
-
-		if actualError.Key != testCase.objectName {
-			t.Fatalf("Test %d: %s: Unexpected object name, expected %s, got %s", i+1, instanceType, testCase.objectName, actualError.Key)
+		if path.Clean(actualError.Resource) != pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName) {
+			t.Fatalf("Test %d: %s: Unexpected resource, expected %s, got %s", i+1, instanceType, pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName), actualError.Resource)
 		}
 
 		// Verify response of the V2 signed HTTP request.
@@ -606,12 +603,8 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}
 
-		if actualError.BucketName != testCase.bucketName {
-			t.Fatalf("Test %d: %s: Unexpected bucket name, expected %s, got %s", i+1, instanceType, testCase.bucketName, actualError.BucketName)
-		}
-
-		if actualError.Key != testCase.objectName {
-			t.Fatalf("Test %d: %s: Unexpected object name, expected %s, got %s", i+1, instanceType, testCase.objectName, actualError.Key)
+		if path.Clean(actualError.Resource) != pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName) {
+			t.Fatalf("Test %d: %s: Unexpected resource, expected %s, got %s", i+1, instanceType, pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName), actualError.Resource)
 		}
 	}
 
@@ -918,7 +911,6 @@ func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucket
 
 	mkGetReqWithPartNumber := func(oindex int, oi ObjectInput, partNumber int) {
 		object := oi.objectName
-		rec := httptest.NewRecorder()
 
 		queries := url.Values{}
 		queries.Add("partNumber", strconv.Itoa(partNumber))
@@ -930,6 +922,7 @@ func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucket
 				object, oindex, partNumber, err)
 		}
 
+		rec := httptest.NewRecorder()
 		apiRouter.ServeHTTP(rec, req)
 
 		// Check response code (we make only valid requests in this test)

cmd/peer-rest-client.go

@@ -400,6 +400,28 @@ func (client *peerRESTClient) GetOSInfo(ctx context.Context) (info madmin.OSInfo
 	return info, err
 }
 
+// GetSELinuxInfo - fetch SELinux information for a remote node.
+func (client *peerRESTClient) GetSELinuxInfo(ctx context.Context) (info madmin.SysServices, err error) {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodSysServices, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
+// GetSysErrors - fetch memory information for a remote node.
+func (client *peerRESTClient) GetSysErrors(ctx context.Context) (info madmin.SysErrors, err error) {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodSysErrors, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
 // GetMemInfo - fetch memory information for a remote node.
 func (client *peerRESTClient) GetMemInfo(ctx context.Context) (info madmin.MemInfo, err error) {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodMemInfo, nil, nil, -1)
@@ -969,3 +991,21 @@ func (client *peerRESTClient) GetPeerMetrics(ctx context.Context) (<-chan Metric
 	}(ch)
 	return ch, nil
 }
+
+func (client *peerRESTClient) Speedtest(ctx context.Context, size, concurrent int, duration time.Duration) (SpeedtestResult, error) {
+	values := make(url.Values)
+	values.Set(peerRESTSize, strconv.Itoa(size))
+	values.Set(peerRESTConcurrent, strconv.Itoa(concurrent))
+	values.Set(peerRESTDuration, duration.String())
+
+	respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedtest, values, nil, -1)
+	if err != nil {
+		return SpeedtestResult{}, err
+	}
+	defer http.DrainBody(respBody)
+
+	dec := gob.NewDecoder(respBody)
+	var result SpeedtestResult
+	err = dec.Decode(&result)
+	return result, err
+}

cmd/peer-rest-common.go

@@ -34,6 +34,8 @@ const (
 	peerRESTMethodOsInfo                   = "/osinfo"
 	peerRESTMethodMemInfo                  = "/meminfo"
 	peerRESTMethodProcInfo                 = "/procinfo"
+	peerRESTMethodSysErrors                = "/syserrors"
+	peerRESTMethodSysServices              = "/sysservices"
 	peerRESTMethodDispatchNetInfo          = "/dispatchnetinfo"
 	peerRESTMethodDeleteBucketMetadata     = "/deletebucketmetadata"
 	peerRESTMethodLoadBucketMetadata       = "/loadbucketmetadata"
@@ -62,6 +64,7 @@ const (
 	peerRESTMethodUpdateMetacacheListing   = "/updatemetacache"
 	peerRESTMethodGetPeerMetrics           = "/peermetrics"
 	peerRESTMethodLoadTransitionTierConfig = "/loadtransitiontierconfig"
+	peerRESTMethodSpeedtest                = "/speedtest"
 )
 
 const (
@@ -81,6 +84,9 @@ const (
 	peerRESTTraceS3        = "s3"
 	peerRESTTraceOS        = "os"
 	peerRESTTraceThreshold = "threshold"
+	peerRESTSize           = "size"
+	peerRESTConcurrent     = "concurrent"
+	peerRESTDuration       = "duration"
 
 	peerRESTListenBucket = "bucket"
 	peerRESTListenPrefix = "prefix"

cmd/peer-rest-server.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"crypto/rand"
 	"encoding/gob"
 	"errors"
 	"fmt"
@@ -27,12 +28,17 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
+	humanize "github.com/dustin/go-humanize"
+	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 )
@@ -130,7 +136,7 @@ func (s *peerRESTServer) LoadPolicyMappingHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	_, isGroup := r.URL.Query()[peerRESTIsGroup]
+	_, isGroup := r.Form[peerRESTIsGroup]
 	if err := globalIAMSys.LoadPolicyMapping(objAPI, userOrGroup, isGroup); err != nil {
 		s.writeErrorResponse(w, err)
 		return
@@ -504,6 +510,40 @@ func (s *peerRESTServer) GetMemInfoHandler(w http.ResponseWriter, r *http.Reques
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
+// GetSysServicesHandler - returns system services information.
+// (only the services that are of concern to minio)
+func (s *peerRESTServer) GetSysServicesHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		fmt.Println("Invalid request")
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+
+	info := madmin.GetSysServices(ctx, r.Host)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
+// GetSysErrorsHandler - returns system level errors
+func (s *peerRESTServer) GetSysErrorsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
+
+	info := madmin.GetSysErrors(ctx, r.Host)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
 // DeleteBucketMetadataHandler - Delete in memory bucket metadata
 func (s *peerRESTServer) DeleteBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -819,7 +859,7 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	values := r.URL.Query()
+	values := r.Form
 
 	var prefix string
 	if len(values[peerRESTListenPrefix]) > 1 {
@@ -910,15 +950,13 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func extractTraceOptsFromPeerRequest(r *http.Request) (opts madmin.ServiceTraceOpts, err error) {
+	opts.S3 = r.Form.Get(peerRESTTraceS3) == "true"
+	opts.OS = r.Form.Get(peerRESTTraceOS) == "true"
+	opts.Storage = r.Form.Get(peerRESTTraceStorage) == "true"
+	opts.Internal = r.Form.Get(peerRESTTraceInternal) == "true"
+	opts.OnlyErrors = r.Form.Get(peerRESTTraceErr) == "true"
 
-	q := r.URL.Query()
-	opts.OnlyErrors = q.Get(peerRESTTraceErr) == "true"
-	opts.Storage = q.Get(peerRESTTraceStorage) == "true"
-	opts.Internal = q.Get(peerRESTTraceInternal) == "true"
-	opts.S3 = q.Get(peerRESTTraceS3) == "true"
-	opts.OS = q.Get(peerRESTTraceOS) == "true"
-
-	if t := q.Get(peerRESTTraceThreshold); t != "" {
+	if t := r.Form.Get(peerRESTTraceThreshold); t != "" {
 		d, err := time.ParseDuration(t)
 		if err != nil {
 			return opts, err
@@ -1056,7 +1094,8 @@ func (s *peerRESTServer) GetBandwidth(w http.ResponseWriter, r *http.Request) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
 		return
 	}
-	bucketsString := r.URL.Query().Get("buckets")
+
+	bucketsString := r.Form.Get("buckets")
 	w.WriteHeader(http.StatusOK)
 	w.(http.Flusher).Flush()
 
@@ -1097,6 +1136,179 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 	w.(http.Flusher).Flush()
 }
 
+// SpeedtestResult return value of the speedtest function
+type SpeedtestResult struct {
+	Uploads   uint64
+	Downloads uint64
+}
+
+// SpeedtestObject implements "random-read" object reader
+type SpeedtestObject struct {
+	buf       []byte
+	remaining int
+}
+
+func (bo *SpeedtestObject) Read(b []byte) (int, error) {
+	if bo.remaining == 0 {
+		return 0, io.EOF
+	}
+	if len(b) == 0 {
+		return 0, nil
+	}
+	if len(b) > len(bo.buf) {
+		b = b[:len(bo.buf)]
+	}
+	if len(b) > bo.remaining {
+		b = b[:bo.remaining]
+	}
+	copy(b, bo.buf)
+	bo.remaining -= len(b)
+	return len(b), nil
+}
+
+// Runs the speedtest on local MinIO process.
+func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration) (SpeedtestResult, error) {
+	var result SpeedtestResult
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return result, errServerNotInitialized
+	}
+
+	bucket := minioMetaSpeedTestBucket
+
+	buf := make([]byte, humanize.MiByte)
+	rand.Read(buf)
+
+	objCountPerThread := make([]uint64, concurrent)
+
+	var objUploadCount uint64
+	var objDownloadCount uint64
+
+	var wg sync.WaitGroup
+
+	doneCh1 := make(chan struct{})
+	go func() {
+		time.Sleep(duration)
+		close(doneCh1)
+	}()
+
+	objNamePrefix := minioMetaSpeedTestBucketPrefix + uuid.New().String()
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			for {
+				hashReader, err := hash.NewReader(&SpeedtestObject{buf, size},
+					int64(size), "", "", int64(size))
+				if err != nil {
+					logger.LogIf(ctx, err)
+					break
+				}
+				reader := NewPutObjReader(hashReader)
+				_, err = objAPI.PutObject(ctx, bucket, fmt.Sprintf("%s.%d.%d",
+					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{})
+				if err != nil {
+					logger.LogIf(ctx, err)
+					break
+				}
+				objCountPerThread[i]++
+				atomic.AddUint64(&objUploadCount, 1)
+				select {
+				case <-doneCh1:
+					return
+				default:
+				}
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	doneCh2 := make(chan struct{})
+	go func() {
+		time.Sleep(duration)
+		close(doneCh2)
+	}()
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			var j uint64
+			for {
+				if objCountPerThread[i] == j {
+					j = 0
+				}
+				r, err := objAPI.GetObjectNInfo(ctx, bucket, fmt.Sprintf("%s.%d.%d",
+					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
+				if err != nil {
+					logger.LogIf(ctx, err)
+					break
+				}
+				_, err = io.Copy(ioutil.Discard, r)
+				r.Close()
+				if err != nil {
+					logger.LogIf(ctx, err)
+					break
+				}
+				j++
+				atomic.AddUint64(&objDownloadCount, 1)
+				select {
+				case <-doneCh2:
+					return
+				default:
+				}
+			}
+		}(i)
+	}
+	wg.Wait()
+	return SpeedtestResult{objUploadCount, objDownloadCount}, nil
+}
+
+func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+	}
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	sizeStr := r.Form.Get(peerRESTSize)
+	durationStr := r.Form.Get(peerRESTDuration)
+	concurrentStr := r.Form.Get(peerRESTConcurrent)
+
+	size, err := strconv.Atoi(sizeStr)
+	if err != nil {
+		size = 64 * humanize.MiByte
+	}
+
+	concurrent, err := strconv.Atoi(concurrentStr)
+	if err != nil {
+		concurrent = 32
+	}
+
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		duration = time.Second * 10
+	}
+
+	result, err := selfSpeedtest(r.Context(), size, concurrent, duration)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	enc := gob.NewEncoder(w)
+	if err := enc.Encode(result); err != nil {
+		s.writeErrorResponse(w, errors.New("Encoding report failed: "+err.Error()))
+		return
+	}
+	w.(http.Flusher).Flush()
+}
+
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
 	server := &peerRESTServer{}
@@ -1106,6 +1318,8 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(httpTraceHdrs(server.GetProcInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(httpTraceHdrs(server.GetMemInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(httpTraceHdrs(server.GetSysErrorsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(httpTraceHdrs(server.GetSysServicesHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))
@@ -1139,4 +1353,5 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodUpdateMetacacheListing).HandlerFunc(httpTraceHdrs(server.UpdateMetacacheListingHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(httpTraceHdrs(server.GetPeerMetrics))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
 }

cmd/s3-zip-handlers.go

@@ -196,7 +196,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 		return
 	}
 
-	setHeadGetRespHeaders(w, r.URL.Query())
+	setHeadGetRespHeaders(w, r.Form)
 
 	httpWriter := ioutil.WriteOnClose(w)
 
@@ -467,7 +467,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
 	}
 
 	// Set any additional requested response headers.
-	setHeadGetRespHeaders(w, r.URL.Query())
+	setHeadGetRespHeaders(w, r.Form)
 
 	// Successful response.
 	if rs != nil {

cmd/server-main.go

@@ -530,7 +530,6 @@ func serverMain(ctx *cli.Context) {
 	if globalIsErasure {
 		initAutoHeal(GlobalContext, newObject)
 		initHealMRF(GlobalContext, newObject)
-		initBackgroundTransition(GlobalContext, newObject)
 	}
 
 	initBackgroundExpiry(GlobalContext, newObject)
@@ -558,6 +557,7 @@ func serverMain(ctx *cli.Context) {
 
 	if globalIsErasure { // to be done after config init
 		initBackgroundReplication(GlobalContext, newObject)
+		initBackgroundTransition(GlobalContext, newObject)
 		globalTierJournal, err = initTierDeletionJournal(GlobalContext)
 		if err != nil {
 			logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")

cmd/server-startup-msg.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"fmt"
 	"net"
+	"net/url"
 	"runtime"
 	"strings"
 
@@ -79,34 +80,41 @@ func printStartupMessage(apiEndpoints []string, err error) {
 	}
 }
 
-// Returns true if input is not IPv4, false if it is.
-func isNotIPv4(host string) bool {
+// Returns true if input is IPv6
+func isIPv6(host string) bool {
 	h, _, err := net.SplitHostPort(host)
 	if err != nil {
 		h = host
 	}
 	ip := net.ParseIP(h)
-	ok := ip.To4() != nil // This is always true of IP is IPv4
-
-	// Returns true if input is not IPv4.
-	return !ok
+	return ip.To16() != nil && ip.To4() == nil
 }
 
 // strip api endpoints list with standard ports such as
 // port "80" and "443" before displaying on the startup
 // banner.  Returns a new list of API endpoints.
 func stripStandardPorts(apiEndpoints []string, host string) (newAPIEndpoints []string) {
+	if len(apiEndpoints) == 1 {
+		return apiEndpoints
+	}
 	newAPIEndpoints = make([]string, len(apiEndpoints))
 	// Check all API endpoints for standard ports and strip them.
 	for i, apiEndpoint := range apiEndpoints {
-		u, err := xnet.ParseHTTPURL(apiEndpoint)
+		_, err := xnet.ParseHTTPURL(apiEndpoint)
 		if err != nil {
 			continue
 		}
-		if host == "" && isNotIPv4(u.Host) {
-			// Skip all non-IPv4 endpoints when we bind to all interfaces.
+		u, err := url.Parse(apiEndpoint)
+		if err != nil {
 			continue
 		}
+		if host == "" && isIPv6(u.Hostname()) {
+			// Skip all IPv6 endpoints
+			continue
+		}
+		if u.Port() == "80" && u.Scheme == "http" || u.Port() == "443" && u.Scheme == "https" {
+			u.Host = u.Hostname()
+		}
 		newAPIEndpoints[i] = u.String()
 	}
 	return newAPIEndpoints

cmd/server-startup-msg_test.go

@@ -58,7 +58,7 @@ func TestStripStandardPorts(t *testing.T) {
 
 	apiEndpoints = []string{"http://%%%%%:9000"}
 	newAPIEndpoints = stripStandardPorts(apiEndpoints, "")
-	if !reflect.DeepEqual([]string{""}, newAPIEndpoints) {
+	if !reflect.DeepEqual(apiEndpoints, newAPIEndpoints) {
 		t.Fatalf("Expected %#v, got %#v", apiEndpoints, newAPIEndpoints)
 	}
 

cmd/signature-v2.go

@@ -78,7 +78,9 @@ const (
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign
 func doesPolicySignatureV2Match(formValues http.Header) (auth.Credentials, APIErrorCode) {
 	accessKey := formValues.Get(xhttp.AmzAccessKeyID)
-	cred, _, s3Err := checkKeyValid(accessKey)
+
+	r := &http.Request{Header: formValues}
+	cred, _, s3Err := checkKeyValid(r, accessKey)
 	if s3Err != ErrNone {
 		return cred, s3Err
 	}
@@ -153,7 +155,7 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 		return ErrInvalidQueryParams
 	}
 
-	cred, _, s3Err := checkKeyValid(accessKey)
+	cred, _, s3Err := checkKeyValid(r, accessKey)
 	if s3Err != ErrNone {
 		return s3Err
 	}
@@ -183,8 +185,8 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 }
 
 func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) {
-	if accessKey := r.URL.Query().Get(xhttp.AmzAccessKeyID); accessKey != "" {
-		return checkKeyValid(accessKey)
+	if accessKey := r.Form.Get(xhttp.AmzAccessKeyID); accessKey != "" {
+		return checkKeyValid(r, accessKey)
 	}
 
 	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
@@ -200,7 +202,7 @@ func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) {
 		return auth.Credentials{}, false, ErrMissingFields
 	}
 
-	return checkKeyValid(keySignFields[0])
+	return checkKeyValid(r, keySignFields[0])
 }
 
 // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;

cmd/signature-v4-parser.go

@@ -50,7 +50,7 @@ func (c credentialHeader) getScope() string {
 }
 
 func getReqAccessKeyV4(r *http.Request, region string, stype serviceType) (auth.Credentials, bool, APIErrorCode) {
-	ch, s3Err := parseCredentialHeader("Credential="+r.URL.Query().Get(xhttp.AmzCredential), region, stype)
+	ch, s3Err := parseCredentialHeader("Credential="+r.Form.Get(xhttp.AmzCredential), region, stype)
 	if s3Err != ErrNone {
 		// Strip off the Algorithm prefix.
 		v4Auth := strings.TrimPrefix(r.Header.Get("Authorization"), signV4Algorithm)
@@ -63,7 +63,7 @@ func getReqAccessKeyV4(r *http.Request, region string, stype serviceType) (auth.
 			return auth.Credentials{}, false, s3Err
 		}
 	}
-	return checkKeyValid(ch.accessKey)
+	return checkKeyValid(r, ch.accessKey)
 }
 
 // parse credentialHeader string into its structured form.

cmd/signature-v4-utils.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/minio/internal/auth"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the
@@ -46,7 +47,7 @@ func skipContentSha256Cksum(r *http.Request) bool {
 	)
 
 	if isRequestPresignedSignatureV4(r) {
-		v, ok = r.URL.Query()[xhttp.AmzContentSha256]
+		v, ok = r.Form[xhttp.AmzContentSha256]
 		if !ok {
 			v, ok = r.Header[xhttp.AmzContentSha256]
 		}
@@ -82,7 +83,7 @@ func getContentSha256Cksum(r *http.Request, stype serviceType) string {
 		// X-Amz-Content-Sha256, if not set in presigned requests, checksum
 		// will default to 'UNSIGNED-PAYLOAD'.
 		defaultSha256Cksum = unsignedPayload
-		v, ok = r.URL.Query()[xhttp.AmzContentSha256]
+		v, ok = r.Form[xhttp.AmzContentSha256]
 		if !ok {
 			v, ok = r.Header[xhttp.AmzContentSha256]
 		}
@@ -120,7 +121,7 @@ func isValidRegion(reqRegion string, confRegion string) bool {
 
 // check if the access key is valid and recognized, additionally
 // also returns if the access key is owner/admin.
-func checkKeyValid(accessKey string) (auth.Credentials, bool, APIErrorCode) {
+func checkKeyValid(r *http.Request, accessKey string) (auth.Credentials, bool, APIErrorCode) {
 	if !globalIAMSys.Initialized() && !globalIsGateway {
 		// Check if server has initialized, then only proceed
 		// to check for IAM users otherwise its okay for clients
@@ -135,7 +136,17 @@ func checkKeyValid(accessKey string) (auth.Credentials, bool, APIErrorCode) {
 		if !ok {
 			return cred, false, ErrInvalidAccessKeyID
 		}
-		owner = cred.AccessKey == ucred.ParentUser
+		claims, s3Err := checkClaimsFromToken(r, ucred)
+		if s3Err != ErrNone {
+			return cred, false, s3Err
+		}
+		ucred.Claims = claims
+		// Now check if we have a sessionPolicy.
+		if _, ok = claims[iampolicy.SessionPolicyName]; ok {
+			owner = false
+		} else {
+			owner = cred.AccessKey == ucred.ParentUser
+		}
 		cred = ucred
 	}
 	return cred, owner, ErrNone
@@ -151,7 +162,7 @@ func sumHMAC(key []byte, data []byte) []byte {
 // extractSignedHeaders extract signed headers from Authorization header
 func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header, APIErrorCode) {
 	reqHeaders := r.Header
-	reqQueries := r.URL.Query()
+	reqQueries := r.Form
 	// find whether "host" is part of list of signed headers.
 	// if not return ErrUnsignedHeaders. "host" is mandatory.
 	if !contains(signedHeaders, "host") {

cmd/signature-v4-utils_test.go

@@ -89,6 +89,7 @@ func TestSkipContentSha256Cksum(t *testing.T) {
 				inputReq.Header.Set(testCase.inputHeaderKey, testCase.inputHeaderValue)
 			}
 		}
+		inputReq.ParseForm()
 
 		actualResult := skipContentSha256Cksum(inputReq)
 		if testCase.expectedResult != actualResult {
@@ -163,6 +164,7 @@ func TestExtractSignedHeaders(t *testing.T) {
 	// set headers value through Get parameter
 	inputQuery.Add("x-amz-server-side-encryption", xhttp.AmzEncryptionAES)
 	r.URL.RawQuery = inputQuery.Encode()
+	r.ParseForm()
 	_, errCode = extractSignedHeaders(signedHeaders, r)
 	if errCode != ErrNone {
 		t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode)
@@ -267,6 +269,7 @@ func TestGetContentSha256Cksum(t *testing.T) {
 		if testCase.h != "" {
 			r.Header.Set("x-amz-content-sha256", testCase.h)
 		}
+		r.ParseForm()
 		got := getContentSha256Cksum(r, serviceS3)
 		if got != testCase.expected {
 			t.Errorf("Test %d: got:%s expected:%s", i+1, got, testCase.expected)

cmd/signature-v4.go

@@ -181,7 +181,8 @@ func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIEr
 		return auth.Credentials{}, s3Err
 	}
 
-	cred, _, s3Err := checkKeyValid(credHeader.accessKey)
+	r := &http.Request{Header: formValues}
+	cred, _, s3Err := checkKeyValid(r, credHeader.accessKey)
 	if s3Err != ErrNone {
 		return cred, s3Err
 	}
@@ -209,12 +210,12 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	req := *r
 
 	// Parse request query string.
-	pSignValues, err := parsePreSignV4(req.URL.Query(), region, stype)
+	pSignValues, err := parsePreSignV4(req.Form, region, stype)
 	if err != ErrNone {
 		return err
 	}
 
-	cred, _, s3Err := checkKeyValid(pSignValues.Credential.accessKey)
+	cred, _, s3Err := checkKeyValid(r, pSignValues.Credential.accessKey)
 	if s3Err != ErrNone {
 		return s3Err
 	}
@@ -241,12 +242,12 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 	// Construct new query.
 	query := make(url.Values)
-	clntHashedPayload := req.URL.Query().Get(xhttp.AmzContentSha256)
+	clntHashedPayload := req.Form.Get(xhttp.AmzContentSha256)
 	if clntHashedPayload != "" {
 		query.Set(xhttp.AmzContentSha256, hashedPayload)
 	}
 
-	token := req.URL.Query().Get(xhttp.AmzSecurityToken)
+	token := req.Form.Get(xhttp.AmzSecurityToken)
 	if token != "" {
 		query.Set(xhttp.AmzSecurityToken, cred.SessionToken)
 	}
@@ -271,7 +272,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	)
 
 	// Add missing query parameters if any provided in the request URL
-	for k, v := range req.URL.Query() {
+	for k, v := range req.Form {
 		if !defaultSigParams.Contains(k) {
 			query[k] = v
 		}
@@ -281,19 +282,19 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	encodedQuery := query.Encode()
 
 	// Verify if date query is same.
-	if req.URL.Query().Get(xhttp.AmzDate) != query.Get(xhttp.AmzDate) {
+	if req.Form.Get(xhttp.AmzDate) != query.Get(xhttp.AmzDate) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if expires query is same.
-	if req.URL.Query().Get(xhttp.AmzExpires) != query.Get(xhttp.AmzExpires) {
+	if req.Form.Get(xhttp.AmzExpires) != query.Get(xhttp.AmzExpires) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if signed headers query is same.
-	if req.URL.Query().Get(xhttp.AmzSignedHeaders) != query.Get(xhttp.AmzSignedHeaders) {
+	if req.Form.Get(xhttp.AmzSignedHeaders) != query.Get(xhttp.AmzSignedHeaders) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if credential query is same.
-	if req.URL.Query().Get(xhttp.AmzCredential) != query.Get(xhttp.AmzCredential) {
+	if req.Form.Get(xhttp.AmzCredential) != query.Get(xhttp.AmzCredential) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if sha256 payload query is same.
@@ -321,7 +322,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	newSignature := getSignature(presignedSigningKey, presignedStringToSign)
 
 	// Verify signature.
-	if !compareSignatureV4(req.URL.Query().Get(xhttp.AmzSignature), newSignature) {
+	if !compareSignatureV4(req.Form.Get(xhttp.AmzSignature), newSignature) {
 		return ErrSignatureDoesNotMatch
 	}
 	return ErrNone
@@ -349,7 +350,7 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
 		return errCode
 	}
 
-	cred, _, s3Err := checkKeyValid(signV4Values.Credential.accessKey)
+	cred, _, s3Err := checkKeyValid(r, signV4Values.Credential.accessKey)
 	if s3Err != ErrNone {
 		return s3Err
 	}
@@ -369,7 +370,7 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
 	}
 
 	// Query string.
-	queryStr := req.URL.Query().Encode()
+	queryStr := req.Form.Encode()
 
 	// Get canonical request.
 	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method)

cmd/signature-v4_test.go

@@ -293,6 +293,9 @@ func TestDoesPresignedSignatureMatch(t *testing.T) {
 			req.Header.Set(key, value)
 		}
 
+		// parse form.
+		req.ParseForm()
+
 		// Check if it matches!
 		err := doesPresignedSignatureMatch(payloadSHA256, req, testCase.region, serviceS3)
 		if err != testCase.expected {

cmd/storage-datatypes.go

@@ -179,6 +179,8 @@ type FileInfo struct {
 
 	NumVersions      int
 	SuccessorModTime time.Time
+
+	Fresh bool // indicates this is a first time call to write FileInfo.
 }
 
 // InlineData returns true if object contents are inlined alongside its metadata.
@@ -188,7 +190,10 @@ func (fi FileInfo) InlineData() bool {
 }
 
 // SetInlineData marks object (version) as inline.
-func (fi FileInfo) SetInlineData() {
+func (fi *FileInfo) SetInlineData() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string, 1)
+	}
 	fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] = "true"
 }
 

cmd/storage-datatypes_gen.go

@@ -550,8 +550,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 24 {
-		err = msgp.ArrayError{Wanted: 24, Got: zb0001}
+	if zb0001 != 25 {
+		err = msgp.ArrayError{Wanted: 25, Got: zb0001}
 		return
 	}
 	z.Volume, err = dc.ReadString()
@@ -715,13 +715,18 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "SuccessorModTime")
 		return
 	}
+	z.Fresh, err = dc.ReadBool()
+	if err != nil {
+		err = msgp.WrapError(err, "Fresh")
+		return
+	}
 	return
 }
 
 // EncodeMsg implements msgp.Encodable
 func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 24
-	err = en.Append(0xdc, 0x0, 0x18)
+	// array header, size 25
+	err = en.Append(0xdc, 0x0, 0x19)
 	if err != nil {
 		return
 	}
@@ -864,14 +869,19 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "SuccessorModTime")
 		return
 	}
+	err = en.WriteBool(z.Fresh)
+	if err != nil {
+		err = msgp.WrapError(err, "Fresh")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 24
-	o = append(o, 0xdc, 0x0, 0x18)
+	// array header, size 25
+	o = append(o, 0xdc, 0x0, 0x19)
 	o = msgp.AppendString(o, z.Volume)
 	o = msgp.AppendString(o, z.Name)
 	o = msgp.AppendString(o, z.VersionID)
@@ -911,6 +921,7 @@ func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendBytes(o, z.Data)
 	o = msgp.AppendInt(o, z.NumVersions)
 	o = msgp.AppendTime(o, z.SuccessorModTime)
+	o = msgp.AppendBool(o, z.Fresh)
 	return
 }
 
@@ -922,8 +933,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 24 {
-		err = msgp.ArrayError{Wanted: 24, Got: zb0001}
+	if zb0001 != 25 {
+		err = msgp.ArrayError{Wanted: 25, Got: zb0001}
 		return
 	}
 	z.Volume, bts, err = msgp.ReadStringBytes(bts)
@@ -1087,6 +1098,11 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "SuccessorModTime")
 		return
 	}
+	z.Fresh, bts, err = msgp.ReadBoolBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Fresh")
+		return
+	}
 	o = bts
 	return
 }
@@ -1104,7 +1120,7 @@ func (z *FileInfo) Msgsize() (s int) {
 	for za0003 := range z.Parts {
 		s += z.Parts[za0003].Msgsize()
 	}
-	s += z.Erasure.Msgsize() + msgp.BoolSize + msgp.StringPrefixSize + len(z.DeleteMarkerReplicationStatus) + msgp.StringPrefixSize + len(string(z.VersionPurgeStatus)) + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize
+	s += z.Erasure.Msgsize() + msgp.BoolSize + msgp.StringPrefixSize + len(z.DeleteMarkerReplicationStatus) + msgp.StringPrefixSize + len(string(z.VersionPurgeStatus)) + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize
 	return
 }
 

cmd/storage-errors.go

@@ -88,9 +88,6 @@ var errFileAccessDenied = StorageErr("file access denied")
 // errFileCorrupt - file has an unexpected size, or is not readable
 var errFileCorrupt = StorageErr("file is corrupted")
 
-// errFileParentIsFile - cannot have overlapping objects, parent is already a file.
-var errFileParentIsFile = StorageErr("parent is a file")
-
 // errBitrotHashAlgoInvalid - the algo for bit-rot hash
 // verification is empty or invalid.
 var errBitrotHashAlgoInvalid = StorageErr("bit-rot hash algorithm is invalid")

cmd/storage-interface.go

@@ -71,7 +71,6 @@ type StorageAPI interface {
 	ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error)
 	RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error
 	CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
-	CheckFile(ctx context.Context, volume string, path string) (err error)
 	Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
 	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
 	StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error)

cmd/storage-rest-client.go

@@ -250,7 +250,7 @@ func (client *storageRESTClient) NSScanner(ctx context.Context, cache dataUsageC
 	if err == io.EOF {
 		err = nil
 	}
-	return cache, err
+	return newCache, err
 }
 
 func (client *storageRESTClient) GetDiskID() (string, error) {
@@ -434,16 +434,6 @@ func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, pa
 	return err
 }
 
-// CheckFile - stat a file metadata.
-func (client *storageRESTClient) CheckFile(ctx context.Context, volume string, path string) error {
-	values := make(url.Values)
-	values.Set(storageRESTVolume, volume)
-	values.Set(storageRESTFilePath, path)
-	respBody, err := client.call(ctx, storageRESTMethodCheckFile, values, nil, -1)
-	defer xhttp.DrainBody(respBody)
-	return err
-}
-
 // CheckParts - stat all file parts.
 func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error {
 	values := make(url.Values)

cmd/storage-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v37" // cleanup behavior change at storage layer.
+	storageRESTVersion       = "v39" // Add FileInfo.Fresh field
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )
@@ -42,7 +42,6 @@ const (
 	storageRESTMethodReadVersion    = "/readversion"
 	storageRESTMethodRenameData     = "/renamedata"
 	storageRESTMethodCheckParts     = "/checkparts"
-	storageRESTMethodCheckFile      = "/checkfile"
 	storageRESTMethodReadAll        = "/readall"
 	storageRESTMethodReadFile       = "/readfile"
 	storageRESTMethodReadFileStream = "/readfilestream"

cmd/storage-rest-server.go

@@ -119,7 +119,7 @@ func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool
 		return false
 	}
 
-	diskID := r.URL.Query().Get(storageRESTDiskID)
+	diskID := r.Form.Get(storageRESTDiskID)
 	if diskID == "" {
 		// Request sent empty disk-id, we allow the request
 		// as the peer might be coming up and trying to read format.json
@@ -282,7 +282,7 @@ func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Requ
 	}
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
-	forceDelete := r.URL.Query().Get(storageRESTForceDelete) == "true"
+	forceDelete := r.Form.Get(storageRESTForceDelete) == "true"
 	err := s.storage.DeleteVol(r.Context(), volume, forceDelete)
 	if err != nil {
 		s.writeErrorResponse(w, err)
@@ -487,20 +487,6 @@ func (s *storageRESTServer) CheckPartsHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
-// CheckFileHandler - check if a file metadata exists.
-func (s *storageRESTServer) CheckFileHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
-	}
-	vars := mux.Vars(r)
-	volume := vars[storageRESTVolume]
-	filePath := vars[storageRESTFilePath]
-
-	if err := s.storage.CheckFile(r.Context(), volume, filePath); err != nil {
-		s.writeErrorResponse(w, err)
-	}
-}
-
 // ReadAllHandler - read all the contents of a file.
 func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -655,10 +641,8 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	vars := r.URL.Query()
-	volume := vars.Get(storageRESTVolume)
-
-	totalVersions, err := strconv.Atoi(vars.Get(storageRESTTotalVersions))
+	volume := r.Form.Get(storageRESTVolume)
+	totalVersions, err := strconv.Atoi(r.Form.Get(storageRESTTotalVersions))
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
@@ -1121,8 +1105,6 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCreateFile).HandlerFunc(httpTraceHdrs(server.CreateFileHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTLength)...)
 
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCheckFile).HandlerFunc(httpTraceHdrs(server.CheckFileHandler)).
-				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCheckParts).HandlerFunc(httpTraceHdrs(server.CheckPartsHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 

cmd/storage-rest_test.go

@@ -166,7 +166,7 @@ func testStorageAPIDeleteVol(t *testing.T, storage StorageAPI) {
 	}
 }
 
-func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
+func testStorageAPIStatInfoFile(t *testing.T, storage StorageAPI) {
 	err := storage.MakeVol(context.Background(), "foo")
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
@@ -187,7 +187,7 @@ func testStorageAPICheckFile(t *testing.T, storage StorageAPI) {
 	}
 
 	for i, testCase := range testCases {
-		err := storage.CheckFile(context.Background(), testCase.volumeName, testCase.objectName)
+		_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile)
 		expectErr := (err != nil)
 
 		if expectErr != testCase.expectErr {
@@ -515,7 +515,7 @@ func TestStorageRESTClientDeleteVol(t *testing.T) {
 	testStorageAPIDeleteVol(t, restClient)
 }
 
-func TestStorageRESTClientCheckFile(t *testing.T) {
+func TestStorageRESTClientStatInfoFile(t *testing.T) {
 	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
@@ -523,7 +523,7 @@ func TestStorageRESTClientCheckFile(t *testing.T) {
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPICheckFile(t, restClient)
+	testStorageAPIStatInfoFile(t, restClient)
 }
 
 func TestStorageRESTClientListDir(t *testing.T) {

cmd/storagemetric_string.go

@@ -22,23 +22,22 @@ func _() {
 	_ = x[storageMetricRenameFile-11]
 	_ = x[storageMetricRenameData-12]
 	_ = x[storageMetricCheckParts-13]
-	_ = x[storageMetricCheckFile-14]
-	_ = x[storageMetricDelete-15]
-	_ = x[storageMetricDeleteVersions-16]
-	_ = x[storageMetricVerifyFile-17]
-	_ = x[storageMetricWriteAll-18]
-	_ = x[storageMetricDeleteVersion-19]
-	_ = x[storageMetricWriteMetadata-20]
-	_ = x[storageMetricUpdateMetadata-21]
-	_ = x[storageMetricReadVersion-22]
-	_ = x[storageMetricReadAll-23]
-	_ = x[storageStatInfoFile-24]
-	_ = x[storageMetricLast-25]
+	_ = x[storageMetricDelete-14]
+	_ = x[storageMetricDeleteVersions-15]
+	_ = x[storageMetricVerifyFile-16]
+	_ = x[storageMetricWriteAll-17]
+	_ = x[storageMetricDeleteVersion-18]
+	_ = x[storageMetricWriteMetadata-19]
+	_ = x[storageMetricUpdateMetadata-20]
+	_ = x[storageMetricReadVersion-21]
+	_ = x[storageMetricReadAll-22]
+	_ = x[storageStatInfoFile-23]
+	_ = x[storageMetricLast-24]
 }
 
-const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsCheckFileDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadAllstorageStatInfoFileLast"
+const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadAllstorageStatInfoFileLast"
 
-var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 137, 143, 157, 167, 175, 188, 201, 215, 226, 233, 252, 256}
+var _storageMetric_index = [...]uint8{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 224, 243, 247}
 
 func (i storageMetric) String() string {
 	if i >= storageMetric(len(_storageMetric_index)-1) {

cmd/streaming-signature-v4.go

@@ -93,7 +93,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 		return cred, "", "", time.Time{}, errCode
 	}
 
-	cred, _, errCode = checkKeyValid(signV4Values.Credential.accessKey)
+	cred, _, errCode = checkKeyValid(r, signV4Values.Credential.accessKey)
 	if errCode != ErrNone {
 		return cred, "", "", time.Time{}, errCode
 	}
@@ -117,7 +117,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 	}
 
 	// Query string.
-	queryStr := req.URL.Query().Encode()
+	queryStr := req.Form.Encode()
 
 	// Get canonical request.
 	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method)

cmd/sts-datatypes.go

@@ -112,7 +112,7 @@ type WebIdentityResult struct {
 	PackedPolicySize int `xml:",omitempty"`
 
 	// The issuing authority of the web identity token presented. For OpenID Connect
-	// ID tokens, this contains the value of the iss field. For OAuth 2.0 access tokens,
+	// ID tokens, this contains the value of the iss field. For OAuth 2.0 id_tokens,
 	// this contains the value of the ProviderId parameter that was passed in the
 	// AssumeRoleWithWebIdentity request.
 	Provider string `xml:",omitempty"`
@@ -162,7 +162,7 @@ type ClientGrantsResult struct {
 	PackedPolicySize int `xml:",omitempty"`
 
 	// The issuing authority of the web identity token presented. For OpenID Connect
-	// ID tokens, this contains the value of the iss field. For OAuth 2.0 access tokens,
+	// ID tokens, this contains the value of the iss field. For OAuth 2.0 id_tokens,
 	// this contains the value of the ProviderId parameter that was passed in the
 	// AssumeRoleWithClientGrants request.
 	Provider string `xml:",omitempty"`

cmd/sts-handlers.go

@@ -58,6 +58,8 @@ const (
 	// JWT claim keys
 	expClaim = "exp"
 	subClaim = "sub"
+	audClaim = "aud"
+	azpClaim = "azp"
 	issClaim = "iss"
 
 	// JWT claim to check the parent user
@@ -93,14 +95,14 @@ func registerSTSRouter(router *mux.Router) {
 	stsRouter.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
 		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType))
 		authOk := wildcard.MatchSimple(signV4Algorithm+"*", r.Header.Get(xhttp.Authorization))
-		noQueries := len(r.URL.Query()) == 0
+		noQueries := len(r.URL.RawQuery) == 0
 		return ctypeOk && authOk && noQueries
 	}).HandlerFunc(httpTraceAll(sts.AssumeRole))
 
 	// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
 	stsRouter.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
 		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType))
-		noQueries := len(r.URL.Query()) == 0
+		noQueries := len(r.URL.RawQuery) == 0
 		return ctypeOk && noQueries
 	}).HandlerFunc(httpTraceAll(sts.AssumeRoleWithSSO))
 
@@ -153,6 +155,18 @@ func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (user auth.Creden
 	return user, true, ErrSTSNone
 }
 
+func parseForm(r *http.Request) error {
+	if err := r.ParseForm(); err != nil {
+		return err
+	}
+	for k, v := range r.PostForm {
+		if _, ok := r.Form[k]; !ok {
+			r.Form[k] = v
+		}
+	}
+	return nil
+}
+
 // AssumeRole - implementation of AWS STS API AssumeRole to get temporary
 // credentials for regular users on Minio.
 // https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
@@ -165,7 +179,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := r.ParseForm(); err != nil {
+	if err := parseForm(r); err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
 		return
 	}
@@ -273,7 +287,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	ctx := newContext(r, w, "AssumeRoleSSOCommon")
 
 	// Parse the incoming form data.
-	if err := r.ParseForm(); err != nil {
+	if err := parseForm(r); err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
 		return
 	}
@@ -332,13 +346,50 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// REQUIRED. Audience(s) that this ID Token is intended for.
+	// It MUST contain the OAuth 2.0 client_id of the Relying Party
+	// as an audience value. It MAY also contain identifiers for
+	// other audiences. In the general case, the aud value is an
+	// array of case sensitive strings. In the common special case
+	// when there is one audience, the aud value MAY be a single
+	// case sensitive
+	audValues, ok := iampolicy.GetValuesFromClaims(m, audClaim)
+	if !ok {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+			errors.New("STS JWT Token has `aud` claim invalid, `aud` must match configured OpenID Client ID"))
+		return
+	}
+	if !audValues.Contains(globalOpenIDConfig.ClientID) {
+		// if audience claims is missing, look for "azp" claims.
+		// OPTIONAL. Authorized party - the party to which the ID
+		// Token was issued. If present, it MUST contain the OAuth
+		// 2.0 Client ID of this party. This Claim is only needed
+		// when the ID Token has a single audience value and that
+		// audience is different than the authorized party. It MAY
+		// be included even when the authorized party is the same
+		// as the sole audience. The azp value is a case sensitive
+		// string containing a StringOrURI value
+		azpValues, ok := iampolicy.GetValuesFromClaims(m, azpClaim)
+		if !ok {
+			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+				errors.New("STS JWT Token has `aud` claim invalid, `aud` must match configured OpenID Client ID"))
+			return
+		}
+		if !azpValues.Contains(globalOpenIDConfig.ClientID) {
+			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+				errors.New("STS JWT Token has `azp` claim invalid, `azp` must match configured OpenID Client ID"))
+			return
+		}
+	}
+
 	var subFromToken string
 	if v, ok := m[subClaim]; ok {
 		subFromToken, _ = v.(string)
 	}
 
 	if subFromToken == "" {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, errors.New("STS JWT Token has `sub` claim missing, `sub` claim is mandatory"))
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+			errors.New("STS JWT Token has `sub` claim missing, `sub` claim is mandatory"))
 		return
 	}
 
@@ -475,7 +526,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	defer logger.AuditLog(ctx, w, r, nil, stsLDAPPassword)
 
 	// Parse the incoming form data.
-	if err := r.ParseForm(); err != nil {
+	if err := parseForm(r); err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
 		return
 	}
@@ -541,7 +592,12 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		return
 	}
 
-	expiryDur := globalLDAPConfig.GetExpiryDuration()
+	expiryDur, err := globalLDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
+	if err != nil {
+		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		return
+	}
+
 	m := map[string]interface{}{
 		expClaim:  UTCNow().Add(expiryDur).Unix(),
 		ldapUser:  ldapUserDN,

cmd/test-utils_test.go

@@ -45,6 +45,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"path"
 	"reflect"
 	"sort"
 	"strconv"
@@ -1648,12 +1649,8 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
 			t.Fatal(failTestStr(unknownSignTestStr, "error response failed to parse error XML"))
 		}
 
-		if actualError.BucketName != bucketName {
-			t.Fatal(failTestStr(unknownSignTestStr, "error response bucket name differs from expected value"))
-		}
-
-		if actualError.Key != objectName {
-			t.Fatal(failTestStr(unknownSignTestStr, "error response object name differs from expected value"))
+		if path.Clean(actualError.Resource) != pathJoin(SlashSeparator, bucketName, SlashSeparator, objectName) {
+			t.Fatal(failTestStr(unknownSignTestStr, "error response resource differs from expected value"))
 		}
 	}
 
@@ -2035,13 +2032,15 @@ func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFuncti
 func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Handler {
 	// initialize a new mux router.
 	// goriilla/mux is the library used to register all the routes and handle them.
-	muxRouter := mux.NewRouter().SkipClean(true)
+	muxRouter := mux.NewRouter().SkipClean(true).UseEncodedPath()
 	if len(apiFunctions) > 0 {
 		// Iterate the list of API functions requested for and register them in mux HTTP handler.
 		registerAPIFunctions(muxRouter, objLayer, apiFunctions...)
+		muxRouter.Use(globalHandlers...)
 		return muxRouter
 	}
 	registerAPIRouter(muxRouter)
+	muxRouter.Use(globalHandlers...)
 	return muxRouter
 }