fix: allow CORS to work by default (#16713)

Co-authored-by: Harshavardhana <harsha@minio.io>

.github/workflows/enterprise-operator.yaml

@@ -1,25 +0,0 @@
-name: Pull Request Action
-on: [release]
-jobs:
-  create-pull-request:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Create PR
-        run: |
-          echo ""
-          echo "####################"
-          echo "# Branch Name:"
-          echo "####################"
-          echo ""
-          branchName=${{github.ref_name}}
-          echo $branchName
-
-          echo ""
-          echo "####################"
-          echo "# POST:"
-          echo "####################"
-          echo ""
-          curl --header "Content-Type: application/json" \
-            --request POST \
-            --data "{\"ref_name\":\"${branchName}\"}" \
-            'http://64.71.151.78:23411/' || true

CREDITS

@@ -58,6 +58,33 @@ SOFTWARE.
 
 ================================================================
 
+aead.dev/minisign
+https://aead.dev/minisign
+----------------------------------------------------------------
+MIT License
+
+Copyright (c) 2021 Andreas Auernhammer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 cloud.google.com/go
 https://cloud.google.com/go
 ----------------------------------------------------------------
@@ -17075,8 +17102,8 @@ https://github.com/minio/highwayhash
 
 ================================================================
 
-github.com/minio/kes
-https://github.com/minio/kes
+github.com/minio/kes-go
+https://github.com/minio/kes-go
 ----------------------------------------------------------------
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
@@ -21024,6 +21051,34 @@ https://github.com/minio/sio
 
 ================================================================
 
+github.com/minio/websocket
+https://github.com/minio/websocket
+----------------------------------------------------------------
+Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+  Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+  Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/minio/xxml
 https://github.com/minio/xxml
 ----------------------------------------------------------------

Dockerfile.release

@@ -29,7 +29,7 @@ COPY LICENSE /licenses/LICENSE
 RUN \
      microdnf clean all && \
      microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
      mkdir -p /opt/bin && chmod -R 777 /opt/bin && \

NOTICE

@@ -1,4 +1,4 @@
-MinIO Project, (C) 2015-2021 MinIO, Inc.
+MinIO Project, (C) 2015-2023 MinIO, Inc.
 
 This product includes software developed at MinIO, Inc.
 (https://min.io/).

cmd/acl-handlers.go

@@ -90,7 +90,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 	if aclHeader == "" {
 		acl := &accessControlPolicy{}
 		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
-			if err == io.EOF {
+			if terr, ok := err.(*xml.SyntaxError); ok && terr.Msg == io.EOF.Error() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
 					r.URL)
 				return

cmd/admin-bucket-handlers.go

@@ -31,7 +31,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -107,10 +107,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	}
 
 	// Call site replication hook.
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -172,7 +169,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
+	cred, _, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -202,7 +199,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		ops = madmin.GetTargetUpdateOps(r.Form)
 	} else {
 		var exists bool // true if arn exists
-		target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target)
+		target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target, "")
 		if exists && target.Arn != "" { // return pre-existing ARN
 			data, err := json.Marshal(target.Arn)
 			if err != nil {
@@ -228,10 +225,12 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		for _, op := range ops {
 			switch op {
 			case madmin.CredentialsUpdateType:
-				if !globalSiteReplicationSys.isEnabled() {
-					tgt.Credentials = target.Credentials
-					tgt.TargetBucket = target.TargetBucket
+				if globalSiteReplicationSys.isEnabled() {
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyEditError, err), r.URL)
+					return
 				}
+				tgt.Credentials = target.Credentials
+				tgt.TargetBucket = target.TargetBucket
 				tgt.Secure = target.Secure
 				tgt.Endpoint = target.Endpoint
 			case madmin.SyncUpdateType:

cmd/admin-handler-utils.go

@@ -23,7 +23,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
@@ -154,15 +154,9 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusForbidden,
 			}
-		case errors.Is(err, errIAMServiceAccount):
+		case errors.Is(err, errIAMServiceAccountNotAllowed):
 			apiErr = APIError{
-				Code:           "XMinioIAMServiceAccount",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, errIAMServiceAccountUsed):
-			apiErr = APIError{
-				Code:           "XMinioIAMServiceAccountUsed",
+				Code:           "XMinioIAMServiceAccountNotAllowed",
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}

cmd/admin-handlers-config-kv.go

@@ -35,6 +35,7 @@ import (
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -87,6 +88,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// Check if subnet proxy being deleted and if so the value of proxy of subnet
+	// target of logger webhook configuration also should be deleted
+	loggerWebhookProxyDeleted := setLoggerWebhookSubnetProxy(subSys, cfg)
+
 	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -101,6 +106,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	dynamic := config.SubSystemsDynamic.Contains(subSys)
 	if dynamic {
 		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
+		if subSys == config.SubnetSubSys && loggerWebhookProxyDeleted {
+			// Logger webhook proxy deleted, apply the dynamic changes
+			applyDynamic(ctx, objectAPI, cfg, config.LoggerWebhookSubSys, r, w)
+		}
 	}
 }
 
@@ -132,9 +141,10 @@ func (bce badConfigErr) Unwrap() error {
 }
 
 type setConfigResult struct {
-	Cfg     config.Config
-	SubSys  string
-	Dynamic bool
+	Cfg                     config.Config
+	SubSys                  string
+	Dynamic                 bool
+	LoggerWebhookCfgUpdated bool
 }
 
 // SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
@@ -175,6 +185,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	if result.Dynamic {
 		applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w)
+		// If logger webhook config updated (proxy due to callhome), explicitly dynamically
+		// apply the config
+		if result.LoggerWebhookCfgUpdated {
+			applyDynamic(ctx, objectAPI, result.Cfg, config.LoggerWebhookSubSys, r, w)
+		}
 	}
 
 	writeSuccessResponseHeadersOnly(w)
@@ -201,6 +216,10 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 		return
 	}
 
+	// Check if subnet proxy being set and if so set the same value to proxy of subnet
+	// target of logger webhook configuration
+	result.LoggerWebhookCfgUpdated = setLoggerWebhookSubnetProxy(result.SubSys, result.Cfg)
+
 	// Update the actual server config on disk.
 	if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil {
 		return
@@ -501,7 +520,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			case config.IdentityLDAPSubSys:
 				off = !xldap.Enabled(item.Config)
 			case config.IdentityTLSSubSys:
-				off = !globalSTSTLSConfig.Enabled
+				off = !globalIAMSys.STSTLSConfig.Enabled
 			case config.IdentityPluginSubSys:
 				off = !idplugin.Enabled(item.Config)
 			}
@@ -518,3 +537,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	writeSuccessResponseJSON(w, econfigData)
 }
+
+// setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
+// one being set for subnet proxy
+func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
+	if subSys == config.SubnetSubSys {
+		subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
+		loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
+		subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
+		if loggerWebhookSubnetProxy != subnetProxy {
+			subnetWebhookCfg.Set(logger.Proxy, subnetProxy)
+			return true
+		}
+	}
+	return false
+}

cmd/admin-handlers-idp-config.go

@@ -178,9 +178,9 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
 	var cfgInfos []madmin.IDPCfgInfo
 	switch subSys {
 	case madmin.IdentityOpenIDSubSys:
-		cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
+		cfgInfos, _ = globalIAMSys.OpenIDConfig.GetConfigInfo(s, cfgTarget)
 	case madmin.IdentityLDAPSubSys:
-		cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
+		cfgInfos, _ = globalIAMSys.LDAPConfig.GetConfigInfo(s, cfgTarget)
 	}
 
 	if len(cfgInfos) > 0 && !isUpdate {
@@ -240,10 +240,10 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
 		cfg := globalServerConfig.Clone()
-		cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
+		cfgList, err = globalIAMSys.OpenIDConfig.GetConfigList(cfg)
 	case madmin.LDAPIDPCfg:
 		cfg := globalServerConfig.Clone()
-		cfgList, err = globalLDAPConfig.GetConfigList(cfg)
+		cfgList, err = globalIAMSys.LDAPConfig.GetConfigList(cfg)
 
 	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -296,9 +296,9 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	var err error
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
-		cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+		cfgInfos, err = globalIAMSys.OpenIDConfig.GetConfigInfo(cfg, cfgName)
 	case madmin.LDAPIDPCfg:
-		cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
+		cfgInfos, err = globalIAMSys.LDAPConfig.GetConfigInfo(cfg, cfgName)
 	}
 	if err != nil {
 		if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
@@ -355,7 +355,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
 		subSys = config.IdentityOpenIDSubSys
-		cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
+		cfgInfos, err := globalIAMSys.OpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
 		if err != nil {
 			if errors.Is(err, openid.ErrProviderConfigNotFound) {
 				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
@@ -380,7 +380,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		}
 	case madmin.LDAPIDPCfg:
 		subSys = config.IdentityLDAPSubSys
-		cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
+		cfgInfos, err := globalIAMSys.LDAPConfig.GetConfigInfo(cfgCopy, cfgName)
 		if err != nil {
 			if errors.Is(err, openid.ErrProviderConfigNotFound) {
 				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)

cmd/admin-handlers-users.go

@@ -67,17 +67,14 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
 			IsDeleteReq: true,
 		},
 		UpdatedAt: UTCNow(),
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // ListBucketUsers - GET /minio/admin/v3/list-users?bucket={bucket}
@@ -178,7 +175,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -195,9 +192,9 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.GetUserAdminAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 		DenyOnly:        checkDenyOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
@@ -262,16 +259,13 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemGroupInfo,
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: updReq,
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // GetGroup - /minio/admin/v3/group?group=mygroup1
@@ -361,7 +355,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemGroupInfo,
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: madmin.GroupAddRemove{
@@ -371,10 +365,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 			},
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
@@ -404,7 +395,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -414,10 +405,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 			},
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
@@ -436,7 +424,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -480,9 +468,9 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.CreateUserAdminAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 		DenyOnly:        checkDenyOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
@@ -516,7 +504,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -524,10 +512,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 			UserReq:     &ureq,
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // TemporaryAccountInfo - GET /minio/admin/v3/temporary-account-info
@@ -543,7 +528,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -557,10 +542,11 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 
 	if !globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
 		Action:          iampolicy.ListTemporaryAccountsAdminAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
@@ -626,7 +612,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -664,10 +650,11 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	opts := newServiceAccountOpts{
-		accessKey: createReq.AccessKey,
-		secretKey: createReq.SecretKey,
-		comment:   createReq.Comment,
-		claims:    make(map[string]interface{}),
+		accessKey:  createReq.AccessKey,
+		secretKey:  createReq.SecretKey,
+		comment:    createReq.Comment,
+		expiration: createReq.Expiration,
+		claims:     make(map[string]interface{}),
 	}
 
 	// Find the user for the request sender (as it may be sent via a service
@@ -686,8 +673,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// It could be a regular user account or the root account.
 		_, isRegularUser := globalIAMSys.GetUser(ctx, targetUser)
 		if !isRegularUser && targetUser != globalActiveCred.AccessKey {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
-				fmt.Errorf("parent user %s does not exist. Cannot create service account", targetUser)), r.URL)
+			apiErr := toAdminAPIErr(ctx, errNoSuchUser)
+			apiErr.Description = fmt.Sprintf("Specified target user %s does not exist", targetUser)
+			writeErrorResponseJSON(ctx, w, apiErr, r.URL)
 			return
 		}
 	}
@@ -710,9 +698,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			AccountName:     requestorUser,
 			Groups:          requestorGroups,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
-			Claims:          claims,
+			Claims:          cred.Claims,
 			DenyOnly:        true,
 		}) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
@@ -744,9 +732,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			AccountName:     requestorUser,
 			Groups:          requestorGroups,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
-			Claims:          claims,
+			Claims:          cred.Claims,
 		}) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
@@ -754,9 +742,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 		// In case of LDAP we need to resolve the targetUser to a DN and
 		// query their groups:
-		if globalLDAPConfig.Enabled() {
+		if globalIAMSys.LDAPConfig.Enabled() {
 			opts.claims[ldapUserN] = targetUser // simple username
-			targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
+			targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
@@ -788,8 +776,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 	createResp := madmin.AddServiceAccountResp{
 		Credentials: madmin.Credentials{
-			AccessKey: newCred.AccessKey,
-			SecretKey: newCred.SecretKey,
+			AccessKey:  newCred.AccessKey,
+			SecretKey:  newCred.SecretKey,
+			Expiration: newCred.Expiration,
 		},
 	}
 
@@ -810,7 +799,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	// Call hook for cluster-replication if the service account is not for a
 	// root user.
 	if newCred.ParentUser != globalActiveCred.AccessKey {
-		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Create: &madmin.SRSvcAccCreate{
@@ -822,14 +811,11 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 					Claims:        opts.claims,
 					SessionPolicy: createReq.Policy,
 					Status:        auth.AccountOn,
+					Expiration:    createReq.Expiration,
 				},
 			},
 			UpdatedAt: updatedAt,
-		})
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return
-		}
+		}))
 	}
 }
 
@@ -846,7 +832,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -866,10 +852,11 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 
 	if !globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
 		Action:          iampolicy.UpdateServiceAccountAdminAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		requestUser := cred.AccessKey
 		if cred.ParentUser != "" {
@@ -907,6 +894,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		secretKey:     updateReq.NewSecretKey,
 		status:        updateReq.NewStatus,
 		comment:       updateReq.NewComment,
+		expiration:    updateReq.NewExpiration,
 		sessionPolicy: sp,
 	}
 	updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
@@ -917,7 +905,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Call site replication hook - non-root user accounts are replicated.
 	if svcAccount.ParentUser != globalActiveCred.AccessKey {
-		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Update: &madmin.SRSvcAccUpdate{
@@ -926,14 +914,11 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 					Status:        opts.status,
 					Comment:       opts.comment,
 					SessionPolicy: updateReq.NewPolicy,
+					Expiration:    updateReq.NewExpiration,
 				},
 			},
 			UpdatedAt: updatedAt,
-		})
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
+		}))
 	}
 
 	writeSuccessNoContent(w)
@@ -952,7 +937,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -972,10 +957,11 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 
 	if !globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
 		Action:          iampolicy.ListServiceAccountsAdminAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		requestUser := cred.AccessKey
 		if cred.ParentUser != "" {
@@ -1007,12 +993,18 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	var expiration *time.Time
+	if !svcAccount.Expiration.IsZero() && !svcAccount.Expiration.Equal(timeSentinel) {
+		expiration = &svcAccount.Expiration
+	}
+
 	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
 		Comment:       svcAccount.Comment,
 		AccountStatus: svcAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
+		Expiration:    expiration,
 	}
 
 	data, err := json.Marshal(infoResp)
@@ -1043,7 +1035,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -1057,10 +1049,11 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	if user != "" && user != cred.AccessKey {
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
 			Action:          iampolicy.ListServiceAccountsAdminAction,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
-			Claims:          claims,
+			Claims:          cred.Claims,
 		}) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
@@ -1117,7 +1110,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -1135,10 +1128,11 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 	adminPrivilege := globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
 		Action:          iampolicy.RemoveServiceAccountAdminAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	})
 
 	if !adminPrivilege {
@@ -1162,7 +1156,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Call site replication hook - non-root user accounts are replicated.
 	if svcAccount.ParentUser != "" && svcAccount.ParentUser != globalActiveCred.AccessKey {
-		if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Delete: &madmin.SRSvcAccDelete{
@@ -1170,10 +1164,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 				},
 			},
 			UpdatedAt: UTCNow(),
-		}); err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
+		}))
 	}
 
 	writeSuccessNoContent(w)
@@ -1192,7 +1183,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -1213,10 +1204,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			Groups:          cred.Groups,
 			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			ObjectName:      "",
-			Claims:          claims,
+			Claims:          cred.Claims,
 		}) {
 			rd = true
 		}
@@ -1226,10 +1217,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			Groups:          cred.Groups,
 			Action:          iampolicy.GetBucketLocationAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			ObjectName:      "",
-			Claims:          claims,
+			Claims:          cred.Claims,
 		}) {
 			rd = true
 		}
@@ -1239,10 +1230,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			Groups:          cred.Groups,
 			Action:          iampolicy.PutObjectAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			ObjectName:      "",
-			Claims:          claims,
+			Claims:          cred.Claims,
 		}) {
 			wr = true
 		}
@@ -1287,8 +1278,8 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		accountName = cred.ParentUser
 	}
 
-	roleArn := iampolicy.Args{Claims: claims}.GetRoleArn()
-	policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(claims, iamPolicyClaimNameOpenID())
+	roleArn := iampolicy.Args{Claims: cred.Claims}.GetRoleArn()
+	policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
 	var effectivePolicy iampolicy.Policy
 
 	var buf []byte
@@ -1534,14 +1525,11 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	// Call cluster-replication policy creation hook to replicate policy deletion to
 	// other minio clusters.
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type:      madmin.SRIAMItemPolicy,
 		Name:      policyName,
 		UpdatedAt: UTCNow(),
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // AddCannedPolicy - PUT /minio/admin/v3/add-canned-policy?name=<policy_name>
@@ -1602,15 +1590,12 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	// Call cluster-replication policy creation hook to replicate policy to
 	// other minio clusters.
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type:      madmin.SRIAMItemPolicy,
 		Name:      policyName,
 		Policy:    iamPolicyBytes,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // SetPolicyForUserOrGroup - PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
@@ -1669,7 +1654,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: entityName,
@@ -1678,10 +1663,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 			Policy:      policyName,
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 }
 
 // ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/polciy-entities?policy=xxx&user=xxx&group=xxx
@@ -1737,7 +1719,7 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
+	cred, _, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -1836,7 +1818,7 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: userOrGroup,
@@ -1845,10 +1827,7 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 			Policy:      strings.Join(policiesToAttach, ","),
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	writeResponse(w, http.StatusCreated, nil, mimeNone)
 }
@@ -1864,7 +1843,7 @@ func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
+	cred, _, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -1969,7 +1948,7 @@ func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: userOrGroup,
@@ -1978,10 +1957,7 @@ func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 			Policy:      strings.Join(policiesToDetach, ","),
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Return successful JSON response
 	writeSuccessNoContent(w)
@@ -2070,9 +2046,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case allUsersFile:
 			userIdentities := make(map[string]UserIdentity)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadUsers(ctx, regUser, userIdentities)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2100,9 +2074,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case allGroupsFile:
 			groups := make(map[string]GroupInfo)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadGroups(ctx, groups)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2119,9 +2091,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case allSvcAcctsFile:
 			serviceAccounts := make(map[string]UserIdentity)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2170,9 +2140,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case userPolicyMappingsFile:
 			userPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2189,9 +2157,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case groupPolicyMappingsFile:
 			groupPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2208,9 +2174,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case stsUserPolicyMappingsFile:
 			userPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2226,9 +2190,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case stsGroupPolicyMappingsFile:
 			groupPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2258,7 +2220,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -2370,9 +2332,9 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					AccountName:     cred.AccessKey,
 					Groups:          cred.Groups,
 					Action:          iampolicy.CreateUserAdminAction,
-					ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
-					Claims:          claims,
+					Claims:          cred.Claims,
 					DenyOnly:        checkDenyOnly,
 				}) {
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allUsersFile, accessKey), r.URL)
@@ -2461,12 +2423,12 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					return
 				}
 				if !globalIAMSys.IsAllowed(iampolicy.Args{
-					AccountName:     svcAcctReq.AccessKey,
-					Groups:          svcAcctReq.Groups,
+					AccountName:     cred.AccessKey,
+					Groups:          cred.Groups,
 					Action:          iampolicy.CreateServiceAccountAdminAction,
-					ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
-					Claims:          claims,
+					Claims:          cred.Claims,
 				}) {
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAccessDenied, err, allSvcAcctsFile, user), r.URL)
 					return
@@ -2485,6 +2447,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 						secretKey:     svcAcctReq.SecretKey,
 						status:        svcAcctReq.Status,
 						comment:       svcAcctReq.Comment,
+						expiration:    svcAcctReq.Expiration,
 						sessionPolicy: sp,
 					}
 					_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
@@ -2500,13 +2463,14 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					sessionPolicy: sp,
 					claims:        svcAcctReq.Claims,
 					comment:       svcAcctReq.Comment,
+					expiration:    svcAcctReq.Expiration,
 				}
 
 				// In case of LDAP we need to resolve the targetUser to a DN and
 				// query their groups:
-				if globalLDAPConfig.Enabled() {
+				if globalIAMSys.LDAPConfig.Enabled() {
 					opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
-					targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
+					targetUser, _, err := globalIAMSys.LDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
 					if err != nil {
 						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 						return

cmd/admin-handlers-users_test.go

@@ -1261,6 +1261,52 @@ func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) {
 	}
 }
 
+func (c *check) mustNotDelete(ctx context.Context, client *minio.Client, bucket string, vid string) {
+	c.Helper()
+
+	err := client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{VersionID: vid})
+	if err == nil {
+		c.Fatalf("user must not be allowed to delete")
+	}
+
+	err = client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{})
+	if err != nil {
+		c.Fatal("user must be able to create delete marker")
+	}
+}
+
+func (c *check) mustDownload(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
+	rd, err := client.GetObject(ctx, bucket, "some-object", minio.GetObjectOptions{})
+	if err != nil {
+		c.Fatalf("download did not succeed got %#v", err)
+	}
+	if _, err = io.Copy(io.Discard, rd); err != nil {
+		c.Fatalf("download did not succeed got %#v", err)
+	}
+}
+
+func (c *check) mustUploadReturnVersions(ctx context.Context, client *minio.Client, bucket string) []string {
+	c.Helper()
+	versions := []string{}
+	for i := 0; i < 5; i++ {
+		ui, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
+		if err != nil {
+			c.Fatalf("upload did not succeed got %#v", err)
+		}
+		versions = append(versions, ui.VersionID)
+	}
+	return versions
+}
+
+func (c *check) mustUpload(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
+	_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
+	if err != nil {
+		c.Fatalf("upload did not succeed got %#v", err)
+	}
+}
+
 func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
 	c.Helper()
 	_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})

cmd/admin-handlers.go

@@ -53,9 +53,9 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/logger/message/log"
 	xnet "github.com/minio/pkg/net"
 	"github.com/secure-io/sio-go"
 )
@@ -803,6 +803,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	for {
 		select {
 		case <-ctx.Done():
+			globalProfilerMu.Lock()
+			defer globalProfilerMu.Unlock()
 			for k, v := range globalProfiler {
 				v.Stop()
 				delete(globalProfiler, k)
@@ -1013,7 +1015,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					if hr.errBody == "" {
 						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr,
 							r.URL.Path, w.Header().Get(xhttp.AmzRequestID),
-							globalDeploymentID))
+							w.Header().Get(xhttp.AmzRequestHostID)))
 					} else {
 						errorRespJSON = encodeResponseJSON(APIErrorResponse{
 							Code:      hr.apiErr.Code,
@@ -1217,6 +1219,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	storageClass := strings.TrimSpace(r.Form.Get(peerRESTStorageClass))
 	customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
 	autotune := r.Form.Get("autotune") == "true"
+	noClear := r.Form.Get("noclear") == "true"
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1258,14 +1261,16 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 			return
 		}
 
-		if !bucketExists {
+		if !noClear && !bucketExists {
 			defer deleteObjectPerfBucket(objectAPI)
 		}
 	}
 
-	defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
-		DeletePrefix: true,
-	})
+	if !noClear {
+		defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
+			DeletePrefix: true,
+		})
+	}
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1776,12 +1781,48 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
+func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]madmin.ErasureSetInfo, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	z, _ := objectAPI.(*erasureServerPools)
+
+	poolsInfo := make(map[int]map[int]madmin.ErasureSetInfo)
+	for _, d := range allDisks {
+		poolInfo, ok := poolsInfo[d.PoolIndex]
+		if !ok {
+			poolInfo = make(map[int]madmin.ErasureSetInfo)
+		}
+		erasureSet, ok := poolInfo[d.SetIndex]
+		if !ok {
+			erasureSet.ID = d.SetIndex
+			cache := dataUsageCache{}
+			if err := cache.load(ctx, z.serverPools[d.PoolIndex].sets[d.SetIndex], dataUsageCacheName); err == nil {
+				dataUsageInfo := cache.dui(dataUsageRoot, nil)
+				erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount
+				erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount
+				erasureSet.Usage = dataUsageInfo.ObjectsTotalSize
+			}
+		}
+		erasureSet.RawCapacity += d.TotalSpace
+		erasureSet.RawUsage += d.UsedSpace
+		if d.Healing {
+			erasureSet.HealDisks = 1
+		}
+		poolInfo[d.SetIndex] = erasureSet
+		poolsInfo[d.PoolIndex] = poolInfo
+	}
+	return poolsInfo, nil
+}
+
+func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) madmin.InfoMessage {
 	kmsStat := fetchKMSStatus()
 
 	ldap := madmin.LDAP{}
-	if globalLDAPConfig.Enabled() {
-		ldapConn, err := globalLDAPConfig.LDAP.Connect()
+	if globalIAMSys.LDAPConfig.Enabled() {
+		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
 		//nolint:gocritic
 		if err != nil {
 			ldap.Status = string(madmin.ItemOffline)
@@ -1805,7 +1846,9 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 
 	assignPoolNumbers(servers)
 
+	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
 	var backend interface{}
+
 	mode := madmin.ItemInitializing
 
 	buckets := madmin.Buckets{}
@@ -1832,25 +1875,23 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 
 		// Fetching the backend information
 		backendInfo := objectAPI.BackendInfo()
-		if backendInfo.Type == madmin.Erasure {
-			// Calculate the number of online/offline disks of all nodes
-			var allDisks []madmin.Disk
-			for _, s := range servers {
-				allDisks = append(allDisks, s.Disks...)
-			}
-			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
+		// Calculate the number of online/offline disks of all nodes
+		var allDisks []madmin.Disk
+		for _, s := range servers {
+			allDisks = append(allDisks, s.Disks...)
+		}
+		onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
 
-			backend = madmin.ErasureBackend{
-				Type:             madmin.ErasureType,
-				OnlineDisks:      onlineDisks.Sum(),
-				OfflineDisks:     offlineDisks.Sum(),
-				StandardSCParity: backendInfo.StandardSCParity,
-				RRSCParity:       backendInfo.RRSCParity,
-			}
-		} else {
-			backend = madmin.FSBackend{
-				Type: madmin.FsType,
-			}
+		backend = madmin.ErasureBackend{
+			Type:             madmin.ErasureType,
+			OnlineDisks:      onlineDisks.Sum(),
+			OfflineDisks:     offlineDisks.Sum(),
+			StandardSCParity: backendInfo.StandardSCParity,
+			RRSCParity:       backendInfo.RRSCParity,
+		}
+
+		if poolsInfoEnabled {
+			poolsInfo, _ = getPoolsInfo(ctx, allDisks)
 		}
 	}
 
@@ -1876,6 +1917,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 		Services:     services,
 		Backend:      backend,
 		Servers:      servers,
+		Pools:        poolsInfo,
 	}
 }
 
@@ -1901,7 +1943,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 		ki.Error = err.Error()
 		return ki
 	}
-
+	defer resp.Body.Close()
 	decoder := json.NewDecoder(resp.Body)
 	if err := decoder.Decode(&ki); err != nil {
 		ki.Error = err.Error()
@@ -2078,8 +2120,8 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 			// No ellipses pattern. Anonymize host name from every pool arg
 			pools := strings.Fields(poolsArgs)
 			anonPools = make([]string, len(pools))
-			for _, arg := range pools {
-				anonPools = append(anonPools, anonAddr(arg))
+			for index, arg := range pools {
+				anonPools[index] = anonAddr(arg)
 			}
 			return cmdLineWithoutPools + strings.Join(anonPools, " ")
 		}
@@ -2182,7 +2224,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		getAndWriteSysConfig()
 
 		if query.Get("minioinfo") == "true" {
-			infoMessage := getServerInfo(healthCtx, nil)
+			infoMessage := getServerInfo(healthCtx, false, nil)
 			servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
@@ -2262,7 +2304,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	errResp := func(err error) {
 		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
-			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+			w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 		encodedErrorResponse := encodeResponse(errorResponse)
 		healthInfo.Error = string(encodedErrorResponse)
 		logger.LogIf(ctx, enc.Encode(healthInfo))
@@ -2357,7 +2399,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(getServerInfo(ctx, r))
+	jsonBytes, err := json.Marshal(getServerInfo(ctx, true, r))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-heal-ops.go

@@ -407,9 +407,6 @@ type healSequence struct {
 	// bucket, and object on which heal seq. was initiated
 	bucket, object string
 
-	// A channel of entities with heal result
-	respCh chan healResult
-
 	// Report healing progress
 	reportProgress bool
 
@@ -472,7 +469,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 	clientToken := mustGetUUID()
 
 	return &healSequence{
-		respCh:         make(chan healResult),
 		bucket:         bucket,
 		object:         objPrefix,
 		reportProgress: true,
@@ -719,28 +715,30 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 			if serverDebugLog {
 				logger.Info("Task in the queue: %#v", task)
 			}
-		case <-h.ctx.Done():
-			return nil
 		default:
 			// task queue is full, no more workers, we shall move on and heal later.
 			return nil
 		}
-	} else {
-		// respCh must be set for guaranteed result
-		task.respCh = h.respCh
-		select {
-		case globalBackgroundHealRoutine.tasks <- task:
-			if serverDebugLog {
-				logger.Info("Task in the queue: %#v", task)
-			}
-		case <-h.ctx.Done():
-			return nil
+		// Don't wait for result
+		return nil
+	}
+
+	// respCh must be set to wait for result.
+	// We make it size 1, so a result can always be written
+	// even if we aren't listening.
+	task.respCh = make(chan healResult, 1)
+	select {
+	case globalBackgroundHealRoutine.tasks <- task:
+		if serverDebugLog {
+			logger.Info("Task in the queue: %#v", task)
 		}
+	case <-h.ctx.Done():
+		return nil
 	}
 
 	// task queued, now wait for the response.
 	select {
-	case res := <-h.respCh:
+	case res := <-task.respCh:
 		if !h.reportProgress {
 			if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
 				return nil

cmd/admin-server-info.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -118,7 +119,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		config.EnvRootUser:          {},
 		config.EnvRootPassword:      {},
 		config.EnvMinIOSubnetAPIKey: {},
-		config.EnvKMSSecretKey:      {},
+		kms.EnvKMSSecretKey:         {},
 	}
 	for _, v := range os.Environ() {
 		if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {

cmd/api-errors.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -132,6 +132,7 @@ const (
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrReplicationDenyEditError
+	ErrRemoteTargetDenyEditError
 	ErrReplicationNoExistingObjects
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -164,7 +165,6 @@ const (
 	ErrMalformedDate
 	ErrMalformedPresignedDate
 	ErrMalformedCredentialDate
-	ErrMalformedCredentialRegion
 	ErrMalformedExpires
 	ErrNegativeExpires
 	ErrAuthHeaderEmpty
@@ -219,6 +219,7 @@ const (
 	ErrIncompatibleEncryptionMethod
 	ErrKMSNotConfigured
 	ErrKMSKeyNotFoundException
+	ErrKMSDefaultKeyAlreadyConfigured
 
 	ErrNoAccessKey
 	ErrInvalidToken
@@ -242,8 +243,6 @@ const (
 	// Add new extended error codes here.
 
 	// MinIO extended errors.
-	ErrReadQuorum
-	ErrWriteQuorum
 	ErrStorageFull
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
@@ -410,6 +409,8 @@ const (
 	ErrPostPolicyConditionInvalidFormat
 
 	ErrInvalidChecksum
+
+	apiErrCodeEnd // This is used only for the testing code
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -919,6 +920,11 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching ExistingsObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrRemoteTargetDenyEditError: {
+		Code:           "XMinioAdminRemoteTargetDenyEdit",
+		Description:    "Cannot alter remote target endpoint since this server is in a cluster replication setup. use `mc admin replicate update`",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrReplicationDenyEditError: {
 		Code:           "XMinioReplicationDenyEdit",
 		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
@@ -1166,6 +1172,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid keyId",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrKMSDefaultKeyAlreadyConfigured: {
+		Code:           "KMS.DefaultKeyAlreadyConfiguredException",
+		Description:    "A default encryption already exists and cannot be changed on KMS",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrNoAccessKey: {
 		Code:           "AccessDenied",
 		Description:    "No AWSAccessKey was presented",
@@ -1509,7 +1520,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBusy: {
-		Code:           "Busy",
+		Code:           "ServerBusy",
 		Description:    "The service is unavailable. Please retry.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
@@ -1990,8 +2001,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminGroupNotEmpty
 	case errNoSuchJob:
 		apiErr = ErrAdminNoSuchJob
-	case errNoSuchPolicy:
-		apiErr = ErrAdminNoSuchPolicy
 	case errNoPolicyToAttachOrDetach:
 		apiErr = ErrAdminPolicyChangeAlreadyApplied
 	case errSignatureMismatch:
@@ -2041,7 +2050,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case errKMSKeyNotFound:
 		apiErr = ErrKMSKeyNotFoundException
-
+	case errKMSDefaultKeyAlreadyConfigured:
+		apiErr = ErrKMSDefaultKeyAlreadyConfigured
 	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
@@ -2056,6 +2066,11 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectLockInvalidHeaders
 	case objectlock.ErrMalformedXML:
 		apiErr = ErrMalformedXML
+	default:
+		switch {
+		case errors.Is(err, errNoSuchPolicy):
+			apiErr = ErrAdminNoSuchPolicy
+		}
 	}
 
 	// Compression errors
@@ -2186,7 +2201,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrTransitionStorageClassNotFoundError
 	case InvalidObjectState:
 		apiErr = ErrInvalidObjectState
-
+	case PreConditionFailed:
+		apiErr = ErrPreconditionFailed
 	case BucketQuotaExceeded:
 		apiErr = ErrAdminBucketQuotaExceeded
 	case *event.ErrInvalidEventName:

cmd/api-errors_test.go

@@ -80,3 +80,19 @@ func TestAPIErrCode(t *testing.T) {
 		}
 	}
 }
+
+// Check if an API error is properly defined
+func TestAPIErrCodeDefinition(t *testing.T) {
+	for errAPI := ErrNone + 1; errAPI < apiErrCodeEnd; errAPI++ {
+		errCode, ok := errorCodes[errAPI]
+		if !ok {
+			t.Fatal(errAPI, "error code is not defined in the API error code table")
+		}
+		if errCode.Code == "" {
+			t.Fatal(errAPI, "error code has an empty XML code")
+		}
+		if errCode.HTTPStatusCode == 0 {
+			t.Fatal(errAPI, "error code has a zero HTTP status code")
+		}
+	}
+}

cmd/api-response.go

@@ -875,7 +875,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
-		w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -893,7 +893,7 @@ func writeErrorResponseString(ctx context.Context, w http.ResponseWriter, err AP
 // useful for admin APIs.
 func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }

cmd/api-router.go

@@ -516,7 +516,11 @@ func corsHandler(handler http.Handler) http.Handler {
 
 	return cors.New(cors.Options{
 		AllowOriginFunc: func(origin string) bool {
-			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
+			allowedOrigins := globalAPIConfig.getCorsAllowOrigins()
+			if len(allowedOrigins) == 0 {
+				allowedOrigins = []string{"*"}
+			}
+			for _, allowedOrigin := range allowedOrigins {
 				if wildcard.MatchSimple(allowedOrigin, origin) {
 					return true
 				}

cmd/auth-handler.go

@@ -135,7 +135,7 @@ func getRequestAuthType(r *http.Request) authType {
 	return authTypeUnknown
 }
 
-func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, map[string]interface{}, bool, APIErrorCode) {
+func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, bool, APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	s3Err := ErrAccessDenied
@@ -144,24 +144,24 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 		// We only support admin credentials to access admin APIs.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
-			return cred, nil, owner, s3Err
+			return cred, owner, s3Err
 		}
 
 		// we only support V4 (no presign) with auth body
 		s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
-		return cred, nil, owner, s3Err
+		return cred, owner, s3Err
 	}
 
-	return cred, cred.Claims, owner, ErrNone
+	return cred, owner, ErrNone
 }
 
 // checkAdminRequestAuth checks for authentication and authorization for the incoming
 // request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
 // are automatically rejected.
 func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
+	cred, owner, s3Err := validateAdminSignature(ctx, r, region)
 	if s3Err != ErrNone {
 		return cred, s3Err
 	}
@@ -169,9 +169,9 @@ func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolic
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.Action(action),
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		// Request is allowed return the appropriate access key.
 		return cred, ErrNone
@@ -253,7 +253,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrNoAccessKey
 	}
 
-	if token == "" && cred.IsTemp() {
+	if token == "" && cred.IsTemp() && !cred.IsServiceAccount() {
 		// Temporary credentials should always have x-amz-security-token
 		return nil, ErrInvalidToken
 	}
@@ -263,7 +263,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrInvalidToken
 	}
 
-	if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+	if !cred.IsServiceAccount() && cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
 		// validate token for temporary credentials only.
 		return nil, ErrInvalidToken
 	}
@@ -300,6 +300,17 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 	return s3Err
 }
 
+// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
+// passes versionID additionally.
+func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
+	logger.GetReqInfo(ctx).BucketName = bucketName
+	logger.GetReqInfo(ctx).ObjectName = objectName
+	logger.GetReqInfo(ctx).VersionID = versionID
+
+	_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
+	return s3Err
+}
+
 func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	if logger.GetReqInfo(ctx) == nil {
 		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
@@ -377,9 +388,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 		// Anonymous checks are not meant for ListAllBuckets action
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
 			Action:          action,
 			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, region, "", nil),
+			ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
 			IsOwner:         false,
 			ObjectName:      object,
 		}) {
@@ -392,9 +404,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 			// verify as a fallback.
 			if globalPolicySys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
+				Groups:          cred.Groups,
 				Action:          policy.ListBucketAction,
 				BucketName:      bucket,
-				ConditionValues: getConditionValues(r, region, "", nil),
+				ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
 				IsOwner:         false,
 				ObjectName:      object,
 			}) {
@@ -411,7 +424,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 			Groups:          cred.Groups,
 			Action:          iampolicy.Action(policy.DeleteObjectVersionAction),
 			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			ObjectName:      object,
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -425,7 +438,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 		Groups:          cred.Groups,
 		Action:          iampolicy.Action(action),
 		BucketName:      bucket,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		ObjectName:      object,
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -442,7 +455,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 			Groups:          cred.Groups,
 			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			ObjectName:      object,
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -637,7 +650,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		return ErrAccessDenied
 	}
 
-	conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
+	conditions := getConditionValues(r, "", cred)
 	conditions["object-lock-mode"] = []string{string(retMode)}
 	conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
 	if retDays > 0 {
@@ -711,7 +724,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 			Groups:          cred.Groups,
 			Action:          policy.Action(action),
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", "", nil),
+			ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 			IsOwner:         false,
 			ObjectName:      objectName,
 		}) {
@@ -725,7 +738,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 		Groups:          cred.Groups,
 		Action:          action,
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		ObjectName:      objectName,
 		IsOwner:         owner,
 		Claims:          cred.Claims,

cmd/auth-handler_test.go

@@ -492,7 +492,7 @@ func TestValidateAdminSignature(t *testing.T) {
 		if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
 			t.Fatalf("Unable to inititalized new signed http request %s", err)
 		}
-		_, _, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
+		_, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
 		if s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i+1, testCase.ErrCode, s3Error)
 		}

cmd/background-heal-ops.go

@@ -19,9 +19,14 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"runtime"
+	"strconv"
+	"time"
 
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/env"
 )
 
 // healTask represents what to heal along with options
@@ -56,13 +61,43 @@ func activeListeners() int {
 	return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
 }
 
-func waitForLowHTTPReq() {
-	var currentIO func() int
-	if httpServer := newHTTPServerFn(); httpServer != nil {
-		currentIO = httpServer.GetRequestCount
+func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) {
+	// No need to wait run at full speed.
+	if maxIO <= 0 {
+		return
 	}
 
-	globalHealConfig.Wait(currentIO, activeListeners)
+	const waitTick = 100 * time.Millisecond
+
+	tmpMaxWait := maxWait
+
+	for currentIO() >= maxIO {
+		if tmpMaxWait > 0 {
+			if tmpMaxWait < waitTick {
+				time.Sleep(tmpMaxWait)
+			} else {
+				time.Sleep(waitTick)
+			}
+			tmpMaxWait -= waitTick
+		}
+		if tmpMaxWait <= 0 {
+			return
+		}
+	}
+}
+
+func currentHTTPIO() int {
+	httpServer := newHTTPServerFn()
+	if httpServer == nil {
+		return 0
+	}
+
+	return httpServer.GetRequestCount() - activeListeners()
+}
+
+func waitForLowHTTPReq() {
+	maxIO, maxWait, _ := globalHealConfig.Clone()
+	waitForLowIO(maxIO, maxWait, currentHTTPIO)
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
@@ -111,9 +146,19 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 
 func newHealRoutine() *healRoutine {
 	workers := runtime.GOMAXPROCS(0) / 2
+
+	if envHealWorkers := env.Get("_MINIO_HEAL_WORKERS", ""); envHealWorkers != "" {
+		if numHealers, err := strconv.Atoi(envHealWorkers); err != nil {
+			logger.LogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
+		} else {
+			workers = numHealers
+		}
+	}
+
 	if workers == 0 {
 		workers = 4
 	}
+
 	return &healRoutine{
 		tasks:   make(chan healTask),
 		workers: workers,

cmd/background-newdisks-heal-ops.go

@@ -299,7 +299,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	defer disk.Close()
 	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
 	if poolIdx < 0 {
-		return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
+		return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
 	}
 
 	// Calculate the set index where the current endpoint belongs
@@ -310,14 +310,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 	if setIdx < 0 {
-		return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
+		return fmt.Errorf("unexpected set index (%d) found for  %s", setIdx, disk.Endpoint())
 	}
 
 	// Prevent parallel erasure set healing
 	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
 	if err != nil {
-		return err
+		return fmt.Errorf("Healing of drive '%v' on %s pool, belonging to %s erasure set already in progress: %w",
+			disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1), err)
 	}
 	ctx = lkctx.Context()
 	defer locker.Unlock(lkctx)
@@ -325,19 +326,20 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	// Load healing tracker in this disk
 	tracker, err := loadHealingTracker(ctx, disk)
 	if err != nil {
-		// A healing track can be not found when another disk in the same
-		// erasure set and same healing-id successfully finished healing.
-		if err == errFileNotFound {
+		// A healing tracker may be deleted if another disk in the
+		// same erasure set with same healing-id successfully finished
+		// healing.
+		if errors.Is(err, errFileNotFound) {
 			return nil
 		}
-		logger.LogIf(ctx, fmt.Errorf("Unable to load a healing tracker on '%s': %w", disk, err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
 		tracker = newHealingTracker(disk, mustGetUUID())
 	}
 
-	logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
+	logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
 
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
-	// Buckets data are dispersed in multiple zones/sets, make
+	// Buckets data are dispersed in multiple pools/sets, make
 	// sure to heal all bucket metadata configuration.
 	buckets = append(buckets, BucketInfo{
 		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
@@ -355,7 +357,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	})
 
 	if serverDebugLog {
-		logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
+		logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
 	}
 
 	// Load bucket totals
@@ -378,9 +380,13 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	if tracker.ItemsFailed > 0 {
-		logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
 	} else {
-		logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	}
+
+	if len(tracker.QueuedBuckets) > 0 {
+		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
 	}
 
 	if serverDebugLog {
@@ -388,7 +394,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		logger.Info("\n")
 	}
 
-	if tracker.HealID == "" { // HealID is empty only before Feb 2023
+	if tracker.HealID == "" { // HealID was empty only before Feb 2023
 		logger.LogIf(ctx, tracker.delete(ctx))
 		return nil
 	}
@@ -397,7 +403,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
-			if err != errFileNotFound {
+			if !errors.Is(err, errFileNotFound) {
 				logger.LogIf(ctx, err)
 			}
 			continue
@@ -442,10 +448,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 			for _, disk := range healDisks {
 				go func(disk Endpoint) {
 					globalBackgroundHealState.setDiskHealingStatus(disk, true)
-					err := healFreshDisk(ctx, z, disk)
-					if err != nil {
+					if err := healFreshDisk(ctx, z, disk); err != nil {
 						globalBackgroundHealState.setDiskHealingStatus(disk, false)
-						printEndpointError(disk, err, false)
+						if !errors.Is(err, context.Canceled) {
+							printEndpointError(disk, err, false)
+						}
 						return
 					}
 					// Only upon success pop the healed disk.

cmd/batch-handlers.go

@@ -28,6 +28,8 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -41,7 +43,9 @@ import (
 	"github.com/minio/minio/internal/auth"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	"github.com/minio/pkg/wildcard"
 	"gopkg.in/yaml.v2"
@@ -356,6 +360,8 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 
 // batchJobInfo current batch replication information
 type batchJobInfo struct {
+	mu sync.RWMutex `json:"-" msg:"-"`
+
 	Version       int       `json:"-" msg:"v"`
 	JobID         string    `json:"jobID" msg:"jid"`
 	JobType       string    `json:"jobType" msg:"jt"`
@@ -425,6 +431,9 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 		return fmt.Errorf("batchRepl: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
 	}
 
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+
 	// OK, parse data.
 	if _, err = ri.UnmarshalMsg(data[4:]); err != nil {
 		return err
@@ -439,8 +448,11 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 	return nil
 }
 
-func (ri batchJobInfo) clone() batchJobInfo {
-	return batchJobInfo{
+func (ri *batchJobInfo) clone() *batchJobInfo {
+	ri.mu.RLock()
+	defer ri.mu.RUnlock()
+
+	return &batchJobInfo{
 		Version:          ri.Version,
 		JobID:            ri.JobID,
 		JobType:          ri.JobType,
@@ -458,21 +470,6 @@ func (ri batchJobInfo) clone() batchJobInfo {
 	}
 }
 
-func (ri batchJobInfo) save(ctx context.Context, api ObjectLayer, jobLocation string) error {
-	data := make([]byte, 4, ri.Msgsize()+4)
-
-	// Initialize the header.
-	binary.LittleEndian.PutUint16(data[0:2], batchReplFormat)
-	binary.LittleEndian.PutUint16(data[2:4], batchReplVersion)
-
-	buf, err := ri.MarshalMsg(data)
-	if err != nil {
-		return err
-	}
-
-	return saveConfig(ctx, api, pathJoin(jobLocation, batchReplName), buf)
-}
-
 func (ri *batchJobInfo) countItem(size int64, dmarker, success bool) {
 	if ri == nil {
 		return
@@ -499,14 +496,28 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 		return errInvalidArgument
 	}
 	now := UTCNow()
+	ri.mu.Lock()
 	if now.Sub(ri.LastUpdate) >= duration {
 		if serverDebugLog {
 			console.Debugf("batchReplicate: persisting batchReplication info on drive: threshold:%s, batchRepl:%#v\n", now.Sub(ri.LastUpdate), ri)
 		}
 		ri.LastUpdate = now
 		ri.Version = batchReplVersionV1
-		return ri.save(ctx, api, jobLocation)
+
+		data := make([]byte, 4, ri.Msgsize()+4)
+
+		// Initialize the header.
+		binary.LittleEndian.PutUint16(data[0:2], batchReplFormat)
+		binary.LittleEndian.PutUint16(data[2:4], batchReplVersion)
+
+		buf, err := ri.MarshalMsg(data)
+		ri.mu.Unlock()
+		if err != nil {
+			return err
+		}
+		return saveConfig(ctx, api, pathJoin(jobLocation, batchReplName), buf)
 	}
+	ri.mu.Unlock()
 	return nil
 }
 
@@ -514,6 +525,10 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
 	if ri == nil {
 		return
 	}
+
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+
 	ri.Bucket = bucket
 	ri.Object = info.Name
 	ri.countItem(info.Size, info.DeleteMarker, failed)
@@ -529,7 +544,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if err := ri.load(ctx, api, job); err != nil {
 		return err
 	}
-	globalBatchJobsMetrics.save(job.ID, ri.clone())
+	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
 	delay := job.Replicate.Flags.Retry.Delay
@@ -618,10 +633,24 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if err != nil {
 		return err
 	}
+	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
 
 	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		attempts := attempts
+
 		ctx, cancel := context.WithCancel(ctx)
 
 		results := make(chan ObjectInfo, 100)
@@ -635,31 +664,43 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		}
 
 		for result := range results {
-			stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, result)
-			success := true
-			if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
-				if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
-					// object must be deleted concurrently, allow
-					// these failures but do not count them
-					continue
+			result := result
+			wk.Take()
+			go func() {
+				defer wk.Give()
+
+				stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, result)
+				success := true
+				if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
+					if miniogo.ToErrorResponse(err).Code == "PreconditionFailed" {
+						// pre-condition failed means we already have the object copied over.
+						return
+					}
+					// object must be deleted concurrently, allow these failures but do not count them
+					if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
+						return
+					}
+					stopFn(err)
+					logger.LogIf(ctx, err)
+					success = false
+				} else {
+					stopFn(nil)
 				}
-				stopFn(err)
-				logger.LogIf(ctx, err)
-				success = false
-			} else {
-				stopFn(nil)
-			}
-			ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
-			globalBatchJobsMetrics.save(job.ID, ri.clone())
-			// persist in-memory state to disk after every 10secs.
-			logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job.Location))
+				ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
+				globalBatchJobsMetrics.save(job.ID, ri)
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job.Location))
+			}()
 		}
+		wk.Wait()
 
 		ri.RetryAttempts = attempts
 		ri.Complete = ri.ObjectsFailed == 0
 		ri.Failed = ri.ObjectsFailed > 0
 
-		globalBatchJobsMetrics.save(job.ID, ri.clone())
+		globalBatchJobsMetrics.save(job.ID, ri)
+		// persist in-memory state to disk.
+		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job.Location))
 
 		buf, _ := json.Marshal(ri)
 		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
@@ -697,7 +738,7 @@ func (e batchReplicationJobError) Error() string {
 }
 
 // Validate validates the job definition input
-func (r *BatchJobReplicateV1) Validate(ctx context.Context, o ObjectLayer) error {
+func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
 	if r == nil {
 		return nil
 	}
@@ -773,6 +814,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, o ObjectLayer) error
 	if err != nil {
 		return err
 	}
+	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
 	vcfg, err := c.GetBucketVersioning(ctx, r.Target.Bucket)
 	if err != nil {
@@ -811,7 +853,7 @@ func (j BatchJobRequest) Type() madmin.BatchJobType {
 // persisting the job request
 func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 	if j.Replicate != nil {
-		return j.Replicate.Validate(ctx, o)
+		return j.Replicate.Validate(ctx, j, o)
 	}
 	return errInvalidArgument
 }
@@ -1029,7 +1071,6 @@ type BatchJobPool struct {
 	mu           sync.Mutex
 	jobCh        chan *BatchJobRequest
 	workerKillCh chan struct{}
-	workerWg     sync.WaitGroup
 	workerSize   int
 }
 
@@ -1074,7 +1115,6 @@ func (j *BatchJobPool) AddWorker() {
 	if j == nil {
 		return
 	}
-	defer j.workerWg.Done()
 	for {
 		select {
 		case <-j.ctx.Done():
@@ -1110,7 +1150,6 @@ func (j *BatchJobPool) ResizeWorkers(n int) {
 
 	for j.workerSize < n {
 		j.workerSize++
-		j.workerWg.Add(1)
 		go j.AddWorker()
 	}
 	for j.workerSize > n {
@@ -1138,11 +1177,11 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 //msgp:ignore batchJobMetrics
 type batchJobMetrics struct {
 	sync.RWMutex
-	metrics map[string]batchJobInfo
+	metrics map[string]*batchJobInfo
 }
 
 var globalBatchJobsMetrics = batchJobMetrics{
-	metrics: make(map[string]batchJobInfo),
+	metrics: make(map[string]*batchJobInfo),
 }
 
 //msgp:ignore batchReplicationMetric
@@ -1210,11 +1249,11 @@ func (m *batchJobMetrics) delete(jobID string) {
 	delete(m.metrics, jobID)
 }
 
-func (m *batchJobMetrics) save(jobID string, ri batchJobInfo) {
+func (m *batchJobMetrics) save(jobID string, ri *batchJobInfo) {
 	m.Lock()
 	defer m.Unlock()
 
-	m.metrics[jobID] = ri
+	m.metrics[jobID] = ri.clone()
 }
 
 func (m *batchJobMetrics) trace(d batchReplicationMetric, job string, attempts int, info ObjectInfo) func(err error) {

cmd/bucket-encryption-handlers.go

@@ -25,7 +25,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -114,15 +114,12 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: &cfgStr,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -204,16 +201,14 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+
 	// Call site replication hook.
-	//
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: nil,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
+
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers.go

@@ -39,6 +39,7 @@ import (
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/auth"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -359,7 +360,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				Groups:          cred.Groups,
 				Action:          iampolicy.ListBucketAction,
 				BucketName:      bucketInfo.Name,
-				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				ConditionValues: getConditionValues(r, "", cred),
 				IsOwner:         owner,
 				ObjectName:      "",
 				Claims:          cred.Claims,
@@ -371,7 +372,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				Groups:          cred.Groups,
 				Action:          iampolicy.GetBucketLocationAction,
 				BucketName:      bucketInfo.Name,
-				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				ConditionValues: getConditionValues(r, "", cred),
 				IsOwner:         owner,
 				ObjectName:      "",
 				Claims:          cred.Claims,
@@ -494,7 +495,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	vc, _ := globalBucketVersioningSys.Get(bucket)
 	oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
 	for index, object := range deleteObjectsReq.Objects {
-		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
+		if apiErrCode := checkRequestAuthTypeWithVID(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName, object.VersionID); apiErrCode != ErrNone {
 			if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
 				return
@@ -742,7 +743,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				AccountName:     cred.AccessKey,
 				Groups:          cred.Groups,
 				Action:          action,
-				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				ConditionValues: getConditionValues(r, "", cred),
 				BucketName:      bucket,
 				IsOwner:         owner,
 				Claims:          cred.Claims,
@@ -754,19 +755,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Parse incoming location constraint.
-	location, s3Error := parseLocationConstraint(r)
+	_, s3Error = parseLocationConstraint(r)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 
-	// Validate if location sent by the client is valid, reject
-	// requests which do not follow valid region requirements.
-	if !isValidLocation(location) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL)
-		return
-	}
-
 	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
 		logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
@@ -851,7 +845,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Call site replication hook
-	globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+	logger.LogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
 
 	// Make sure to add Location information here only for bucket
 	if cp := pathClean(r.URL.Path); cp != "" {
@@ -983,7 +977,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.PutObjectAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		BucketName:      bucket,
 		ObjectName:      object,
 		IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
@@ -1141,6 +1135,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
+	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
+		defer sendEvent(eventArgs{
+			EventName:    event.ObjectManyVersions,
+			BucketName:   objInfo.Bucket,
+			Object:       objInfo,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         handlers.GetSourceIP(r),
+		})
+	}
 
 	if redirectURL != nil { // success_action_redirect is valid and set.
 		v := redirectURL.Query()
@@ -1200,7 +1205,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	readable := globalPolicySys.IsAllowed(policy.Args{
 		Action:          policy.ListBucketAction,
 		BucketName:      bucket,
-		ConditionValues: getConditionValues(r, "", "", nil),
+		ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 		IsOwner:         false,
 	})
 
@@ -1208,7 +1213,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	writable := globalPolicySys.IsAllowed(policy.Args{
 		Action:          policy.PutObjectAction,
 		BucketName:      bucket,
-		ConditionValues: getConditionValues(r, "", "", nil),
+		ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 		IsOwner:         false,
 	})
 
@@ -1352,11 +1357,9 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
+
 	// Call site replication hook.
-	if err := globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	logger.LogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))
 
 	// Write success response.
 	writeSuccessNoContent(w)
@@ -1425,15 +1428,12 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 		Bucket:           bucket,
 		ObjectLockConfig: &cfgStr,
 		UpdatedAt:        updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -1532,15 +1532,12 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		Tags:      &cfgStr,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -1611,14 +1608,11 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)

cmd/bucket-lifecycle.go

@@ -152,7 +152,6 @@ type transitionTask struct {
 }
 
 type transitionState struct {
-	once         sync.Once
 	transitionCh chan transitionTask
 
 	ctx        context.Context
@@ -169,10 +168,7 @@ type transitionState struct {
 
 func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 	select {
-	case <-GlobalContext.Done():
-		t.once.Do(func() {
-			close(t.transitionCh)
-		})
+	case <-t.ctx.Done():
 	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
 	default:
 	}
@@ -180,20 +176,32 @@ func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 
 var globalTransitionState *transitionState
 
-func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
+// newTransitionState returns a transitionState object ready to be initialized
+// via its Init method.
+func newTransitionState(ctx context.Context) *transitionState {
 	return &transitionState{
 		transitionCh: make(chan transitionTask, 10000),
 		ctx:          ctx,
-		objAPI:       objAPI,
 		killCh:       make(chan struct{}),
 		lastDayStats: make(map[string]*lastDayTierStats),
 	}
 }
 
+// Init initializes t with given objAPI and instantiates the configured number
+// of transition workers.
+func (t *transitionState) Init(objAPI ObjectLayer) {
+	n := globalAPIConfig.getTransitionWorkers()
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	t.objAPI = objAPI
+	t.updateWorkers(n)
+}
+
 // PendingTasks returns the number of ILM transition tasks waiting for a worker
 // goroutine.
 func (t *transitionState) PendingTasks() int {
-	return len(globalTransitionState.transitionCh)
+	return len(t.transitionCh)
 }
 
 // ActiveTasks returns the number of active (ongoing) ILM transition tasks.
@@ -202,20 +210,20 @@ func (t *transitionState) ActiveTasks() int {
 }
 
 // worker waits for transition tasks
-func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
+func (t *transitionState) worker(objectAPI ObjectLayer) {
 	for {
 		select {
 		case <-t.killCh:
 			return
-		case <-ctx.Done():
+		case <-t.ctx.Done():
 			return
 		case task, ok := <-t.transitionCh:
 			if !ok {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
+			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.tier); err != nil {
+				logger.LogIf(t.ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
 					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
@@ -258,9 +266,15 @@ func (t *transitionState) getDailyAllTierStats() DailyAllTierStats {
 func (t *transitionState) UpdateWorkers(n int) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
+	if t.objAPI == nil { // Init hasn't been called yet.
+		return
+	}
+	t.updateWorkers(n)
+}
 
+func (t *transitionState) updateWorkers(n int) {
 	for t.numWorkers < n {
-		go t.worker(t.ctx, t.objAPI)
+		go t.worker(t.objAPI)
 		t.numWorkers++
 	}
 
@@ -270,12 +284,6 @@ func (t *transitionState) UpdateWorkers(n int) {
 	}
 }
 
-func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
-	globalTransitionState = newTransitionState(ctx, objectAPI)
-	n := globalAPIConfig.getTransitionWorkers()
-	globalTransitionState.UpdateWorkers(n)
-}
-
 var errInvalidStorageClass = errors.New("invalid storage class")
 
 func validateTransitionTier(lc *lifecycle.Lifecycle) error {
@@ -366,7 +374,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 			EventName:  eventName,
 			BucketName: oi.Bucket,
 			Object:     objInfo,
-			Host:       "Internal: [ILM-EXPIRY]",
+			Host:       "Internal: [ILM-Expiry]",
 		})
 
 	case expireRestoredObj:
@@ -660,7 +668,9 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	if len(objInfo.UserTags) != 0 {
 		meta[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-
+	// Set restore object status
+	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
+	meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
 	return ObjectOptions{
 		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
 		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),

cmd/bucket-metadata-sys.go

@@ -21,10 +21,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"runtime"
 	"sync"
 	"time"
 
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	bucketsse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -40,6 +42,8 @@ import (
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
 type BucketMetadataSys struct {
+	objAPI ObjectLayer
+
 	sync.RWMutex
 	metadataMap map[string]BucketMetadata
 }
@@ -53,13 +57,29 @@ func (sys *BucketMetadataSys) Count() int {
 }
 
 // Remove bucket metadata from memory.
-func (sys *BucketMetadataSys) Remove(bucket string) {
+func (sys *BucketMetadataSys) Remove(buckets ...string) {
 	sys.Lock()
-	delete(sys.metadataMap, bucket)
-	globalBucketMonitor.DeleteBucket(bucket)
+	for _, bucket := range buckets {
+		delete(sys.metadataMap, bucket)
+		globalBucketMonitor.DeleteBucket(bucket)
+	}
 	sys.Unlock()
 }
 
+// RemoveStaleBuckets removes all stale buckets in memory that are not on disk.
+func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	for bucket := range sys.metadataMap {
+		if diskBuckets.Contains(bucket) {
+			continue
+		} // doesn't exist on disk remove from memory.
+		delete(sys.metadataMap, bucket)
+		globalBucketMonitor.DeleteBucket(bucket)
+	}
+}
+
 // Set - sets a new metadata in-memory.
 // Only a shallow copy is saved and fields with references
 // cannot be modified without causing a race condition,
@@ -182,7 +202,7 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 // GetVersioningConfig returns configured versioning config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, meta.Created, nil
@@ -195,7 +215,7 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket}
@@ -211,7 +231,7 @@ func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.
 // GetObjectLockConfig returns configured object lock config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket}
@@ -227,7 +247,7 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
 // GetLifecycleConfig returns configured lifecycle config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketLifecycleNotFound{Bucket: bucket}
@@ -243,7 +263,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -253,7 +273,7 @@ func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Confi
 // GetSSEConfig returns configured SSE config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket}
@@ -268,7 +288,7 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 
 // CreatedAt returns the time of creation of bucket
 func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return time.Time{}, err
 	}
@@ -278,7 +298,7 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
@@ -294,7 +314,7 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, ti
 // GetQuotaConfig returns configured bucket quota
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) {
-	meta, err := sys.GetConfig(ctx, bucket)
+	meta, _, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketQuotaConfigNotFound{Bucket: bucket}
@@ -307,7 +327,7 @@ func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string)
 // GetReplicationConfig returns configured bucket replication config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, time.Time, error) {
-	meta, err := sys.GetConfig(ctx, bucket)
+	meta, reloaded, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
@@ -318,13 +338,18 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 	if meta.replicationConfig == nil {
 		return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
 	}
+	if reloaded {
+		globalBucketTargetSys.set(BucketInfo{
+			Name: bucket,
+		}, meta)
+	}
 	return meta.replicationConfig, meta.ReplicationConfigUpdatedAt, nil
 }
 
 // GetBucketTargetsConfig returns configured bucket targets for this bucket
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, reloaded, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketRemoteTargetNotFound{Bucket: bucket}
@@ -334,12 +359,16 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	if meta.bucketTargetConfig == nil {
 		return nil, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
+	if reloaded {
+		globalBucketTargetSys.set(BucketInfo{
+			Name: bucket,
+		}, meta)
+	}
 	return meta.bucketTargetConfig, nil
 }
 
-// GetConfig returns a specific configuration from the bucket metadata.
-// The returned object may not be modified.
-func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (BucketMetadata, error) {
+// GetConfigFromDisk read bucket metadata config from disk.
+func (sys *BucketMetadataSys) GetConfigFromDisk(ctx context.Context, bucket string) (BucketMetadata, error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return newBucketMetadata(bucket), errServerNotInitialized
@@ -349,21 +378,37 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (Buc
 		return newBucketMetadata(bucket), errInvalidArgument
 	}
 
+	return loadBucketMetadata(ctx, objAPI, bucket)
+}
+
+// GetConfig returns a specific configuration from the bucket metadata.
+// The returned object may not be modified.
+// reloaded will be true if metadata refreshed from disk
+func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (meta BucketMetadata, reloaded bool, err error) {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return newBucketMetadata(bucket), reloaded, errServerNotInitialized
+	}
+
+	if isMinioMetaBucketName(bucket) {
+		return newBucketMetadata(bucket), reloaded, errInvalidArgument
+	}
+
 	sys.RLock()
 	meta, ok := sys.metadataMap[bucket]
 	sys.RUnlock()
 	if ok {
-		return meta, nil
+		return meta, reloaded, nil
 	}
-	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
+	meta, err = loadBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
-		return meta, err
+		return meta, reloaded, err
 	}
 	sys.Lock()
 	sys.metadataMap[bucket] = meta
 	sys.Unlock()
 
-	return meta, nil
+	return meta, true, nil
 }
 
 // Init - initializes bucket metadata system for all buckets.
@@ -372,59 +417,129 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return errServerNotInitialized
 	}
 
+	sys.objAPI = objAPI
+
 	// Load bucket metadata sys in background
-	go sys.load(ctx, buckets, objAPI)
+	go sys.init(ctx, buckets)
+	return nil
+}
+
+func (sys *BucketMetadataSys) loadBucketMetadata(ctx context.Context, bucket BucketInfo) error {
+	meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket.Name)
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	sys.metadataMap[bucket.Name] = meta
+	sys.Unlock()
+
 	return nil
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo) {
 	g := errgroup.WithNErrs(len(buckets))
+	bucketMetas := make([]BucketMetadata, len(buckets))
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
-			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
+			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
 				// Ensure heal opts for bucket metadata be deep healed all the time.
 				ScanMode: madmin.HealDeepScan,
 				Recreate: true,
 			})
-			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
+			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index].Name)
 			if err != nil {
-				if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
-					meta = newBucketMetadata(buckets[index].Name)
-				} else {
-					return err
-				}
+				return err
 			}
-			sys.Lock()
-			sys.metadataMap[buckets[index].Name] = meta
-			sys.Unlock()
-
-			globalEventNotifier.set(buckets[index], meta) // set notification targets
-
-			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
-
+			bucketMetas[index] = meta
 			return nil
 		}, index)
 	}
-	for _, err := range g.Wait() {
+
+	errs := g.Wait()
+	for _, err := range errs {
 		if err != nil {
 			logger.LogIf(ctx, err)
 		}
 	}
+
+	// Hold lock here to update in-memory map at once,
+	// instead of serializing the Go routines.
+	sys.Lock()
+	for i, meta := range bucketMetas {
+		if errs[i] != nil {
+			continue
+		}
+		sys.metadataMap[buckets[i].Name] = meta
+	}
+	sys.Unlock()
+
+	for i, meta := range bucketMetas {
+		if errs[i] != nil {
+			continue
+		}
+		globalEventNotifier.set(buckets[i], meta)   // set notification targets
+		globalBucketTargetSys.set(buckets[i], meta) // set remote replication targets
+	}
+}
+
+func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
+	const bucketMetadataRefresh = 15 * time.Minute
+
+	t := time.NewTimer(bucketMetadataRefresh)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
+			if err != nil {
+				logger.LogIf(ctx, err)
+				continue
+			}
+
+			// Handle if we have some buckets in-memory those are stale.
+			// first delete them and then replace the newer state()
+			// from disk.
+			diskBuckets := set.CreateStringSet()
+			for _, bucket := range buckets {
+				diskBuckets.Add(bucket.Name)
+			}
+			sys.RemoveStaleBuckets(diskBuckets)
+
+			for _, bucket := range buckets {
+				err := sys.loadBucketMetadata(ctx, bucket)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+				// Check if there is a spare procs, wait 100ms instead
+				waitForLowIO(runtime.GOMAXPROCS(0), 100*time.Millisecond, currentHTTPIO)
+			}
+
+			t.Reset(bucketMetadataRefresh)
+		}
+	}
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
 	count := 100 // load 100 bucket metadata at a time.
 	for {
 		if len(buckets) < count {
-			sys.concurrentLoad(ctx, buckets, objAPI)
-			return
+			sys.concurrentLoad(ctx, buckets)
+			break
 		}
-		sys.concurrentLoad(ctx, buckets[:count], objAPI)
+		sys.concurrentLoad(ctx, buckets[:count])
 		buckets = buckets[count:]
 	}
+
+	if globalIsDistErasure {
+		go sys.refreshBucketsMetadataLoop(ctx)
+	}
 }
 
 // Reset the state of the BucketMetadataSys.

cmd/bucket-object-lock.go

@@ -156,11 +156,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 				return ErrNone
 			}
 			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
-			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention
-			// or s3:GetBucketObjectLockConfiguration permissions, the operation will succeed.
-			govBypassPerms1 := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
-			govBypassPerms2 := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, object.ObjectName)
-			if govBypassPerms1 != ErrNone && govBypassPerms2 != ErrNone {
+			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
+			if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
 				return ErrAccessDenied
 			}
 		}

cmd/bucket-policy-handlers.go

@@ -86,7 +86,11 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "ErrMalformedXML",
+			HTTPStatusCode: http.StatusBadRequest,
+			Description:    err.Error(),
+		}, r.URL)
 		return
 	}
 
@@ -109,15 +113,12 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Call site replication hook.
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		Policy:    bucketPolicyBytes,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -156,14 +157,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Call site replication hook.
-	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Success.
 	writeSuccessNoContent(w)

cmd/bucket-policy.go

@@ -28,6 +28,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -65,9 +66,20 @@ func NewPolicySys() *PolicySys {
 	return &PolicySys{}
 }
 
-func getConditionValues(r *http.Request, lc string, username string, claims map[string]interface{}) map[string][]string {
+func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
 	currTime := UTCNow()
 
+	var (
+		username = cred.AccessKey
+		claims   = cred.Claims
+		groups   = cred.Groups
+	)
+
+	if cred.IsTemp() || cred.IsServiceAccount() {
+		// For derived credentials, check the parent user's permissions.
+		username = cred.ParentUser
+	}
+
 	principalType := "Anonymous"
 	if username != "" {
 		principalType = "User"
@@ -192,17 +204,11 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	for k, v := range claims {
 		vStr, ok := v.(string)
 		if ok {
-			// Special case for AD/LDAP STS users
-			switch k {
-			case ldapUser:
-				args["user"] = []string{vStr}
-			case ldapUserN:
-				args["username"] = []string{vStr}
-			default:
-				args[k] = []string{vStr}
-			}
+			// Trim any LDAP specific prefix
+			args[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{vStr}
 		}
 	}
+
 	// Add groups claim which could be a list. This will ensure that the claim
 	// `jwt:groups` works.
 	if grpsVal, ok := claims["groups"]; ok {
@@ -219,6 +225,13 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		}
 	}
 
+	// if not claim groups are available use the one with auth.Credentials
+	if _, ok := args["groups"]; !ok {
+		if len(groups) > 0 {
+			args["groups"] = groups
+		}
+	}
+
 	return args
 }
 

cmd/bucket-replication.go

@@ -415,7 +415,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, rcfg.RoleArn))
+		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, dobj.TargetArn))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -938,7 +938,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			Host:       "Internal: [Replication]",
 		})
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, cfg.RoleArn))
+		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, ri.TargetArn))
 		return
 	}
 	ctx = lkctx.Context()
@@ -1162,14 +1162,14 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	if objInfo.isMultipart() {
 		if err := replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
 			r, objInfo, putOpts); err != nil {
-			if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
 				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
 		}
 	} else {
 		if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
-			if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
 				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
@@ -1363,7 +1363,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if objInfo.isMultipart() {
 			if err := replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
 				r, objInfo, putOpts); err != nil {
-				if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
 					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 				} else {
@@ -1372,7 +1372,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			}
 		} else {
 			if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
-				if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
 					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 				} else {
@@ -1535,7 +1535,6 @@ type ReplicationPool struct {
 	// workers:
 	workers         []chan ReplicationWorkerOperation
 	existingWorkers chan ReplicationWorkerOperation
-	workerWg        sync.WaitGroup
 
 	// mrf:
 	mrfWorkerKillCh chan struct{}
@@ -1544,7 +1543,6 @@ type ReplicationPool struct {
 	mrfStopCh       chan struct{}
 	mrfWorkerSize   int
 	saveStateCh     chan struct{}
-	mrfWorkerWg     sync.WaitGroup
 }
 
 // ReplicationWorkerOperation is a shared interface of replication operations.
@@ -1607,7 +1605,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 
 	pool.ResizeWorkers(workers, 0)
 	pool.ResizeFailedWorkers(failedWorkers)
-	pool.workerWg.Add(1)
 	go pool.AddWorker(pool.existingWorkers, nil)
 	go pool.resyncer.PersistToDisk(ctx, o)
 	go pool.processMRF()
@@ -1619,7 +1616,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 // AddMRFWorker adds a pending/failed replication worker to handle requests that could not be queued
 // to the other workers
 func (p *ReplicationPool) AddMRFWorker() {
-	defer p.mrfWorkerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1646,7 +1642,6 @@ func (p *ReplicationPool) AddMRFWorker() {
 // An optional pointer to a tracker that will be atomically
 // incremented when operations are running can be provided.
 func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opTracker *int32) {
-	defer p.workerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1704,7 +1699,6 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 		input := make(chan ReplicationWorkerOperation, 10000)
 		p.workers = append(p.workers, input)
 
-		p.workerWg.Add(1)
 		go p.AddWorker(input, &p.activeWorkers)
 	}
 	for len(p.workers) > n {
@@ -1748,7 +1742,6 @@ func (p *ReplicationPool) ResizeFailedWorkers(n int) {
 
 	for p.mrfWorkerSize < n {
 		p.mrfWorkerSize++
-		p.mrfWorkerWg.Add(1)
 		go p.AddMRFWorker()
 	}
 	for p.mrfWorkerSize > n {
@@ -1918,7 +1911,7 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		return nil, proxy, err
 	}
 	c := minio.Core{Client: tgt.Client}
-	obj, _, h, err := c.GetObject(ctx, bucket, object, gopts)
+	obj, _, h, err := c.GetObject(ctx, tgt.Bucket, object, gopts)
 	if err != nil {
 		return nil, proxy, err
 	}
@@ -1943,7 +1936,7 @@ func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	if opts.VersionSuspended {
 		return &madmin.BucketTargets{}
 	}
-	if !opts.ProxyRequest {
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) {
 		return &madmin.BucketTargets{}
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)

cmd/bucket-targets.go

@@ -98,7 +98,7 @@ func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
 		select {
 		case <-hcTimer.C:
 			sys.hMutex.RLock()
-			var eps []madmin.ServerProperties
+			eps := make([]madmin.ServerProperties, 0, len(sys.hc))
 			for _, ep := range sys.hc {
 				eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme})
 			}
@@ -106,16 +106,12 @@ func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
 
 			if len(eps) > 0 {
 				cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
-				m := map[string]epHealth{}
+				m := make(map[string]epHealth, len(eps))
 				for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
-					var online bool
-					if result.Error == nil {
-						online = result.Online
-					}
 					m[result.Endpoint.Host] = epHealth{
 						Endpoint: result.Endpoint.Host,
 						Scheme:   result.Endpoint.Scheme,
-						Online:   online,
+						Online:   result.Online,
 					}
 				}
 				cancel()
@@ -457,6 +453,8 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if err != nil {
 		return nil, err
 	}
+	api.SetAppInfo("minio-replication-target", ReleaseTag+" "+tcfg.Arn)
+
 	hcDuration := defaultHealthCheckDuration
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
@@ -477,20 +475,23 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 }
 
 // getRemoteARN gets existing ARN for an endpoint or generates a new one.
-func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) (arn string, exists bool) {
+func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget, deplID string) (arn string, exists bool) {
 	if target == nil {
 		return
 	}
 	tgts := sys.targetsMap[bucket]
 	for _, tgt := range tgts {
-		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() && tgt.Credentials.AccessKey == target.Credentials.AccessKey {
+		if tgt.Type == target.Type &&
+			tgt.TargetBucket == target.TargetBucket &&
+			target.URL().String() == tgt.URL().String() &&
+			tgt.Credentials.AccessKey == target.Credentials.AccessKey {
 			return tgt.Arn, true
 		}
 	}
 	if !target.Type.IsValid() {
 		return
 	}
-	return generateARN(target), false
+	return generateARN(target, deplID), false
 }
 
 // getRemoteARNForPeer returns the remote target for a peer site in site replication
@@ -510,10 +511,14 @@ func (sys *BucketTargetSys) getRemoteARNForPeer(bucket string, peer madmin.PeerI
 }
 
 // generate ARN that is unique to this target type
-func generateARN(t *madmin.BucketTarget) string {
+func generateARN(t *madmin.BucketTarget, deplID string) string {
+	uuid := deplID
+	if uuid == "" {
+		uuid = mustGetUUID()
+	}
 	arn := madmin.ARN{
 		Type:   t.Type,
-		ID:     mustGetUUID(),
+		ID:     uuid,
 		Region: t.Region,
 		Bucket: t.TargetBucket,
 	}

cmd/bucket-versioning-handler.go

@@ -108,15 +108,12 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,
 		UpdatedAt:  updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/common-main.go

@@ -51,7 +51,7 @@ import (
 	consoleCerts "github.com/minio/console/pkg/certs"
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -205,7 +205,7 @@ func minioConfigToConsoleFeatures() {
 		}
 	}
 	// Enable if LDAP is enabled.
-	if globalLDAPConfig.Enabled() {
+	if globalIAMSys.LDAPConfig.Enabled() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
@@ -215,8 +215,9 @@ func minioConfigToConsoleFeatures() {
 }
 
 func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
-	m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))
-	for name, cfg := range globalOpenIDConfig.ProviderCfgs {
+	pcfgs := globalIAMSys.OpenIDConfig.ProviderCfgs
+	m := make(map[string]consoleoauth2.ProviderConfig, len(pcfgs))
+	for name, cfg := range pcfgs {
 		callback := getConsoleEndpoints()[0] + "/oauth_callback"
 		if cfg.RedirectURI != "" {
 			callback = cfg.RedirectURI
@@ -600,13 +601,13 @@ func loadEnvVarsFromFiles() {
 		}
 	}
 
-	if env.IsSet(config.EnvKMSSecretKeyFile) {
-		kmsSecret, err := readFromSecret(env.Get(config.EnvKMSSecretKeyFile, ""))
+	if env.IsSet(kms.EnvKMSSecretKeyFile) {
+		kmsSecret, err := readFromSecret(env.Get(kms.EnvKMSSecretKeyFile, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
 		}
 		if kmsSecret != "" {
-			os.Setenv(config.EnvKMSSecretKey, kmsSecret)
+			os.Setenv(kms.EnvKMSSecretKey, kmsSecret)
 		}
 	}
 
@@ -639,7 +640,7 @@ func handleCommonEnvVars() {
 			if !((u.Scheme == "http" || u.Scheme == "https") &&
 				u.Opaque == "" &&
 				!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
-				err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u)
+				err := fmt.Errorf("URL contains unexpected resources, expected URL to be one of http(s)://console.example.com or as a subpath via API endpoint http(s)://minio.example.com/minio format: %v", u)
 				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value is environment variable")
 			}
 			globalBrowserRedirectURL = u
@@ -783,20 +784,29 @@ func handleCommonEnvVars() {
 // It depends on KMS env variables and global cli flags.
 func handleKMSConfig() {
 	switch {
-	case env.IsSet(config.EnvKMSSecretKey) && env.IsSet(config.EnvKESEndpoint):
-		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKMSSecretKey, config.EnvKESEndpoint))
+	case env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint):
+		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
 	}
 
-	if env.IsSet(config.EnvKMSSecretKey) {
-		KMS, err := kms.Parse(env.Get(config.EnvKMSSecretKey, ""))
+	if env.IsSet(kms.EnvKMSSecretKey) {
+		KMS, err := kms.Parse(env.Get(kms.EnvKMSSecretKey, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
 		}
 		GlobalKMS = KMS
 	}
-	if env.IsSet(config.EnvKESEndpoint) {
+	if env.IsSet(kms.EnvKESEndpoint) {
+		if env.IsSet(kms.EnvKESAPIKey) {
+			if env.IsSet(kms.EnvKESClientKey) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientKey))
+			}
+			if env.IsSet(kms.EnvKESClientCert) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientCert))
+			}
+		}
+
 		var endpoints []string
-		for _, endpoint := range strings.Split(env.Get(config.EnvKESEndpoint, ""), ",") {
+		for _, endpoint := range strings.Split(env.Get(kms.EnvKESEndpoint, ""), ",") {
 			if strings.TrimSpace(endpoint) == "" {
 				continue
 			}
@@ -812,67 +822,82 @@ func handleKMSConfig() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
+		rootCAs, err := certs.GetRootCAs(env.Get(kms.EnvKESServerCA, globalCertsCADir.Get()))
 		if err != nil {
-			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
+			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(kms.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
-		loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
-			// Manually load the certificate and private key into memory.
-			// We need to check whether the private key is encrypted, and
-			// if so, decrypt it using the user-provided password.
-			certBytes, err := os.ReadFile(certFile)
+		var kmsConf kms.Config
+		if env.IsSet(kms.EnvKESAPIKey) {
+			key, err := kes.ParseAPIKey(env.Get(kms.EnvKESAPIKey, ""))
 			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				logger.Fatal(err, fmt.Sprintf("Failed to parse KES API key from %q", env.Get(kms.EnvKESAPIKey, "")))
 			}
-			keyBytes, err := os.ReadFile(keyFile)
-			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+			kmsConf = kms.Config{
+				Endpoints:    endpoints,
+				Enclave:      env.Get(kms.EnvKESEnclave, ""),
+				DefaultKeyID: env.Get(kms.EnvKESKeyName, ""),
+				APIKey:       key,
+				RootCAs:      rootCAs,
 			}
-			privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
-			if len(rest) != 0 {
-				return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
-			}
-			if x509.IsEncryptedPEMBlock(privateKeyPEM) {
-				keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+		} else {
+			loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
+				// Manually load the certificate and private key into memory.
+				// We need to check whether the private key is encrypted, and
+				// if so, decrypt it using the user-provided password.
+				certBytes, err := os.ReadFile(certFile)
 				if err != nil {
-					return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
 				}
-				keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+				keyBytes, err := os.ReadFile(keyFile)
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+				}
+				privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+				if len(rest) != 0 {
+					return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
+				}
+				if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+					keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(kms.EnvKESClientPassword, "")))
+					if err != nil {
+						return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+					}
+					keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+				}
+				certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				}
+				return certificate, nil
 			}
-			certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+
+			reloadCertEvents := make(chan tls.Certificate, 1)
+			certificate, err := certs.NewCertificate(env.Get(kms.EnvKESClientCert, ""), env.Get(kms.EnvKESClientKey, ""), loadX509KeyPair)
 			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				logger.Fatal(err, "Failed to load KES client certificate")
+			}
+			certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
+			certificate.Notify(reloadCertEvents)
+
+			kmsConf = kms.Config{
+				Endpoints:        endpoints,
+				Enclave:          env.Get(kms.EnvKESEnclave, ""),
+				DefaultKeyID:     env.Get(kms.EnvKESKeyName, ""),
+				Certificate:      certificate,
+				ReloadCertEvents: reloadCertEvents,
+				RootCAs:          rootCAs,
 			}
-			return certificate, nil
 		}
 
-		reloadCertEvents := make(chan tls.Certificate, 1)
-		certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
-		if err != nil {
-			logger.Fatal(err, "Failed to load KES client certificate")
-		}
-		certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
-		certificate.Notify(reloadCertEvents)
-
-		defaultKeyID := env.Get(config.EnvKESKeyName, "")
-		KMS, err := kms.NewWithConfig(kms.Config{
-			Endpoints:        endpoints,
-			Enclave:          env.Get(config.EnvKESEnclave, ""),
-			DefaultKeyID:     defaultKeyID,
-			Certificate:      certificate,
-			ReloadCertEvents: reloadCertEvents,
-			RootCAs:          rootCAs,
-		})
+		KMS, err := kms.NewWithConfig(kmsConf)
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
-
 		// We check that the default key ID exists or try to create it otherwise.
 		// This implicitly checks that we can communicate to KES. We don't treat
 		// a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(context.Background(), defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+		if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
 		GlobalKMS = KMS

cmd/config-current.go

@@ -493,15 +493,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
 	}
 
-	globalSTSTLSConfig, err = xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
-	}
-
-	if globalSTSTLSConfig.InsecureSkipVerify {
-		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
-	}
-
 	transport := NewHTTPTransport()
 
 	bootstrapTrace("lookup the event notification targets")
@@ -701,37 +692,42 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 		h = config.HelpKVS{value}
 	}
 
-	envHelp := config.HelpKVS{}
-	if envOnly {
-		// Only for multiple targets, make sure
-		// to list the ENV, for regular k/v EnableKey is
-		// implicit, for ENVs we cannot make it implicit.
-		if subSysHelp.MultipleTargets {
-			envK := config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(madmin.EnableKey)
-			envHelp = append(envHelp, config.HelpKV{
-				Key:         envK,
-				Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
-				Optional:    false,
-				Type:        "on|off",
-			})
+	help := config.HelpKVS{}
+
+	// Only for multiple targets, make sure
+	// to list the ENV, for regular k/v EnableKey is
+	// implicit, for ENVs we cannot make it implicit.
+	if subSysHelp.MultipleTargets {
+		key := madmin.EnableKey
+		if envOnly {
+			key = config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(madmin.EnableKey)
 		}
-		for _, hkv := range h {
-			envK := config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(hkv.Key)
-			envHelp = append(envHelp, config.HelpKV{
-				Key:         envK,
-				Description: hkv.Description,
-				Optional:    hkv.Optional,
-				Type:        hkv.Type,
-			})
+		help = append(help, config.HelpKV{
+			Key:         key,
+			Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
+			Optional:    false,
+			Type:        "on|off",
+		})
+	}
+
+	for _, hkv := range h {
+		key := hkv.Key
+		if envOnly {
+			key = config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(hkv.Key)
 		}
-		h = envHelp
+		help = append(help, config.HelpKV{
+			Key:         key,
+			Description: hkv.Description,
+			Optional:    hkv.Optional,
+			Type:        hkv.Type,
+		})
 	}
 
 	return Help{
 		SubSys:          subSys,
 		Description:     subSysHelp.Description,
 		MultipleTargets: subSysHelp.MultipleTargets,
-		KeysHelp:        h,
+		KeysHelp:        help,
 	}, nil
 }
 

cmd/config-migrate.go

@@ -46,7 +46,7 @@ import (
 	"github.com/minio/pkg/quick"
 )
 
-// DO NOT EDIT following message template, please open a GitHub issue to discuss instead.
+// Do not edit following message template, please open a GitHub issue to discuss instead.
 var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully."
 
 // Save config file to corresponding backend
@@ -2515,20 +2515,13 @@ func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (
 			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 				minioMetaBucket: path.Join(minioMetaBucket, configFile),
 			})
-			if err != nil {
-				data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-				if err != nil {
-					if err == madmin.ErrMaliciousData {
-						return false, nil, config.ErrInvalidCredentialsBackendEncrypted(nil)
-					}
-					return false, nil, err
-				}
-			}
-		} else {
+		}
+
+		if GlobalKMS == nil && err != nil {
 			data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 			if err != nil {
-				if err == madmin.ErrMaliciousData {
-					return false, nil, config.ErrInvalidCredentialsBackendEncrypted(nil)
+				if err == madmin.ErrMaliciousData || err == madmin.ErrUnexpectedHeader {
+					return false, nil, config.ErrInvalidConfigDecryptionKey(nil)
 				}
 				return false, nil, err
 			}

cmd/consolelogger.go

@@ -25,10 +25,10 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/logger/target/console"
 	"github.com/minio/minio/internal/logger/target/types"
 	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/pkg/logger/message/log"
 	xnet "github.com/minio/pkg/net"
 )
 

cmd/data-scanner-metric.go

@@ -69,9 +69,9 @@ const (
 
 // log scanner action.
 // Use for s > scannerMetricStartTrace
-func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
+func (p *scannerMetrics) log(s scannerMetric, paths ...string) func(custom map[string]string) {
 	startTime := time.Now()
-	return func() {
+	return func(custom map[string]string) {
 		duration := time.Since(startTime)
 
 		atomic.AddUint64(&p.operations[s], 1)
@@ -80,7 +80,7 @@ func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
 		}
 
 		if s > scannerMetricStartTrace && globalTrace.NumSubscribers(madmin.TraceScanner) > 0 {
-			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " ")))
+			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " "), custom))
 		}
 	}
 }

cmd/data-scanner.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/binary"
 	"encoding/json"
@@ -29,11 +28,11 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/bits-and-blooms/bloom/v3"
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -59,6 +58,9 @@ const (
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
 	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
+
+	dataScannerExcessiveVersionsThreshold = 1000  // Issue a warning when a single object has more versions than this
+	dataScannerExcessiveFoldersThreshold  = 50000 // Issue a warning when a folder has more subfolders than this in a *set*
 )
 
 var (
@@ -158,7 +160,6 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 
 	// Load current bloom cycle
 	var cycleInfo currentScannerCycle
-	cycleInfo.next = intDataUpdateTracker.current() + 1
 
 	buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
 	if len(buf) == 8 {
@@ -203,11 +204,13 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
+			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
-			logger.LogIf(ctx, err)
-			stopFn()
+			res := map[string]string{"cycle": fmt.Sprint(cycleInfo.current)}
+			if err != nil {
+				res["error"] = err.Error()
+			}
+			stopFn(res)
 			if err == nil {
 				// Store new cycle...
 				cycleInfo.next++
@@ -240,10 +243,8 @@ type folderScanner struct {
 	oldCache    dataUsageCache
 	newCache    dataUsageCache
 	updateCache dataUsageCache
-	withFilter  *bloomFilter
 
 	dataUsageScannerDebug bool
-	healFolderInclude     uint32 // Include a clean folder one in n cycles.
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 	scanMode              madmin.HealScanMode
 
@@ -302,8 +303,6 @@ type folderScanner struct {
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
 func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
-	logPrefix := color.Green("data-usage: ")
-
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
@@ -317,8 +316,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		oldCache:              cache,
 		newCache:              dataUsageCache{Info: cache.Info},
 		updateCache:           dataUsageCache{Info: cache.Info},
-		dataUsageScannerDebug: intDataUpdateTracker.debug,
-		healFolderInclude:     0,
+		dataUsageScannerDebug: false,
 		healObjectSelect:      0,
 		scanMode:              scanMode,
 		updates:               cache.Info.updates,
@@ -341,19 +339,9 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 
 	// Enable healing in XL mode.
 	if globalIsErasure && !cache.Info.SkipHealing {
-		// Include a clean folder one in n cycles.
-		s.healFolderInclude = healFolderIncludeProb
 		// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 		s.healObjectSelect = healObjectSelectProb
 	}
-	if len(cache.Info.BloomFilter) > 0 {
-		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-		_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
-		if err != nil {
-			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
-			s.withFilter = nil
-		}
-	}
 
 	done := ctx.Done()
 
@@ -410,14 +398,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			return ctx.Err()
 		default:
 		}
-		existing, ok := f.oldCache.Cache[thisHash.Key()]
 		var abandonedChildren dataUsageHashMap
 		if !into.Compacted {
 			abandonedChildren = f.oldCache.findChildrenCopy(thisHash)
 		}
 
 		// If there are lifecycle rules for the prefix, remove the filter.
-		filter := f.withFilter
 		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
 		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
@@ -425,33 +411,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
 			activeLifeCycle = f.oldCache.Info.lifeCycle
-			filter = nil
 		}
 		// If there are replication rules for the prefix, remove the filter.
 		var replicationCfg replicationConfig
 		if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
 			replicationCfg = f.oldCache.Info.replication
-			filter = nil
 		}
 		// Check if we can skip it due to bloom filter...
-		if filter != nil && ok && existing.Compacted {
-			// If folder isn't in filter and we have data, skip it completely.
-			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
-					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-					if f.dataUsageScannerDebug {
-						console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
-					}
-					return nil
-				}
-				if f.dataUsageScannerDebug {
-					console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
-				}
-				// If probability was already scannerHealFolderInclude, keep it.
-				folder.objectHealProbDiv = f.healFolderInclude
-			}
-		}
 		scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
 
 		var existingFolders, newFolders []cachedFolder
@@ -566,6 +532,19 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
 			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
 
+		if len(existingFolders)+len(newFolders) > dataScannerExcessiveFoldersThreshold {
+			// Notify object accessed via a GET request.
+			sendEvent(eventArgs{
+				EventName:  event.PrefixManyFolders,
+				BucketName: f.root,
+				Object: ObjectInfo{
+					Name: strings.TrimSuffix(folder.name, "/") + "/",
+					Size: int64(len(existingFolders) + len(newFolders)),
+				},
+				UserAgent: "scanner",
+				Host:      globalMinioHost,
+			})
+		}
 		if !into.Compacted && shouldCompact {
 			into.Compacted = true
 			newFolders = append(newFolders, existingFolders...)
@@ -632,7 +611,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			f.updateCurrentPath(folder.name)
 			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
-			stopFn()
+			stopFn(map[string]string{"type": "new"})
+
 			// Add new folders if this is new and we don't have existing.
 			if !into.Compacted {
 				parent := f.updateCache.find(thisHash.Key())
@@ -651,19 +631,16 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
-						// Transfer and add as child...
-						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
-						into.addChild(h)
-						continue
-					}
-					folder.objectHealProbDiv = f.healFolderInclude
+					// Transfer and add as child...
+					f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
+					into.addChild(h)
+					continue
 				}
 			}
 			f.updateCurrentPath(folder.name)
-			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name, "EXISTING")
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
-			stopFn()
+			stopFn(map[string]string{"type": "existing"})
 		}
 
 		// Scan for healing
@@ -803,7 +780,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
 				stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, this.name, "HEALED")
 				scanFolder(this)
-				stopFn()
+				stopFn(map[string]string{"type": "healed"})
 			}
 		}
 		break
@@ -817,9 +794,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		flat.Compacted = true
 		var compact bool
 		if flat.Objects < dataScannerCompactLeastObject {
-			if flat.Objects > 1 {
-				globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
-			}
 			compact = true
 		} else {
 			// Compact if we only have objects as children...
@@ -832,13 +806,20 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 				}
 			}
-			if compact {
-				globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
-			}
+
 		}
 		if compact {
+			stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
 			f.newCache.deleteRecursive(thisHash)
 			f.newCache.replaceHashed(thisHash, folder.parent, *flat)
+			total := map[string]string{
+				"objects": fmt.Sprint(flat.Objects),
+				"size":    fmt.Sprint(flat.Size),
+			}
+			if flat.Versions > 0 {
+				total["versions"] = fmt.Sprint(flat.Versions)
+			}
+			stop(total)
 		}
 
 	}
@@ -1082,8 +1063,27 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			}
 		}
 	}
+	fivs, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+	if err != nil {
+		return fivs, err
+	}
 
-	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
+	if len(fivs) > dataScannerExcessiveVersionsThreshold {
+		// Notify object accessed via a GET request.
+		sendEvent(eventArgs{
+			EventName:  event.ObjectManyVersions,
+			BucketName: i.bucket,
+			Object: ObjectInfo{
+				Name: i.objectPath(),
+			},
+			UserAgent:    "scanner",
+			Host:         globalMinioHost,
+			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(fivs))},
+		})
+	}
+
+	return fivs, nil
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1203,7 +1203,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		EventName:  eventName,
 		BucketName: obj.Bucket,
 		Object:     obj,
-		Host:       "Internal: [ILM-EXPIRY]",
+		Host:       "Internal: [ILM-Expiry]",
 	})
 
 	return true

cmd/data-update-tracker.go

@@ -1,682 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/binary"
-	"errors"
-	"io"
-	"os"
-	"path"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/bits-and-blooms/bloom/v3"
-	"github.com/minio/minio/internal/color"
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/console"
-)
-
-const (
-	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 200000
-	// ... we want this false positive rate:
-	dataUpdateTrackerFP        = 0.1
-	dataUpdateTrackerQueueSize = 0
-
-	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 7
-	dataUpdateTrackerSaveInterval = 5 * time.Minute
-)
-
-var intDataUpdateTracker *dataUpdateTracker
-
-func init() {
-	intDataUpdateTracker = newDataUpdateTracker()
-}
-
-type dataUpdateTracker struct {
-	mu         sync.Mutex
-	input      chan string
-	save       chan struct{}
-	debug      bool
-	saveExited chan struct{}
-	dirty      bool
-
-	Current dataUpdateFilter
-	History dataUpdateTrackerHistory
-	Saved   time.Time
-}
-
-// newDataUpdateTracker returns a dataUpdateTracker with default settings.
-func newDataUpdateTracker() *dataUpdateTracker {
-	d := &dataUpdateTracker{
-		Current: dataUpdateFilter{
-			idx: 1,
-		},
-		debug:      serverDebugLog,
-		input:      make(chan string, dataUpdateTrackerQueueSize),
-		save:       make(chan struct{}, 1),
-		saveExited: make(chan struct{}),
-	}
-	d.Current.bf = d.newBloomFilter()
-	d.dirty = true
-	return d
-}
-
-type dataUpdateTrackerHistory []dataUpdateFilter
-
-type dataUpdateFilter struct {
-	idx uint64
-	bf  bloomFilter
-}
-
-type bloomFilter struct {
-	*bloom.BloomFilter
-}
-
-// emptyBloomFilter returns an empty bloom filter.
-func emptyBloomFilter() bloomFilter {
-	return bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-}
-
-// containsDir returns whether the bloom filter contains a directory.
-// Note that objects in XL mode are also considered directories.
-func (b bloomFilter) containsDir(in string) bool {
-	split := splitPathDeterministic(path.Clean(in))
-
-	if len(split) == 0 {
-		return false
-	}
-	return b.TestString(hashPath(path.Join(split...)).String())
-}
-
-// bytes returns the bloom filter serialized as a byte slice.
-func (b *bloomFilter) bytes() []byte {
-	if b == nil || b.BloomFilter == nil {
-		return nil
-	}
-	var buf bytes.Buffer
-	_, err := b.WriteTo(&buf)
-	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
-	}
-	return buf.Bytes()
-}
-
-// sort the dataUpdateTrackerHistory, newest first.
-// Returns whether the history is complete.
-func (d dataUpdateTrackerHistory) sort() bool {
-	if len(d) == 0 {
-		return true
-	}
-	sort.Slice(d, func(i, j int) bool {
-		return d[i].idx > d[j].idx
-	})
-	return d[0].idx-d[len(d)-1].idx == uint64(len(d))
-}
-
-// removeOlderThan will remove entries older than index 'n'.
-func (d *dataUpdateTrackerHistory) removeOlderThan(n uint64) {
-	d.sort()
-	dd := *d
-	end := len(dd)
-	for i := end - 1; i >= 0; i-- {
-		if dd[i].idx < n {
-			end = i
-		}
-	}
-	dd = dd[:end]
-	*d = dd
-}
-
-// newBloomFilter returns a new bloom filter with default settings.
-func (d *dataUpdateTracker) newBloomFilter() bloomFilter {
-	return bloomFilter{bloom.NewWithEstimates(dataUpdateTrackerEstItems, dataUpdateTrackerFP)}
-}
-
-// current returns the current index.
-func (d *dataUpdateTracker) current() uint64 {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	return d.Current.idx
-}
-
-// latestWithDir returns the highest index that contains the directory.
-// This means that any cycle higher than this does NOT contain the entry.
-func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
-	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
-	bucket, _ := path2BucketObjectWithBasePath("", dir)
-	if bucket == "" {
-		if d.debug && len(dir) > 0 {
-			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
-		}
-		return d.current()
-	}
-	if isReservedOrInvalidBucket(bucket, false) {
-		return d.current()
-	}
-
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
-		return d.Current.idx
-	}
-	if d.debug {
-		console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
-	}
-
-	idx := d.Current.idx - 1
-	for {
-		f := d.History.find(idx)
-		if f == nil || f.bf.containsDir(dir) || idx == 0 {
-			break
-		}
-		idx--
-	}
-	return idx
-}
-
-// start will load the current data from the drives start collecting information and
-// start a saver goroutine.
-// All of these will exit when the context is canceled.
-func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
-	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No local drives specified"))
-		return
-	}
-	d.load(ctx, drives...)
-	go d.startCollector(ctx)
-	// startSaver will unlock.
-	d.mu.Lock()
-	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives...)
-}
-
-// load will attempt to load data tracking information from the supplied drives.
-// The data will only be loaded if d.Saved is older than the one found on disk.
-// The newest working cache will be kept in d.
-// If no valid data usage tracker can be found d will remain unchanged.
-// If object is shared the caller should lock it.
-func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
-	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No local drives specified"))
-		return
-	}
-	for _, drive := range drives {
-
-		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-		f, err := OpenFile(cacheFormatPath, readMode, 0o666)
-		if err != nil {
-			if osIsNotExist(err) {
-				continue
-			}
-			logger.LogIf(ctx, err)
-			continue
-		}
-		err = d.deserialize(f, d.Saved)
-		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-			logger.LogIf(ctx, err)
-		}
-		f.Close()
-	}
-}
-
-// startSaver will start a saver that will write d to all supplied drives at specific intervals.
-// 'd' must be write locked when started and will be unlocked.
-// The saver will save and exit when supplied context is closed.
-func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives ...string) {
-	if len(drives) == 0 {
-		return
-	}
-
-	saveNow := d.save
-	exited := make(chan struct{})
-	d.saveExited = exited
-	d.mu.Unlock()
-	t := time.NewTicker(interval)
-	defer t.Stop()
-	defer close(exited)
-	var buf bytes.Buffer
-	for {
-		var exit bool
-		select {
-		case <-ctx.Done():
-			exit = true
-		case <-t.C:
-		case <-saveNow:
-		}
-		buf.Reset()
-		d.mu.Lock()
-		if !d.dirty {
-			d.mu.Unlock()
-			if exit {
-				return
-			}
-			continue
-		}
-		d.Saved = UTCNow()
-		err := d.serialize(&buf)
-		if d.debug {
-			console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
-		}
-		d.dirty = false
-		d.mu.Unlock()
-		if err != nil {
-			logger.LogIf(ctx, err, "Error serializing usage tracker data")
-			if exit {
-				return
-			}
-			continue
-		}
-		if buf.Len() == 0 {
-			logger.LogIf(ctx, errors.New("zero sized output, skipping save"))
-			continue
-		}
-		for _, drive := range drives {
-			cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-			err := os.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
-			if err != nil {
-				if osIsNotExist(err) {
-					continue
-				}
-				logger.LogIf(ctx, err)
-				continue
-			}
-		}
-		if exit {
-			return
-		}
-	}
-}
-
-// serialize all data in d to dst.
-// Caller should hold lock if d is expected to be shared.
-// If an error is returned, there will likely be partial data written to dst.
-func (d *dataUpdateTracker) serialize(dst io.Writer) (err error) {
-	ctx := GlobalContext
-	var tmp [8]byte
-	o := bufio.NewWriter(dst)
-	defer func() {
-		if err == nil {
-			err = o.Flush()
-		}
-	}()
-
-	// Version
-	if err := o.WriteByte(dataUpdateTrackerVersion); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	// Timestamp.
-	binary.LittleEndian.PutUint64(tmp[:], uint64(d.Saved.Unix()))
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// Current
-	binary.LittleEndian.PutUint64(tmp[:], d.Current.idx)
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	if _, err := d.Current.bf.WriteTo(o); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// History
-	binary.LittleEndian.PutUint64(tmp[:], uint64(len(d.History)))
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	for _, bf := range d.History {
-		// Current
-		binary.LittleEndian.PutUint64(tmp[:], bf.idx)
-		if _, err := o.Write(tmp[:]); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-
-		if _, err := bf.bf.WriteTo(o); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-	}
-	return nil
-}
-
-// deserialize will deserialize the supplied input if the input is newer than the supplied time.
-func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) error {
-	ctx := GlobalContext
-	var dst dataUpdateTracker
-	var tmp [8]byte
-
-	// Version
-	if _, err := io.ReadFull(src, tmp[:1]); err != nil {
-		if d.debug {
-			if err != io.EOF {
-				logger.LogIf(ctx, err)
-			}
-		}
-		return err
-	}
-	switch tmp[0] {
-	case 1, 2, 3, 4, 5, 6:
-		if intDataUpdateTracker.debug {
-			console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
-		}
-		return nil
-	case dataUpdateTrackerVersion:
-	default:
-		return errors.New("dataUpdateTracker: Unknown data version")
-	}
-	// Timestamp.
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	t := time.Unix(int64(binary.LittleEndian.Uint64(tmp[:])), 0)
-	if !t.After(newerThan) {
-		return nil
-	}
-
-	// Current
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	dst.Current.idx = binary.LittleEndian.Uint64(tmp[:])
-	dst.Current.bf = emptyBloomFilter()
-	if _, err := dst.Current.bf.ReadFrom(src); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// History
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	n := binary.LittleEndian.Uint64(tmp[:])
-	dst.History = make(dataUpdateTrackerHistory, int(n))
-	for i, e := range dst.History {
-		if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-		e.idx = binary.LittleEndian.Uint64(tmp[:])
-		e.bf = emptyBloomFilter()
-		if _, err := e.bf.ReadFrom(src); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-		dst.History[i] = e
-	}
-	// Ignore what remains on the stream.
-	// Update d:
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	d.Current = dst.Current
-	d.History = dst.History
-	d.Saved = dst.Saved
-	return nil
-}
-
-// start a collector that picks up entries from objectUpdatedCh
-// and adds them  to the current bloom filter.
-func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case in := <-d.input:
-			bucket, _ := path2BucketObjectWithBasePath("", in)
-			if bucket == "" {
-				if d.debug && len(in) > 0 {
-					console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
-				}
-				continue
-			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				continue
-			}
-			split := splitPathDeterministic(in)
-
-			// Add all paths until done.
-			d.mu.Lock()
-			for i := range split {
-				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-			}
-			d.dirty = d.dirty || len(split) > 0
-			d.mu.Unlock()
-		}
-	}
-}
-
-// markDirty adds the supplied path to the current bloom filter.
-func (d *dataUpdateTracker) markDirty(bucket, prefix string) {
-	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
-	if bucket == "" && d.debug {
-		console.Debugf(dateUpdateTrackerLogPrefix + " no bucket specified\n")
-		return
-	}
-
-	if isReservedOrInvalidBucket(bucket, false) && d.debug {
-		return
-	}
-	split := splitPathDeterministic(pathJoin(bucket, prefix))
-
-	// Add all paths until done.
-	d.mu.Lock()
-	for i := range split {
-		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-	}
-	d.dirty = d.dirty || len(split) > 0
-	d.mu.Unlock()
-}
-
-// find entry with specified index.
-// Returns nil if not found.
-func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
-	for _, f := range d {
-		if f.idx == idx {
-			return &f
-		}
-	}
-	return nil
-}
-
-// filterFrom will return a combined bloom filter.
-func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint64) *bloomFilterResponse {
-	bf := d.newBloomFilter()
-	bfr := bloomFilterResponse{
-		OldestIdx:  oldest,
-		CurrentIdx: d.Current.idx,
-		Complete:   true,
-	}
-	// Loop through each index requested.
-	for idx := oldest; idx <= newest; idx++ {
-		v := d.History.find(idx)
-		if v == nil {
-			if d.Current.idx == idx {
-				// Merge current.
-				err := bf.Merge(d.Current.bf.BloomFilter)
-				logger.LogIf(ctx, err)
-				if err != nil {
-					bfr.Complete = false
-				}
-				continue
-			}
-			bfr.Complete = false
-			bfr.OldestIdx = idx + 1
-			continue
-		}
-
-		err := bf.Merge(v.bf.BloomFilter)
-		if err != nil {
-			bfr.Complete = false
-			logger.LogIf(ctx, err)
-			continue
-		}
-		bfr.NewestIdx = idx
-	}
-	var dst bytes.Buffer
-	_, err := bf.WriteTo(&dst)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil
-	}
-	bfr.Filter = dst.Bytes()
-
-	return &bfr
-}
-
-// cycleFilter will cycle the bloom filter to start recording to index y if not already.
-// The response will contain a bloom filter starting at index x up to, but not including index y.
-// If y is 0, the response will not update y, but return the currently recorded information
-// from the oldest (unless 0, then it will be all) until and including current y.
-func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
-	if req.OldestClean != "" {
-		return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
-	}
-	current := req.Current
-	oldest := req.Oldest
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if current == 0 {
-		if len(d.History) == 0 {
-			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
-		}
-		d.History.sort()
-		if oldest == 0 {
-			oldest = d.History[len(d.History)-1].idx
-		}
-		return d.filterFrom(ctx, oldest, d.Current.idx), nil
-	}
-
-	// Move current to history if new one requested
-	if d.Current.idx != current {
-		d.dirty = true
-		if d.debug {
-			console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
-		}
-
-		d.History = append(d.History, d.Current)
-		d.Current.idx = current
-		d.Current.bf = d.newBloomFilter()
-		select {
-		case d.save <- struct{}{}:
-		default:
-		}
-	}
-	d.History.removeOlderThan(oldest)
-	return d.filterFrom(ctx, oldest, current), nil
-}
-
-// splitPathDeterministic will split the provided relative path
-// deterministically and return up to the first 3 elements of the path.
-// slash and dot prefixes are removed.
-// Trailing slashes are removed.
-// Returns 0 length if no parts are found after trimming.
-func splitPathDeterministic(in string) []string {
-	split := strings.Split(decodeDirObject(in), SlashSeparator)
-
-	// Trim empty start/end
-	for len(split) > 0 {
-		if len(split[0]) > 0 && split[0] != "." {
-			break
-		}
-		split = split[1:]
-	}
-	for len(split) > 0 {
-		if len(split[len(split)-1]) > 0 {
-			break
-		}
-		split = split[:len(split)-1]
-	}
-
-	return split
-}
-
-// bloomFilterRequest request bloom filters.
-// Current index will be updated to current and entries back to Oldest is returned.
-type bloomFilterRequest struct {
-	Oldest  uint64
-	Current uint64
-	// If set the oldest clean version will be returned in OldestIdx
-	// and the rest of the request will be ignored.
-	OldestClean string
-}
-
-type bloomFilterResponse struct {
-	// Current index being written to.
-	CurrentIdx uint64
-	// Oldest index in the returned bloom filter.
-	OldestIdx uint64
-	// Newest Index in the returned bloom filter.
-	NewestIdx uint64
-	// Are all indexes between oldest and newest filled?
-	Complete bool
-	// Binary data of the bloom filter.
-	Filter []byte
-}
-
-// NSUpdated indicates namespace has been updated.
-// The function will block until the entry has been picked up.
-func NSUpdated(bucket, prefix string) {
-	if intDataUpdateTracker != nil {
-		intDataUpdateTracker.markDirty(bucket, prefix)
-	}
-}

cmd/data-update-tracker_test.go

@@ -1,299 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"math/rand"
-	"os"
-	"path"
-	"path/filepath"
-	"sync"
-	"testing"
-
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
-	"github.com/minio/minio/internal/logger/target/types"
-)
-
-type testLoggerI interface {
-	Helper()
-	Log(args ...interface{})
-}
-
-type testingLogger struct {
-	mu sync.Mutex
-	t  testLoggerI
-}
-
-func (t *testingLogger) Endpoint() string {
-	return ""
-}
-
-func (t *testingLogger) String() string {
-	return ""
-}
-
-func (t *testingLogger) Init() error {
-	return nil
-}
-
-func (t *testingLogger) Cancel() {
-}
-
-func (t *testingLogger) Type() types.TargetType {
-	return types.TargetHTTP
-}
-
-func (t *testingLogger) IsOnline() bool {
-	return true
-}
-
-// Stats returns the target statistics.
-func (t *testingLogger) Stats() types.TargetStats {
-	return types.TargetStats{}
-}
-
-func (t *testingLogger) Send(entry interface{}) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.t == nil {
-		return nil
-	}
-	e, ok := entry.(log.Entry)
-	if !ok {
-		return fmt.Errorf("unexpected log entry structure %#v", entry)
-	}
-
-	t.t.Helper()
-	t.t.Log(e.Level, ":", e.Message)
-	return nil
-}
-
-func addTestingLogging(t testLoggerI) func() {
-	tl := &testingLogger{t: t}
-	logger.AddSystemTarget(tl)
-	return func() {
-		tl.mu.Lock()
-		defer tl.mu.Unlock()
-		tl.t = nil
-	}
-}
-
-func TestDataUpdateTracker(t *testing.T) {
-	dut := newDataUpdateTracker()
-	// Change some defaults.
-	dut.debug = testing.Verbose()
-	dut.input = make(chan string)
-	dut.save = make(chan struct{})
-
-	defer addTestingLogging(t)()
-
-	dut.Current.bf = dut.newBloomFilter()
-
-	tmpDir := t.TempDir()
-	err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	dut.start(ctx, tmpDir)
-
-	tests := []struct {
-		in    string
-		check []string // if not empty, check against these instead.
-		exist bool
-	}{
-		{
-			in:    "bucket/directory/file.txt",
-			check: []string{"bucket", "bucket/", "/bucket", "bucket/directory", "bucket/directory/", "bucket/directory/file.txt", "/bucket/directory/file.txt"},
-			exist: true,
-		},
-		{
-			// System bucket
-			in:    ".minio.sys/ignoreme/pls",
-			exist: false,
-		},
-		{
-			// Not a valid bucket
-			in:    "./bucket/okfile.txt",
-			check: []string{"./bucket/okfile.txt", "/bucket/okfile.txt", "bucket/okfile.txt"},
-			exist: false,
-		},
-		{
-			// Not a valid bucket
-			in:    "æ/okfile.txt",
-			check: []string{"æ/okfile.txt", "æ/okfile.txt", "æ"},
-			exist: false,
-		},
-		{
-			in:    "/bucket2/okfile2.txt",
-			check: []string{"./bucket2/okfile2.txt", "/bucket2/okfile2.txt", "bucket2/okfile2.txt", "bucket2"},
-			exist: true,
-		},
-		{
-			in:    "/bucket3/prefix/okfile2.txt",
-			check: []string{"./bucket3/prefix/okfile2.txt", "/bucket3/prefix/okfile2.txt", "bucket3/prefix/okfile2.txt", "bucket3/prefix", "bucket3"},
-			exist: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.in, func(t *testing.T) {
-			dut.input <- tt.in
-			dut.input <- "" // Sending empty string ensures the previous is added to filter.
-			dut.mu.Lock()
-			defer dut.mu.Unlock()
-			if len(tt.check) == 0 {
-				got := dut.Current.bf.containsDir(tt.in)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
-				}
-				return
-			}
-			for _, check := range tt.check {
-				got := dut.Current.bf.containsDir(check)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
-				}
-				continue
-			}
-		})
-	}
-	// Cycle to history
-	req := bloomFilterRequest{
-		Oldest:  1,
-		Current: 2,
-	}
-
-	_, err = dut.cycleFilter(ctx, req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	dut.input <- "cycle2/file.txt"
-	dut.input <- "" // Sending empty string ensures the previous is added to filter.
-
-	tests = append(tests, struct {
-		in    string
-		check []string
-		exist bool
-	}{in: "cycle2/file.txt", exist: true})
-
-	// Shut down
-	cancel()
-	<-dut.saveExited
-
-	if dut.current() != 2 {
-		t.Fatal("wrong current idx after save. want 2, got:", dut.current())
-	}
-
-	ctx, cancel = context.WithCancel(context.Background())
-	// Reload...
-	dut = newDataUpdateTracker()
-	dut.start(ctx, tmpDir)
-	defer func() {
-		cancel()
-		<-dut.saveExited
-	}()
-
-	if dut.current() != 2 {
-		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
-	}
-	req = bloomFilterRequest{
-		Oldest:  1,
-		Current: 3,
-	}
-	bfr2, err := dut.cycleFilter(ctx, req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bfr2.Complete {
-		t.Fatal("Wanted complete, didn't get it")
-	}
-	if bfr2.CurrentIdx != 3 {
-		t.Fatal("wanted index 3, got", bfr2.CurrentIdx)
-	}
-	if bfr2.OldestIdx != 1 {
-		t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
-	}
-
-	t.Logf("Size of filter %d bytes, M: %d, K:%d", len(bfr2.Filter), dut.Current.bf.Cap(), dut.Current.bf.K())
-	// Rerun test with returned bfr2
-	bf := dut.newBloomFilter()
-	_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, tt := range tests {
-		t.Run(tt.in+"-reloaded", func(t *testing.T) {
-			if len(tt.check) == 0 {
-				got := bf.containsDir(tt.in)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
-				}
-				return
-			}
-			for _, check := range tt.check {
-				got := bf.containsDir(check)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
-				}
-				continue
-			}
-		})
-	}
-}
-
-func BenchmarkDataUpdateTracker(b *testing.B) {
-	dut := newDataUpdateTracker()
-	// Change some defaults.
-	dut.debug = false
-	dut.input = make(chan string)
-	dut.save = make(chan struct{})
-
-	defer addTestingLogging(b)()
-
-	dut.Current.bf = dut.newBloomFilter()
-	// We do this unbuffered. This will very significantly reduce throughput, so this is a worst case.
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go dut.startCollector(ctx)
-	input := make([]string, 1000)
-	rng := rand.New(rand.NewSource(0xabad1dea))
-	tmp := []string{"bucket", "aprefix", "nextprefixlevel", "maybeobjname", "evendeeper", "ok-one-morelevel", "final.object"}
-	for i := range input {
-		tmp := tmp[:1+rng.Intn(cap(tmp)-1)]
-		input[i] = path.Join(tmp...)
-	}
-	b.SetBytes(1)
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		dut.input <- input[rng.Intn(len(input))]
-	}
-}

cmd/data-usage-cache.go

@@ -276,7 +276,6 @@ type dataUsageCacheInfo struct {
 	// indicates if the disk is being healed and scanner
 	// should skip healing the disk
 	SkipHealing bool
-	BloomFilter []byte `msg:"BloomFilter,omitempty"`
 
 	// Active lifecycle, if any on the bucket
 	lifeCycle *lifecycle.Lifecycle `msg:"-"`
@@ -649,10 +648,7 @@ func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compact
 // StringAll returns a detailed string representation of all entries in the cache.
 func (d *dataUsageCache) StringAll() string {
 	// Remove bloom filter from print.
-	bf := d.Info.BloomFilter
-	d.Info.BloomFilter = nil
 	s := fmt.Sprintf("info:%+v\n", d.Info)
-	d.Info.BloomFilter = bf
 	for k, v := range d.Cache {
 		s += fmt.Sprintf("\t%v: %+v\n", k, v)
 	}

cmd/data-usage-cache_gen.go

@@ -629,12 +629,6 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "SkipHealing")
 				return
 			}
-		case "BloomFilter":
-			z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
-			if err != nil {
-				err = msgp.WrapError(err, "BloomFilter")
-				return
-			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -648,24 +642,9 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
-	zb0001Len := uint32(5)
-	var zb0001Mask uint8 /* 5 bits */
-	_ = zb0001Mask
-	if z.BloomFilter == nil {
-		zb0001Len--
-		zb0001Mask |= 0x10
-	}
-	// variable map header, size zb0001Len
-	err = en.Append(0x80 | uint8(zb0001Len))
-	if err != nil {
-		return
-	}
-	if zb0001Len == 0 {
-		return
-	}
+	// map header, size 4
 	// write "Name"
-	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -704,39 +683,15 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "SkipHealing")
 		return
 	}
-	if (zb0001Mask & 0x10) == 0 { // if not empty
-		// write "BloomFilter"
-		err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
-		if err != nil {
-			return
-		}
-		err = en.WriteBytes(z.BloomFilter)
-		if err != nil {
-			err = msgp.WrapError(err, "BloomFilter")
-			return
-		}
-	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
-	zb0001Len := uint32(5)
-	var zb0001Mask uint8 /* 5 bits */
-	_ = zb0001Mask
-	if z.BloomFilter == nil {
-		zb0001Len--
-		zb0001Mask |= 0x10
-	}
-	// variable map header, size zb0001Len
-	o = append(o, 0x80|uint8(zb0001Len))
-	if zb0001Len == 0 {
-		return
-	}
+	// map header, size 4
 	// string "Name"
-	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "NextCycle"
 	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
@@ -747,11 +702,6 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "SkipHealing"
 	o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
 	o = msgp.AppendBool(o, z.SkipHealing)
-	if (zb0001Mask & 0x10) == 0 { // if not empty
-		// string "BloomFilter"
-		o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
-		o = msgp.AppendBytes(o, z.BloomFilter)
-	}
 	return
 }
 
@@ -797,12 +747,6 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "SkipHealing")
 				return
 			}
-		case "BloomFilter":
-			z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
-			if err != nil {
-				err = msgp.WrapError(err, "BloomFilter")
-				return
-			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -817,7 +761,7 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize
 	return
 }
 

cmd/encryption-v1.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -34,7 +34,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
@@ -48,10 +48,11 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
-	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
-	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
-	errKMSKeyNotFound       = errors.New("Invalid KMS keyId")
+	errEncryptedObject                = errors.New("The object was stored using a form of SSE")
+	errInvalidSSEParameters           = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
+	errKMSNotConfigured               = errors.New("KMS not configured for a server side encrypted objects")
+	errKMSKeyNotFound                 = errors.New("Unknown KMS key ID")
+	errKMSDefaultKeyAlreadyConfigured = errors.New("A default encryption already exists on KMS")
 	// Additional MinIO errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
 	// error returned when invalid encryption parameters are specified

cmd/erasure-encode_test.go

@@ -24,7 +24,7 @@ import (
 	"io"
 	"testing"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 type badDisk struct{ StorageAPI }
@@ -41,10 +41,6 @@ func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset
 	return nil, errFaultyDisk
 }
 
-func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
-	return nil, errFaultyDisk
-}
-
 func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
 	return errFaultyDisk
 }

cmd/erasure-healing.go

@@ -66,16 +66,12 @@ func (fi FileInfo) DataShardFixed() bool {
 	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
 }
 
-// Heals a bucket if it doesn't exist on one of the disks, additionally
+// HealBucket heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
 func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
 	result madmin.HealResultItem, err error,
 ) {
-	if !opts.DryRun {
-		defer NSUpdated(bucket, slashSeparator)
-	}
-
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
@@ -411,6 +407,11 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
 	}
 
+	result.ObjectSize, err = latestMeta.GetActualSize()
+	if err != nil {
+		return result, err
+	}
+
 	// Loop to find number of disks with valid data, per-drive
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
@@ -421,9 +422,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		switch {
 		case v != nil:
 			driveState = madmin.DriveStateOk
-			// If data is sane on any one disk, we can
-			// extract the correct object size.
-			result.ObjectSize = partsMetadata[i].Size
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
 		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
@@ -673,9 +671,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		})
 	}
 
-	// Set the size of the object in the heal result
-	result.ObjectSize = latestMeta.Size
-
 	return result, nil
 }
 
@@ -755,7 +750,6 @@ func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object stri
 				}(index, disk)
 			}
 			wg.Wait()
-			NSUpdated(bucket, object)
 		}
 	}
 
@@ -1063,7 +1057,12 @@ func healTrace(funcName healingMetric, startTime time.Time, bucket, object strin
 		Path:      pathJoin(bucket, decodeDirObject(object)),
 	}
 	if opts != nil {
-		tr.Message = fmt.Sprintf("dry:%v, rm:%v, recreate:%v mode:%v", opts.DryRun, opts.Remove, opts.Recreate, opts.ScanMode)
+		tr.Custom = map[string]string{
+			"dry":      fmt.Sprint(opts.DryRun),
+			"remove":   fmt.Sprint(opts.Remove),
+			"recreate": fmt.Sprint(opts.Recreate),
+			"mode":     fmt.Sprint(opts.ScanMode),
+		}
 	}
 	if err != nil {
 		tr.Error = err.Error()

cmd/erasure-metadata.go

@@ -23,15 +23,18 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/sio"
 )
 
 // Object was stored with additional erasure codes due to degraded system at upload time
@@ -104,6 +107,52 @@ func (fi FileInfo) IsValid() bool {
 		correctIndexes)
 }
 
+func (fi FileInfo) checkMultipart() (int64, bool) {
+	if len(fi.Parts) == 0 {
+		return 0, false
+	}
+	if !crypto.IsMultiPart(fi.Metadata) {
+		return 0, false
+	}
+	var size int64
+	for _, part := range fi.Parts {
+		psize, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return 0, false
+		}
+		size += int64(psize)
+	}
+
+	return size, len(extractETag(fi.Metadata)) != 32
+}
+
+// GetActualSize - returns the actual size of the stored object
+func (fi FileInfo) GetActualSize() (int64, error) {
+	if _, ok := fi.Metadata[ReservedMetadataPrefix+"compression"]; ok {
+		sizeStr, ok := fi.Metadata[ReservedMetadataPrefix+"actual-size"]
+		if !ok {
+			return -1, errInvalidDecompressedSize
+		}
+		size, err := strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			return -1, errInvalidDecompressedSize
+		}
+		return size, nil
+	}
+	if _, ok := crypto.IsEncrypted(fi.Metadata); ok {
+		size, ok := fi.checkMultipart()
+		if !ok {
+			size, err := sio.DecryptedSize(uint64(fi.Size))
+			if err != nil {
+				err = errObjectTampered // assign correct error type
+			}
+			return int64(size), err
+		}
+		return size, nil
+	}
+	return fi.Size, nil
+}
+
 // ToObjectInfo - Converts metadata to object info.
 func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInfo {
 	object = decodeDirObject(object)

cmd/erasure-multipart.go

@@ -1216,7 +1216,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
-	defer NSUpdated(bucket, object)
 
 	if !opts.Speedtest && versionsDisparity {
 		listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)

cmd/erasure-object.go

@@ -72,8 +72,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, NotImplemented{}
 	}
 
-	defer NSUpdated(dstBucket, dstObject)
-
 	if !dstOpts.NoLock {
 		lk := er.NewNSLock(dstBucket, dstObject)
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -188,6 +186,14 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	auditObjectErasureSet(ctx, object, &er)
 
+	// This is a special call attempted first to check for SOS-API calls.
+	gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts)
+	if err == nil {
+		return gr, nil
+	}
+	// reset any error to 'nil'
+	err = nil
+
 	var unlockOnDefer bool
 	nsUnlocker := func() {}
 	defer func() {
@@ -475,7 +481,6 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		if opts.VersionID != "" {
 			err = errFileVersionNotFound
 		}
-		defer NSUpdated(bucket, object)
 
 		fi := FileInfo{
 			VersionID: m.VersionID,
@@ -483,7 +488,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		if opts.VersionID != "" {
 			fi.VersionID = opts.VersionID
 		}
-
+		fi.SetTierFreeVersionID(mustGetUUID())
 		disks := er.getDisks()
 		g := errgroup.WithNErrs(len(disks))
 		for index := range disks {
@@ -636,7 +641,10 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 		if errors.Is(reducedErr, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
-			er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			if derr != nil {
+				reducedErr = derr
+			}
 		}
 		return fi, nil, nil, toObjectErr(reducedErr, bucket, object)
 	}
@@ -1260,8 +1268,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
-	defer NSUpdated(bucket, object)
-
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick
@@ -1464,8 +1470,6 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		} else {
 			errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
 		}
-
-		defer NSUpdated(bucket, objects[objIndex].ObjectName)
 	}
 
 	// Check failed deletes across multiple objects
@@ -1557,9 +1561,6 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
-	defer NSUpdated(bucket, object)
-
 	storageDisks := er.getDisks()
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
@@ -1929,7 +1930,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	if fi.TransitionStatus == lifecycle.TransitionComplete {
 		return nil
 	}
-	defer NSUpdated(bucket, object)
 
 	if fi.XLV1 {
 		if _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{NoLock: true}); err != nil {
@@ -2003,15 +2003,13 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket,
 }
 
 // update restore status header in the metadata
-func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions, rerr error) error {
+func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
 	oi := objInfo.Clone()
 	oi.metadataOnly = true // Perform only metadata updates.
 
-	if rerr == nil {
-		oi.UserDefined[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
-	} else { // allow retry in the case of failure to restore
-		delete(oi.UserDefined, xhttp.AmzRestore)
-	}
+	// allow retry in the case of failure to restore
+	delete(oi.UserDefined, xhttp.AmzRestore)
+
 	if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
 		VersionID: oi.VersionID,
 	}, ObjectOptions{
@@ -2027,7 +2025,10 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje
 // as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
 func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
 	setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
-		er.updateRestoreMetadata(ctx, bucket, object, oi, opts, rerr)
+		if rerr == nil {
+			return nil // nothing to do; restore object was successful
+		}
+		er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
 		return rerr
 	}
 	var oi ObjectInfo
@@ -2051,7 +2052,6 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 		}
 		pReader := NewPutObjReader(hashReader)
-		ropts.UserDefined[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
 		_, err = er.PutObject(ctx, bucket, object, pReader, ropts)
 		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 	}

cmd/erasure-server-pool-decom.go

@@ -898,8 +898,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			defer wg.Done()
 			err := set.listObjectsToDecommission(ctx, bi,
 				func(entry metaCacheEntry) {
-					parallelWorkers <- struct{}{}
+					// Wait must be synchronized here.
 					wg.Add(1)
+					parallelWorkers <- struct{}{}
 					go decommissionEntry(entry)
 				},
 			)

cmd/erasure-server-pool-rebalance.go

@@ -133,6 +133,11 @@ func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []st
 	}, len(z.serverPools))
 	var totalCap, totalFree uint64
 	for _, disk := range si.Disks {
+		// Ignore invalid.
+		if disk.PoolIndex < 0 || len(diskStats) <= disk.PoolIndex {
+			// https://github.com/minio/minio/issues/16500
+			continue
+		}
 		totalCap += disk.TotalSpace
 		totalFree += disk.AvailableSpace
 

cmd/erasure-server-pool.go

@@ -56,9 +56,6 @@ type erasureServerPools struct {
 
 	serverPools []*erasureSets
 
-	// Shut down async operations
-	shutdown context.CancelFunc
-
 	// Active decommission canceler
 	decommissionCancelers []context.CancelFunc
 
@@ -160,14 +157,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		break
 	}
 
-	drives := make([]string, 0, len(localDrives))
-	for _, localDrive := range localDrives {
-		drives = append(drives, localDrive.Endpoint().Path)
-	}
-
 	globalLocalDrives = localDrives
-	ctx, z.shutdown = context.WithCancel(ctx)
-	go intDataUpdateTracker.start(ctx, drives...)
 	return z, nil
 }
 
@@ -524,8 +514,6 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri
 }
 
 func (z *erasureServerPools) Shutdown(ctx context.Context) error {
-	defer z.shutdown()
-
 	g := errgroup.WithNErrs(len(z.serverPools))
 
 	for index := range z.serverPools {
@@ -594,7 +582,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) StorageInfo {
 	return globalNotificationSys.StorageInfo(z)
 }
 
-func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
+func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
 	// Updates must be closed before we return.
 	defer close(updates)
 
@@ -639,7 +627,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 					}
 				}()
 				// Start scanner. Blocks until done.
-				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates, healScanMode)
+				err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					mu.Lock()
@@ -712,8 +700,6 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 // even if one of the sets fail to create buckets, we proceed all the successful
 // operations.
 func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
-	defer NSUpdated(bucket, slashSeparator)
-
 	// Verify if bucket is valid.
 	if !isMinioMetaBucketName(bucket) {
 		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
@@ -1207,6 +1193,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	if marker == "" && versionMarker != "" {
 		return loi, NotImplemented{}
 	}
+
 	opts := listPathOptions{
 		Bucket:      bucket,
 		Prefix:      prefix,
@@ -1217,6 +1204,17 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		AskDisks:    globalAPIConfig.getListQuorum(),
 		Versioned:   true,
 	}
+
+	// Shortcut for APN/1.0 Veeam/1.0 Backup/10.0
+	// It requests unique blocks with a specific prefix.
+	// We skip scanning the parent directory for
+	// more objects matching the prefix.
+	ri := logger.GetReqInfo(ctx)
+	if ri != nil && strings.Contains(ri.UserAgent, `1.0 Veeam/1.0 Backup`) && strings.HasSuffix(prefix, ".blk") {
+		opts.BaseDir = prefix
+		opts.Transient = true
+	}
+
 	// set bucket metadata in opts
 	opts.setBucketMeta(ctx)
 
@@ -1625,7 +1623,6 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
-	defer NSUpdated(bucket, slashSeparator)
 	if !opts.NoLock {
 		// Lock the bucket name before creating.
 		lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")

cmd/erasure.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
-	"github.com/minio/pkg/console"
 )
 
 // list all errors that can be ignore in a bucket operation.
@@ -351,7 +350,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
-func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
+func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
 	if len(buckets) == 0 {
 		return nil
 	}
@@ -377,7 +376,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 		},
 		Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
 	}
-	bloom := bf.bytes()
 
 	// Put all buckets into channel.
 	bucketCh := make(chan BucketInfo, len(buckets))
@@ -473,7 +471,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				if cache.Info.Name == "" {
 					cache.Info.Name = bucket.Name
 				}
-				cache.Info.BloomFilter = bloom
 				cache.Info.SkipHealing = healing
 				cache.Info.NextCycle = wantCycle
 				if cache.Info.Name != bucket.Name {
@@ -496,16 +493,12 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 							Parent: dataUsageRoot,
 							Entry:  update,
 						}
-						if intDataUpdateTracker.debug {
-							console.Debugln("z:", er.poolIndex, "s:", er.setIndex, "bucket", name, "got update", update)
-						}
 					}
 				}(cache.Info.Name)
 				// Calc usage
 				before := cache.Info.LastUpdate
 				var err error
 				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
-				cache.Info.BloomFilter = nil
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
 						logger.LogIf(ctx, cache.save(ctx, er, cacheName))

cmd/event-notification.go

@@ -248,6 +248,7 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 
 	respElements := map[string]string{
 		"x-amz-request-id": args.RespElements["requestId"],
+		"x-amz-id-2":       args.RespElements["nodeId"],
 		"x-minio-origin-endpoint": func() string {
 			if globalMinioEndpoint != "" {
 				return globalMinioEndpoint
@@ -255,17 +256,18 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 			return getAPIEndpoints()[0]
 		}(), // MinIO specific custom elements.
 	}
-	// Add deployment as part of
-	if globalDeploymentID != "" {
-		respElements["x-minio-deployment-id"] = globalDeploymentID
-	}
+
+	// Add deployment as part of response elements.
+	respElements["x-minio-deployment-id"] = globalDeploymentID
 	if args.RespElements["content-length"] != "" {
 		respElements["content-length"] = args.RespElements["content-length"]
 	}
+
 	keyName := args.Object.Name
 	if escape {
 		keyName = url.QueryEscape(args.Object.Name)
 	}
+
 	newEvent := event.Event{
 		EventVersion:      "2.0",
 		EventSource:       "minio:s3",
@@ -312,12 +314,13 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 }
 
 func sendEvent(args eventArgs) {
-	args.Object.Size, _ = args.Object.GetActualSize()
-
 	// avoid generating a notification for REPLICA creation event.
 	if _, ok := args.ReqParams[xhttp.MinIOSourceReplicationRequest]; ok {
 		return
 	}
+
+	args.Object.Size, _ = args.Object.GetActualSize()
+
 	// remove sensitive encryption entries in metadata.
 	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
 	crypto.RemoveInternalEntries(args.Object.UserDefined)

cmd/format-erasure.go

@@ -304,11 +304,6 @@ func countErrs(errs []error, err error) int {
 	return i
 }
 
-// Does all errors indicate we need to initialize all disks?.
-func shouldInitErasureDisks(errs []error) bool {
-	return countErrs(errs, errUnformattedDisk) == len(errs)
-}
-
 // Check if unformatted disks are equal to write quorum.
 func quorumUnformattedDisks(errs []error) bool {
 	return countErrs(errs, errUnformattedDisk) >= (len(errs)/2)+1
@@ -750,6 +745,9 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 		hostCount := make(map[string]int, setDriveCount)
 		for j := 0; j < setDriveCount; j++ {
 			disk := storageDisks[i*setDriveCount+j]
+			if disk == nil {
+				continue
+			}
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
 			if distributionAlgo != "" {
@@ -761,26 +759,24 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 			hostCount[disk.Hostname()]++
 			formats[i*setDriveCount+j] = newFormat
 		}
-		if len(hostCount) > 0 {
-			var once sync.Once
-			for host, count := range hostCount {
-				if count > wantAtMost {
-					if host == "" {
-						host = "local"
-					}
-					once.Do(func() {
-						if len(hostCount) == 1 {
-							return
-						}
-						logger.Info(" * Set %v:", i+1)
-						for j := 0; j < setDriveCount; j++ {
-							disk := storageDisks[i*setDriveCount+j]
-							logger.Info("   - Drive: %s", disk.String())
-						}
-					})
-					logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+
-						"A host failure will result in data becoming unavailable.", host, wantAtMost)
+		var once sync.Once
+		for host, count := range hostCount {
+			if count > wantAtMost {
+				if host == "" {
+					host = "local"
 				}
+				once.Do(func() {
+					if len(hostCount) == 1 {
+						return
+					}
+					logger.Info(" * Set %v:", i+1)
+					for j := 0; j < setDriveCount; j++ {
+						disk := storageDisks[i*setDriveCount+j]
+						logger.Info("   - Drive: %s", disk.String())
+					}
+				})
+				logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+
+					"A host failure will result in data becoming unavailable.", host, wantAtMost)
 			}
 		}
 	}

cmd/generic-handlers.go

@@ -306,6 +306,7 @@ func setHTTPStatsHandler(h http.Handler) http.Handler {
 
 		if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
 			strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
+			strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
 			strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
 			globalConnStats.incInputBytes(meteredRequest.BytesRead())
 			globalConnStats.incOutputBytes(meteredResponse.BytesWritten())
@@ -550,6 +551,9 @@ func addCustomHeaders(h http.Handler) http.Handler {
 		// part of the log entry, Error response XML and auditing.
 		// Set custom headers such as x-amz-request-id for each request.
 		w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow()))
+		if globalLocalNodeName != "" {
+			w.Header().Set(xhttp.AmzRequestHostID, globalLocalNodeNameHex)
+		}
 		h.ServeHTTP(xhttp.NewResponseRecorder(w), r)
 	})
 }

cmd/global-heal.go

@@ -21,17 +21,14 @@ import (
 	"context"
 	"fmt"
 	"sort"
-	"strconv"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/storageclass"
-	"github.com/minio/minio/internal/jobtokens"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
-	"github.com/minio/pkg/env"
 	"github.com/minio/pkg/wildcard"
 )
 
@@ -51,7 +48,6 @@ func newBgHealSequence() *healSequence {
 	}
 
 	return &healSequence{
-		respCh:      make(chan healResult),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
 		// run-background heal with reserved bucket
@@ -163,8 +159,6 @@ func mustGetHealSequence(ctx context.Context) *healSequence {
 	}
 }
 
-const envHealWorkers = "_MINIO_HEAL_WORKERS"
-
 // healErasureSet lists and heals all objects in a specific erasure set
 func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
 	bgSeq := mustGetHealSequence(ctx)
@@ -185,16 +179,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 	}
 
-	// numHealers - number of concurrent heal jobs, defaults to 1
-	numHealers, err := strconv.Atoi(env.Get(envHealWorkers, "1"))
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("invalid %s value %v, defaulting to 1", envHealWorkers, err))
-	}
-	if numHealers < 1 {
-		numHealers = 1
-	}
-	// jt will never be nil since we ensure that numHealers > 0
-	jt, _ := jobtokens.New(numHealers)
 	var retErr error
 	// Heal all buckets with all objects
 	for _, bucket := range healBuckets {
@@ -229,11 +213,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 		disks, _ := er.getOnlineDisksWithHealing()
 		if len(disks) == 0 {
-			// all disks are healing in this set, this is allowed
-			// so we simply proceed to next bucket, marking the bucket
-			// as done as there are no objects to heal.
-			tracker.bucketDone(bucket)
-			logger.LogIf(ctx, tracker.update(ctx))
+			logger.LogIf(ctx, fmt.Errorf("no online disks found to heal the bucket `%s`", bucket))
 			continue
 		}
 
@@ -289,9 +269,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}()
 
 		// Note: updates from healEntry to tracker must be sent on results channel.
-		healEntry := func(entry metaCacheEntry) {
-			defer jt.Give()
-
+		healEntry := func(bucket string, entry metaCacheEntry) {
 			if entry.name == "" && len(entry.metadata) == 0 {
 				// ignore entries that don't have metadata.
 				return
@@ -300,6 +278,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				// ignore healing entry.name's with `/` suffix.
 				return
 			}
+
 			// We might land at .metacache, .trash, .multipart
 			// no need to heal them skip, only when bucket
 			// is '.minio.sys'
@@ -324,6 +303,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					versionID: "",
 				}, madmin.HealItemObject)
 				if err != nil {
+					if isErrObjectNotFound(err) {
+						// queueing happens across namespace, ignore
+						// objects that are not found.
+						return
+					}
 					result = healEntryFailure(0)
 					logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
@@ -339,20 +323,19 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				return
 			}
 
-			// erasureObjects layer needs object names to be encoded
-			encodedEntryName := encodeDirObject(entry.name)
-			if healDeleteDangling {
-				err := er.checkAbandonedParts(ctx, bucket, encodedEntryName, madmin.HealOpts{Remove: healDeleteDangling})
-				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", bucket, entry.name, err))
-				}
-			}
+			var versionNotFound int
 			for _, version := range fivs.Versions {
-				if _, err := er.HealObject(ctx, bucket, encodedEntryName,
-					version.VersionID, madmin.HealOpts{
-						ScanMode: scanMode,
-						Remove:   healDeleteDangling,
-					}); err != nil {
+				if err := bgSeq.queueHealTask(healSource{
+					bucket:    bucket,
+					object:    version.Name,
+					versionID: version.VersionID,
+				}, madmin.HealItemObject); err != nil {
+					if isErrObjectNotFound(err) {
+						// queueing happens across namespace, ignore
+						// objects that are not found.
+						versionNotFound++
+						continue
+					}
 					// If not deleted, assume they failed.
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
@@ -371,6 +354,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				case results <- result:
 				}
 			}
+			// All versions resulted in 'ObjectNotFound'
+			if versionNotFound == len(fivs.Versions) {
+				return
+			}
 			select {
 			case <-ctx.Done():
 				return
@@ -381,23 +368,25 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			waitForLowHTTPReq()
 		}
 
+		actualBucket, prefix := path2BucketObject(bucket)
+
 		// How to resolve partial results.
 		resolver := metadataResolutionParams{
 			dirQuorum: 1,
 			objQuorum: 1,
-			bucket:    bucket,
+			bucket:    actualBucket,
 		}
 
-		err = listPathRaw(ctx, listPathRawOptions{
+		err := listPathRaw(ctx, listPathRawOptions{
 			disks:          disks,
-			bucket:         bucket,
+			bucket:         actualBucket,
+			path:           prefix,
 			recursive:      true,
 			forwardTo:      forwardTo,
 			minDisks:       1,
 			reportNotFound: false,
 			agreed: func(entry metaCacheEntry) {
-				jt.Take()
-				go healEntry(entry)
+				healEntry(actualBucket, entry)
 			},
 			partial: func(entries metaCacheEntries, _ []error) {
 				entry, ok := entries.resolve(&resolver)
@@ -406,12 +395,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					// proceed to heal nonetheless.
 					entry, _ = entries.firstFound()
 				}
-				jt.Take()
-				go healEntry(*entry)
+				healEntry(actualBucket, *entry)
 			},
 			finished: nil,
 		})
-		jt.Wait() // synchronize all the concurrent heal jobs
 		close(results)
 		if err != nil {
 			// Set this such that when we return this function

cmd/globals.go

@@ -42,10 +42,7 @@ import (
 	"github.com/minio/minio/internal/config/callhome"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/config/dns"
-	xldap "github.com/minio/minio/internal/config/identity/ldap"
-	"github.com/minio/minio/internal/config/identity/openid"
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
-	xtls "github.com/minio/minio/internal/config/identity/tls"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/config/subnet"
@@ -201,10 +198,6 @@ var (
 
 	globalStorageClass storageclass.Config
 
-	globalLDAPConfig   xldap.Config
-	globalOpenIDConfig openid.Config
-	globalSTSTLSConfig xtls.Config
-
 	globalAuthNPlugin *idplugin.AuthNPlugin
 
 	// CA root certificates, a nil value means system certs pool will be used
@@ -233,7 +226,8 @@ var (
 	globalEndpoints EndpointServerPools
 
 	// The name of this local node, fetched from arguments
-	globalLocalNodeName string
+	globalLocalNodeName    string
+	globalLocalNodeNameHex string
 
 	// The global subnet config
 	globalSubnetConfig subnet.Config

cmd/handler-api.go

@@ -227,6 +227,13 @@ func (t *apiConfig) getClusterDeadline() time.Duration {
 	return t.clusterDeadline
 }
 
+func (t *apiConfig) getRequestsPoolCapacity() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return cap(t.requestsPool)
+}
+
 func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 	t.mu.RLock()
 	defer t.mu.RUnlock()

cmd/handler-utils.go

@@ -58,6 +58,10 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	if location == "" {
 		location = globalSite.Region
 	}
+	if !isValidLocation(location) {
+		return location, ErrInvalidRegion
+	}
+
 	return location, ErrNone
 }
 
@@ -242,6 +246,7 @@ func extractRespElements(w http.ResponseWriter) map[string]string {
 	}
 	return map[string]string{
 		"requestId":      w.Header().Get(xhttp.AmzRequestID),
+		"nodeId":         w.Header().Get(xhttp.AmzRequestHostID),
 		"content-length": w.Header().Get(xhttp.ContentLength),
 		// Add more fields here.
 	}
@@ -403,6 +408,12 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	desc := "Do not upgrade one server at a time - please follow the recommended guidelines mentioned here https://github.com/minio/minio#upgrading-minio for your environment"
 	switch {
+	case strings.HasPrefix(r.URL.Path, peerS3Prefix):
+		writeErrorResponseString(r.Context(), w, APIError{
+			Code:           "XMinioPeerS3VersionMismatch",
+			Description:    desc,
+			HTTPStatusCode: http.StatusUpgradeRequired,
+		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",

cmd/healthcheck-handler.go

@@ -21,13 +21,14 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+	"time"
 
 	xhttp "github.com/minio/minio/internal/http"
 )
 
 const unavailable = "offline"
 
-func shouldProxy() bool {
+func isServerInitialized() bool {
 	return newObjectLayerFn() == nil
 }
 
@@ -35,7 +36,7 @@ func shouldProxy() bool {
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
-	if shouldProxy() {
+	if isServerInitialized() {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
@@ -73,7 +74,7 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
-	if shouldProxy() {
+	if isServerInitialized() {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
@@ -100,11 +101,39 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	if shouldProxy() {
+	if isServerInitialized() {
 		// Service not initialized yet
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 	}
 
+	if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
+		apiErr := getAPIError(ErrBusy)
+		switch r.Method {
+		case http.MethodHead:
+			writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+		case http.MethodGet:
+			writeErrorResponse(r.Context(), w, apiErr, r.URL)
+		}
+		return
+	}
+
+	// Verify if KMS is reachable if its configured
+	if GlobalKMS != nil {
+		ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
+		defer cancel()
+
+		if _, err := GlobalKMS.Stat(ctx); err != nil {
+			switch r.Method {
+			case http.MethodHead:
+				apiErr := toAPIError(r.Context(), err)
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+			}
+			return
+		}
+	}
+
 	if globalEtcdClient != nil {
 		// Borrowed from
 		// https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118

cmd/http-stats.go

@@ -255,6 +255,10 @@ type HTTPStats struct {
 	totalS3Canceled         HTTPAPIStats
 }
 
+func (st *HTTPStats) loadRequestsInQueue() int32 {
+	return atomic.LoadInt32(&st.s3RequestsInQueue)
+}
+
 func (st *HTTPStats) addRequestsInQueue(i int32) {
 	atomic.AddInt32(&st.s3RequestsInQueue, i)
 }

cmd/iam-etcd-store.go

@@ -245,6 +245,13 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.AccessKey == "" {
 		u.Credentials.AccessKey = user
 	}
+	if u.Credentials.SessionToken != "" {
+		jwtClaims, err := extractJWTClaims(u)
+		if err != nil {
+			return err
+		}
+		u.Credentials.Claims = jwtClaims.Map()
+	}
 	m[user] = u
 	return nil
 }

cmd/iam-object-store.go

@@ -184,6 +184,14 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 		u.Credentials.AccessKey = user
 	}
 
+	if u.Credentials.SessionToken != "" {
+		jwtClaims, err := extractJWTClaims(u)
+		if err != nil {
+			return err
+		}
+		u.Credentials.Claims = jwtClaims.Map()
+	}
+
 	m[user] = u
 	return nil
 }

cmd/iam-store.go

@@ -33,6 +33,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
@@ -76,8 +77,13 @@ const (
 	iamFormatFile = "format.json"
 
 	iamFormatVersion1 = 1
+
+	minServiceAccountExpiry time.Duration = 15 * time.Minute
+	maxServiceAccountExpiry time.Duration = 365 * 24 * time.Hour
 )
 
+var errInvalidSvcAcctExpiration = errors.New("invalid service account expiration")
+
 type iamFormat struct {
 	Version int `json:"version"`
 }
@@ -394,6 +400,19 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 	return policies, mp.UpdatedAt, nil
 }
 
+func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
+	if u.Credentials.SessionToken != "" {
+		jwtClaims, err := extractJWTClaims(u)
+		if err != nil {
+			return err
+		}
+		u.Credentials.Claims = jwtClaims.Map()
+	}
+	c.iamUsersMap[key] = u
+	c.updatedAt = time.Now()
+	return nil
+}
+
 // IAMStorageAPI defines an interface for the IAM persistence layer
 type IAMStorageAPI interface {
 	// The role of the read-write lock is to prevent go routines from
@@ -874,6 +893,25 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 	return
 }
 
+// listGroups - lists groups - fetch groups from cache
+func (store *IAMStoreSys) listGroups(ctx context.Context) (res []string, err error) {
+	cache := store.rlock()
+	defer store.runlock()
+
+	if store.getUsersSysType() == MinIOUsersSysType {
+		for k := range cache.iamGroupsMap {
+			res = append(res, k)
+		}
+	}
+
+	if store.getUsersSysType() == LDAPUsersSysType {
+		for k := range cache.iamGroupPolicyMap {
+			res = append(res, k)
+		}
+	}
+	return
+}
+
 // PolicyDBUpdate - adds or removes given policies to/from the user or group's
 // policy associations.
 func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGroup bool,
@@ -1231,6 +1269,20 @@ func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string)
 	return ret, nil
 }
 
+// fetches all policy docs from cache.
+// If bucketName is non-empty, returns policy docs matching the bucket.
+func (store *IAMStoreSys) listPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) {
+	cache := store.rlock()
+	defer store.runlock()
+	ret := map[string]PolicyDoc{}
+	for k, v := range cache.iamPolicyDocsMap {
+		if bucketName == "" || v.Policy.MatchResource(bucketName) {
+			ret[k] = v
+		}
+	}
+	return ret, nil
+}
+
 // helper function - does not take locks.
 func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, iampolicy.Policy) {
 	var policies []string
@@ -1716,14 +1768,15 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	if userType == regUser {
 		for _, ui := range cache.iamUsersMap {
 			u := ui.Credentials
-			if u.IsServiceAccount() && u.ParentUser == accessKey {
-				_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
-				delete(cache.iamUsersMap, u.AccessKey)
-			}
-			// Delete any associated STS users.
-			if u.IsTemp() && u.ParentUser == accessKey {
-				_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
-				delete(cache.iamUsersMap, u.AccessKey)
+			if u.ParentUser == accessKey {
+				switch {
+				case u.IsServiceAccount():
+					_ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser)
+					delete(cache.iamUsersMap, u.AccessKey)
+				case u.IsTemp():
+					_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
+					delete(cache.iamUsersMap, u.AccessKey)
+				}
 			}
 		}
 	}
@@ -2073,8 +2126,9 @@ func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, s
 		return updatedAt, err
 	}
 
-	cache.iamUsersMap[accessKey] = uinfo
-	cache.updatedAt = time.Now()
+	if err := cache.updateUserWithClaims(accessKey, uinfo); err != nil {
+		return updatedAt, err
+	}
 
 	return uinfo.UpdatedAt, nil
 }
@@ -2093,14 +2147,14 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
 	if su, found := cache.iamUsersMap[accessKey]; found {
 		scred := su.Credentials
 		if scred.ParentUser != parentUser {
-			return updatedAt, errIAMServiceAccountUsed
+			return updatedAt, fmt.Errorf("%w: the service account access key is taken by another user", errIAMServiceAccountNotAllowed)
 		}
-		return updatedAt, errIAMServiceAccount
+		return updatedAt, fmt.Errorf("%w: the service account access key already taken", errIAMServiceAccountNotAllowed)
 	}
 
 	// Parent user must not be a service account.
 	if u, found := cache.iamUsersMap[parentUser]; found && u.Credentials.IsServiceAccount() {
-		return updatedAt, errIAMServiceAccount
+		return updatedAt, fmt.Errorf("%w: unable to create a service account for another service account", errIAMServiceAccountNotAllowed)
 	}
 
 	u := newUserIdentity(cred)
@@ -2109,8 +2163,7 @@ func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Crede
 		return updatedAt, err
 	}
 
-	cache.iamUsersMap[u.Credentials.AccessKey] = u
-	cache.updatedAt = time.Now()
+	cache.updateUserWithClaims(u.Credentials.AccessKey, u)
 
 	return u.UpdatedAt, nil
 }
@@ -2137,6 +2190,14 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		cr.Comment = opts.comment
 	}
 
+	if opts.expiration != nil {
+		expirationInUTC := opts.expiration.UTC()
+		if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
+			return updatedAt, err
+		}
+		cr.Expiration = expirationInUTC
+	}
+
 	switch opts.status {
 	// The caller did not ask to update status account, do nothing
 	case "":
@@ -2196,8 +2257,9 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return updatedAt, err
 	}
 
-	cache.iamUsersMap[u.Credentials.AccessKey] = u
-	cache.updatedAt = time.Now()
+	if err := cache.updateUserWithClaims(u.Credentials.AccessKey, u); err != nil {
+		return updatedAt, err
+	}
 
 	return u.UpdatedAt, nil
 }
@@ -2298,8 +2360,9 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq ma
 	if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil {
 		return updatedAt, err
 	}
-
-	cache.iamUsersMap[accessKey] = u
+	if err := cache.updateUserWithClaims(accessKey, u); err != nil {
+		return updatedAt, err
+	}
 
 	return u.UpdatedAt, nil
 }
@@ -2322,8 +2385,7 @@ func (store *IAMStoreSys) UpdateUserSecretKey(ctx context.Context, accessKey, se
 		return err
 	}
 
-	cache.iamUsersMap[accessKey] = u
-	return nil
+	return cache.updateUserWithClaims(accessKey, u)
 }
 
 // GetSTSAndServiceAccounts - returns all STS and Service account credentials.
@@ -2360,8 +2422,8 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
 	if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, ui); err != nil {
 		return err
 	}
-	cache.iamUsersMap[cred.AccessKey] = ui
-	return nil
+
+	return cache.updateUserWithClaims(cred.AccessKey, ui)
 }
 
 // LoadUser - attempts to load user info from storage and updates cache.
@@ -2404,3 +2466,25 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 		}
 	}
 }
+
+func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {
+	jwtClaims, err := auth.ExtractClaims(u.Credentials.SessionToken, u.Credentials.SecretKey)
+	if err != nil {
+		// Session tokens for STS creds will be generated with root secret
+		jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return jwtClaims, nil
+}
+
+func validateSvcExpirationInUTC(expirationInUTC time.Time) error {
+	currentTime := time.Now().UTC()
+	minExpiration := currentTime.Add(minServiceAccountExpiry)
+	maxExpiration := currentTime.Add(maxServiceAccountExpiry)
+	if expirationInUTC.Before(minExpiration) || expirationInUTC.After(maxExpiration) {
+		return errInvalidSvcAcctExpiration
+	}
+	return nil
+}

cmd/iam.go

@@ -42,6 +42,7 @@ import (
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
+	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/policy/opa"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	xhttp "github.com/minio/minio/internal/http"
@@ -87,8 +88,10 @@ type IAMSys struct {
 	sync.Mutex
 
 	iamRefreshInterval time.Duration
-	ldapConfig         xldap.Config  // only valid if usersSysType is LDAPUsers
-	openIDConfig       openid.Config // only valid if OpenID is configured
+
+	LDAPConfig   xldap.Config  // only valid if usersSysType is LDAPUsers
+	OpenIDConfig openid.Config // only valid if OpenID is configured
+	STSTLSConfig xtls.Config   // only valid if STS TLS is configured
 
 	usersSysType UsersSysType
 
@@ -163,7 +166,7 @@ func (sys *IAMSys) LoadServiceAccount(ctx context.Context, accessKey string) err
 
 // initStore initializes IAM stores
 func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
-	if sys.ldapConfig.Enabled() {
+	if sys.LDAPConfig.Enabled() {
 		sys.SetUsersSysType(LDAPUsersSysType)
 	}
 
@@ -213,26 +216,34 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	s := globalServerConfig
 	globalServerConfigMu.RUnlock()
 
-	var err error
-	globalOpenIDConfig, err = openid.LookupConfig(s,
+	openidConfig, err := openid.LookupConfig(s,
 		NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
 	}
 
 	// Initialize if LDAP is enabled
-	globalLDAPConfig, err = xldap.Lookup(s, globalRootCAs)
+	ldapConfig, err := xldap.Lookup(s, globalRootCAs)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
 	}
 
+	stsTLSConfig, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
+	}
+
+	if stsTLSConfig.InsecureSkipVerify {
+		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
+	}
+
 	authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
 		NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
 	}
 
-	setGlobalAuthNPlugin(idplugin.New(authNPluginCfg))
+	setGlobalAuthNPlugin(idplugin.New(GlobalContext, authNPluginCfg))
 
 	authZPluginCfg, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody)
 	if err != nil {
@@ -257,8 +268,10 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	sys.Lock()
 	defer sys.Unlock()
 
-	sys.ldapConfig = globalLDAPConfig.Clone()
-	sys.openIDConfig = globalOpenIDConfig.Clone()
+	sys.LDAPConfig = ldapConfig
+	sys.OpenIDConfig = openidConfig
+	sys.STSTLSConfig = stsTLSConfig
+
 	sys.iamRefreshInterval = iamRefreshInterval
 
 	// Initialize IAM store
@@ -294,7 +307,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
 				continue
 			}
-			logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, unable to write the IAM format"))
+			logger.LogIf(ctx, fmt.Errorf("IAM sub-system is partially initialized, unable to write the IAM format: %w", err))
 			return
 		}
 
@@ -310,7 +323,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 				continue
 			}
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available: %w", err))
 			}
 		}
 		break
@@ -322,7 +335,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	// Set up polling for expired accounts and credentials purging.
 	switch {
-	case sys.openIDConfig.ProviderEnabled():
+	case sys.OpenIDConfig.ProviderEnabled():
 		go func() {
 			timer := time.NewTimer(refreshInterval)
 			defer timer.Stop()
@@ -337,7 +350,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 				}
 			}
 		}()
-	case sys.ldapConfig.Enabled():
+	case sys.LDAPConfig.Enabled():
 		go func() {
 			timer := time.NewTimer(refreshInterval)
 			defer timer.Stop()
@@ -363,7 +376,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	sys.rolesMap = make(map[arn.ARN]string)
 
 	// From OpenID
-	if riMap := globalOpenIDConfig.GetRoleInfo(); riMap != nil {
+	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
 		sys.validateAndAddRolePolicyMappings(ctx, riMap)
 	}
 
@@ -800,7 +813,7 @@ func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyE
 
 	select {
 	case <-sys.configLoaded:
-		pe := sys.store.ListLDAPPolicyMappings(q, sys.ldapConfig.IsLDAPUserDN, sys.ldapConfig.IsLDAPGroupDN)
+		pe := sys.store.ListLDAPPolicyMappings(q, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN)
 		pe.Timestamp = UTCNow()
 		return &pe, nil
 	case <-ctx.Done():
@@ -920,6 +933,7 @@ type newServiceAccountOpts struct {
 	accessKey     string
 	secretKey     string
 	comment       string
+	expiration    *time.Time
 
 	claims map[string]interface{}
 }
@@ -992,6 +1006,14 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	cred.Status = string(auth.AccountOn)
 	cred.Comment = opts.comment
 
+	if opts.expiration != nil {
+		expirationInUTC := opts.expiration.UTC()
+		if err := validateSvcExpirationInUTC(expirationInUTC); err != nil {
+			return auth.Credentials{}, time.Time{}, err
+		}
+		cred.Expiration = expirationInUTC
+	}
+
 	updatedAt, err := sys.store.AddServiceAccount(ctx, cred)
 	if err != nil {
 		return auth.Credentials{}, time.Time{}, err
@@ -1006,6 +1028,7 @@ type updateServiceAccountOpts struct {
 	secretKey     string
 	status        string
 	comment       string
+	expiration    *time.Time
 }
 
 // UpdateServiceAccount - edit a service account
@@ -1145,12 +1168,9 @@ func (sys *IAMSys) getAccountWithClaims(ctx context.Context, accessKey string) (
 		return UserIdentity{}, nil, errNoSuchAccount
 	}
 
-	jwtClaims, err := auth.ExtractClaims(acc.Credentials.SessionToken, acc.Credentials.SecretKey)
+	jwtClaims, err := extractJWTClaims(acc)
 	if err != nil {
-		jwtClaims, err = auth.ExtractClaims(acc.Credentials.SessionToken, globalActiveCred.SecretKey)
-		if err != nil {
-			return UserIdentity{}, nil, err
-		}
+		return UserIdentity{}, nil, err
 	}
 
 	return acc, jwtClaims, nil
@@ -1171,13 +1191,11 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 		return nil, errNoSuchServiceAccount
 	}
 
-	jwtClaims, err := auth.ExtractClaims(sa.Credentials.SessionToken, sa.Credentials.SecretKey)
+	jwtClaims, err := extractJWTClaims(sa)
 	if err != nil {
-		jwtClaims, err = auth.ExtractClaims(sa.Credentials.SessionToken, globalActiveCred.SecretKey)
-		if err != nil {
-			return nil, err
-		}
+		return nil, err
 	}
+
 	return jwtClaims.Map(), nil
 }
 
@@ -1280,7 +1298,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 			continue
 		}
 		roleArn = roleArns[0]
-		u, err := sys.openIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
+		u, err := sys.OpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 			continue
@@ -1302,14 +1320,14 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	parentUsers := sys.store.GetAllParentUsers()
 	var allDistNames []string
 	for parentUser := range parentUsers {
-		if !sys.ldapConfig.IsLDAPUserDN(parentUser) {
+		if !sys.LDAPConfig.IsLDAPUserDN(parentUser) {
 			continue
 		}
 
 		allDistNames = append(allDistNames, parentUser)
 	}
 
-	expiredUsers, err := sys.ldapConfig.GetNonEligibleUserDistNames(allDistNames)
+	expiredUsers, err := sys.LDAPConfig.GetNonEligibleUserDistNames(allDistNames)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		logger.LogIf(GlobalContext, err)
@@ -1331,7 +1349,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	// DN to ldap username mapping for each LDAP user
 	parentUserToLDAPUsernameMap := make(map[string]string)
 	for _, cred := range allCreds {
-		if !sys.ldapConfig.IsLDAPUserDN(cred.ParentUser) {
+		if !sys.LDAPConfig.IsLDAPUserDN(cred.ParentUser) {
 			continue
 		}
 		// Check if this is the first time we are
@@ -1379,7 +1397,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	}
 
 	// 2. Query LDAP server for groups of the LDAP users collected.
-	updatedGroups, err := sys.ldapConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
+	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		logger.LogIf(GlobalContext, err)
@@ -1566,7 +1584,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 	var dn string
 	var isGroup bool
 	if r.User != "" {
-		dn, err = globalLDAPConfig.DoesUsernameExist(r.User)
+		dn, err = sys.LDAPConfig.DoesUsernameExist(r.User)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return updatedAt, nil, err
@@ -1576,7 +1594,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		}
 		isGroup = false
 	} else {
-		if exists, err := globalLDAPConfig.DoesGroupDNExist(r.Group); err != nil {
+		if exists, err := sys.LDAPConfig.DoesGroupDNExist(r.Group); err != nil {
 			logger.LogIf(ctx, err)
 			return updatedAt, nil, err
 		} else if !exists {
@@ -1693,9 +1711,6 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 
 	parentArgs := args
 	parentArgs.AccountName = parentUser
-	// These are dynamic values set them appropriately.
-	parentArgs.ConditionValues["username"] = []string{parentUser}
-	parentArgs.ConditionValues["userid"] = []string{parentUser}
 
 	saPolicyClaim, ok := args.Claims[iamPolicyClaimNameSA()]
 	if !ok {
@@ -1822,10 +1837,6 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 
 	// 3. If an inline session-policy is present, evaluate it.
 
-	// These are dynamic values set them appropriately.
-	args.ConditionValues["username"] = []string{parentUser}
-	args.ConditionValues["userid"] = []string{parentUser}
-
 	// Now check if we have a sessionPolicy.
 	hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicy(args)
 	if hasSessionPolicy {

cmd/kms-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -23,7 +23,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -82,6 +82,11 @@ func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
+	if _, ok := GlobalKMS.(kms.KeyManager); !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
 	metrics, err := GlobalKMS.Metrics(ctx)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -168,7 +173,12 @@ func (a kmsAPIHandlers) KMSVersionHandler(w http.ResponseWriter, r *http.Request
 
 // KMSCreateKeyHandler - POST /minio/kms/v1/key/create?key-id=<master-key-id>
 func (a kmsAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
+	// If env variable MINIO_KMS_SECRET_KEY is populated, prevent creation of new keys
 	ctx := newContext(r, w, "KMSCreateKey")
+	if GlobalKMS != nil && GlobalKMS.IsLocal() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSDefaultKeyAlreadyConfigured), r.URL)
+		return
+	}
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAction)
@@ -223,6 +233,15 @@ func (a kmsAPIHandlers) KMSDeleteKeyHandler(w http.ResponseWriter, r *http.Reque
 // KMSListKeysHandler - GET /minio/kms/v1/key/list?pattern=<pattern>
 func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSListKeys")
+	if GlobalKMS != nil && GlobalKMS.IsLocal() {
+		res, err := json.Marshal(GlobalKMS.List())
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		writeSuccessResponseJSON(w, res)
+		return
+	}
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSListKeysAction)
@@ -236,7 +255,7 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 	}
 	manager, ok := GlobalKMS.(kms.KeyManager)
 	if !ok {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
 	keys, err := manager.ListKeys(ctx, r.Form.Get("pattern"))

cmd/local-locker.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"strconv"
-	"sync"
 	"time"
 
 	"github.com/minio/minio/internal/dsync"
@@ -51,7 +50,7 @@ func isWriteLock(lri []lockRequesterInfo) bool {
 
 // localLocker implements Dsync.NetLocker
 type localLocker struct {
-	mutex   sync.Mutex
+	mutex   chan struct{}
 	lockMap map[string][]lockRequesterInfo
 	lockUID map[string]string // UUID -> resource map.
 }
@@ -70,13 +69,41 @@ func (l *localLocker) canTakeLock(resources ...string) bool {
 	return true
 }
 
+// lockMu locks the "mutex" of the local locker.
+// If "ctx" is canceled before the lock can be obtained false is returned.
+// If "true" is returned unlockMu MUST be called.
+// Behavior is similar to sync.Mutex.
+func (l *localLocker) lockMu(ctx context.Context) (ok bool) {
+	select {
+	case l.mutex <- struct{}{}:
+		return true
+	case <-ctx.Done():
+		return false
+	}
+}
+
+// lockMuBlock will block while getting the mutex.
+// When returned unlockMu *must* be called.
+// Behavior is similar to sync.Mutex.
+func (l *localLocker) lockMuBlock() {
+	l.mutex <- struct{}{}
+}
+
+// unlockMu unlocks an acquired mutex.
+// This may only be called once after successfully getting a mutex.
+func (l *localLocker) unlockMu() {
+	<-l.mutex
+}
+
 func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
 	if len(args.Resources) > maxDeleteList {
 		return false, fmt.Errorf("internal error: localLocker.Lock called with more than %d resources", maxDeleteList)
 	}
 
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+	if !l.lockMu(ctx) {
+		return false, ctx.Err()
+	}
+	defer l.unlockMu()
 
 	if !l.canTakeLock(args.Resources...) {
 		// Not all locks can be taken on resources,
@@ -115,8 +142,9 @@ func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool
 		return false, fmt.Errorf("internal error: localLocker.Unlock called with more than %d resources", maxDeleteList)
 	}
 
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+	l.lockMuBlock()
+	defer l.unlockMu()
+
 	err = nil
 
 	for _, resource := range args.Resources {
@@ -163,8 +191,11 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 		return false, fmt.Errorf("internal error: localLocker.RLock called with more than one resource")
 	}
 
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+	if !l.lockMu(ctx) {
+		return false, ctx.Err()
+	}
+	defer l.unlockMu()
+
 	resource := args.Resources[0]
 	lrInfo := lockRequesterInfo{
 		Name:            resource,
@@ -196,8 +227,9 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
 		return false, fmt.Errorf("internal error: localLocker.RUnlock called with more than one resource")
 	}
 
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+	l.lockMuBlock()
+	defer l.unlockMu()
+
 	var lri []lockRequesterInfo
 
 	resource := args.Resources[0]
@@ -214,8 +246,8 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
 }
 
 func (l *localLocker) DupLockMap() map[string][]lockRequesterInfo {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+	l.lockMuBlock()
+	defer l.unlockMu()
 
 	lockCopy := make(map[string][]lockRequesterInfo, len(l.lockMap))
 	for k, v := range l.lockMap {
@@ -239,106 +271,99 @@ func (l *localLocker) IsLocal() bool {
 }
 
 func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
-	select {
-	case <-ctx.Done():
-		return false, ctx.Err()
-	default:
-		l.mutex.Lock()
-		defer l.mutex.Unlock()
-		if len(args.UID) == 0 {
-			for _, resource := range args.Resources {
-				lris, ok := l.lockMap[resource]
-				if !ok {
-					continue
-				}
-				// Collect uids, so we don't mutate while we delete
-				uids := make([]string, 0, len(lris))
-				for _, lri := range lris {
-					uids = append(uids, lri.UID)
-				}
+	l.lockMuBlock()
+	defer l.unlockMu()
 
-				// Delete collected uids:
-				for _, uid := range uids {
-					lris, ok := l.lockMap[resource]
-					if !ok {
-						// Just to be safe, delete uuids.
-						for idx := 0; idx < maxDeleteList; idx++ {
-							mapID := formatUUID(uid, idx)
-							if _, ok := l.lockUID[mapID]; !ok {
-								break
-							}
-							delete(l.lockUID, mapID)
-						}
-						continue
-					}
-					l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
-				}
-			}
-			return true, nil
-		}
-
-		idx := 0
-		for {
-			mapID := formatUUID(args.UID, idx)
-			resource, ok := l.lockUID[mapID]
-			if !ok {
-				return idx > 0, nil
-			}
+	if len(args.UID) == 0 {
+		for _, resource := range args.Resources {
 			lris, ok := l.lockMap[resource]
 			if !ok {
-				// Unexpected  inconsistency, delete.
-				delete(l.lockUID, mapID)
-				idx++
 				continue
 			}
-			reply = true
-			l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
-			idx++
+			// Collect uids, so we don't mutate while we delete
+			uids := make([]string, 0, len(lris))
+			for _, lri := range lris {
+				uids = append(uids, lri.UID)
+			}
+
+			// Delete collected uids:
+			for _, uid := range uids {
+				lris, ok := l.lockMap[resource]
+				if !ok {
+					// Just to be safe, delete uuids.
+					for idx := 0; idx < maxDeleteList; idx++ {
+						mapID := formatUUID(uid, idx)
+						if _, ok := l.lockUID[mapID]; !ok {
+							break
+						}
+						delete(l.lockUID, mapID)
+					}
+					continue
+				}
+				l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
+			}
 		}
+		return true, nil
+	}
+
+	idx := 0
+	for {
+		mapID := formatUUID(args.UID, idx)
+		resource, ok := l.lockUID[mapID]
+		if !ok {
+			return idx > 0, nil
+		}
+		lris, ok := l.lockMap[resource]
+		if !ok {
+			// Unexpected  inconsistency, delete.
+			delete(l.lockUID, mapID)
+			idx++
+			continue
+		}
+		reply = true
+		l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
+		idx++
 	}
 }
 
 func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refreshed bool, err error) {
-	select {
-	case <-ctx.Done():
+	if !l.lockMu(ctx) {
 		return false, ctx.Err()
-	default:
-		l.mutex.Lock()
-		defer l.mutex.Unlock()
+	}
+	defer l.unlockMu()
 
-		// Check whether uid is still active.
-		resource, ok := l.lockUID[formatUUID(args.UID, 0)]
+	// Check whether uid is still active.
+	resource, ok := l.lockUID[formatUUID(args.UID, 0)]
+	if !ok {
+		return false, nil
+	}
+	idx := 0
+	for {
+		lris, ok := l.lockMap[resource]
 		if !ok {
-			return false, nil
+			// Inconsistent. Delete UID.
+			delete(l.lockUID, formatUUID(args.UID, idx))
+			return idx > 0, nil
 		}
-		idx := 0
-		for {
-			lris, ok := l.lockMap[resource]
-			if !ok {
-				// Inconsistent. Delete UID.
-				delete(l.lockUID, formatUUID(args.UID, idx))
-				return idx > 0, nil
-			}
-			for i := range lris {
-				if lris[i].UID == args.UID {
-					lris[i].TimeLastRefresh = UTCNow()
-				}
-			}
-			idx++
-			resource, ok = l.lockUID[formatUUID(args.UID, idx)]
-			if !ok {
-				// No more resources for UID, but we did update at least one.
-				return true, nil
+		for i := range lris {
+			if lris[i].UID == args.UID {
+				lris[i].TimeLastRefresh = UTCNow()
 			}
 		}
+		idx++
+		resource, ok = l.lockUID[formatUUID(args.UID, idx)]
+		if !ok {
+			// No more resources for UID, but we did update at least one.
+			return true, nil
+		}
 	}
 }
 
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
 // Caller must hold 'l.mutex' lock.
 func (l *localLocker) expireOldLocks(interval time.Duration) {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
+	l.lockMuBlock()
+	defer l.unlockMu()
 
 	for k, lris := range l.lockMap {
 		modified := false
@@ -369,6 +394,7 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 
 func newLocker() *localLocker {
 	return &localLocker{
+		mutex:   make(chan struct{}, 1),
 		lockMap: make(map[string][]lockRequesterInfo, 1000),
 		lockUID: make(map[string]string, 1000),
 	}

cmd/lock-rest-server-common_test.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"os"
 	"reflect"
-	"sync"
 	"testing"
 
 	"github.com/minio/minio/internal/dsync"
@@ -38,10 +37,7 @@ func createLockTestServer(ctx context.Context, t *testing.T) (string, *lockRESTS
 	}
 
 	locker := &lockRESTServer{
-		ll: &localLocker{
-			mutex:   sync.Mutex{},
-			lockMap: make(map[string][]lockRequesterInfo),
-		},
+		ll: newLocker(),
 	}
 	creds := globalActiveCred
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")

cmd/metacache-entries.go

@@ -189,6 +189,43 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
 	return xlMeta.versions[0].header.Type == DeleteType
 }
 
+// isAllFreeVersions returns if all objects are free versions.
+// If metadata is NOT versioned false will always be returned.
+// If v2 and UNABLE to load metadata true will be returned.
+func (e *metaCacheEntry) isAllFreeVersions() bool {
+	if e.cached != nil {
+		if len(e.cached.versions) == 0 {
+			return true
+		}
+		for _, v := range e.cached.versions {
+			if !v.header.FreeVersion() {
+				return false
+			}
+		}
+		return true
+	}
+	if !isXL2V1Format(e.metadata) {
+		return false
+	}
+	if meta, _, err := isIndexedMetaV2(e.metadata); meta != nil {
+		return meta.AllHidden(false)
+	} else if err != nil {
+		return true
+	}
+	// Fall back...
+	xlMeta, err := e.xlmeta()
+	if err != nil || len(xlMeta.versions) == 0 {
+		return true
+	}
+	// Check versions..
+	for _, v := range e.cached.versions {
+		if !v.header.FreeVersion() {
+			return false
+		}
+	}
+	return true
+}
+
 // fileInfo returns the decoded metadata.
 // If entry is a directory it is returned as that.
 // If versioned the latest version will be returned.

cmd/metacache-server-pool.go

@@ -96,7 +96,9 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 
 	// Decode and get the optional list id from the marker.
 	o.parseMarker()
-	o.BaseDir = baseDirFromPrefix(o.Prefix)
+	if o.BaseDir == "" {
+		o.BaseDir = baseDirFromPrefix(o.Prefix)
+	}
 	o.Transient = o.Transient || isReservedOrInvalidBucket(o.Bucket, false)
 	o.SetFilter()
 	if o.Transient {
@@ -314,11 +316,11 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	}
 
 	for _, err := range errs {
-		if err == nil || contextCanceled(ctx) {
-			allAtEOF = false
+		if errors.Is(err, io.EOF) {
 			continue
 		}
-		if err.Error() == io.EOF.Error() {
+		if err == nil || contextCanceled(ctx) || errors.Is(err, context.Canceled) {
+			allAtEOF = false
 			continue
 		}
 		logger.LogIf(ctx, err)
@@ -377,7 +379,12 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 		case <-ctx.Done():
 			return
 		case out <- obj:
-			queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
+			if fiv, err := obj.fileInfoVersions(o.Bucket); err == nil {
+				for _, version := range fiv.Versions {
+					objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
+					queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
+				}
+			}
 		}
 	}
 }

cmd/metacache-set.go

@@ -49,6 +49,7 @@ type listPathOptions struct {
 	Bucket string
 
 	// Directory inside the bucket.
+	// When unset listPath will set this based on Prefix
 	BaseDir string
 
 	// Scan/return only content with prefix.
@@ -367,12 +368,15 @@ func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSor
 				return true
 			}
 			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() {
-				return entries.len() < o.Limit
+				return true
+			}
+			if entry.isAllFreeVersions() {
+				return true
 			}
 			entries.o = append(entries.o, entry)
 			return entries.len() < o.Limit
 		})
-		if (err != nil && err.Error() == io.EOF.Error()) || pastPrefix || r.nextEOF() {
+		if (err != nil && errors.Is(err, io.EOF)) || pastPrefix || r.nextEOF() {
 			return entries, io.EOF
 		}
 		return entries, err
@@ -519,6 +523,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			}()
 
 			tmp := newMetacacheReader(pr)
+			defer tmp.Close()
 			e, err := tmp.filter(o)
 			pr.CloseWithError(err)
 			entries.o = append(entries.o, e.o...)
@@ -530,7 +535,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				// We stopped within the listing, we are done for now...
 				return entries, nil
 			}
-			if err != nil && err.Error() != io.EOF.Error() {
+			if err != nil && !errors.Is(err, io.EOF) {
 				switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) {
 				case ObjectNotFound:
 					retries++
@@ -888,6 +893,11 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	}
 	askDisks := len(disks)
 	readers := make([]*metacacheReader, askDisks)
+	defer func() {
+		for _, r := range readers {
+			r.Close()
+		}
+	}()
 	for i := range disks {
 		r, w := io.Pipe()
 		// Make sure we close the pipe so blocked writes doesn't stay around.
@@ -936,15 +946,6 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 				}
 			}
 			w.CloseWithError(werr)
-
-			if werr != io.EOF && werr != nil &&
-				werr.Error() != errFileNotFound.Error() &&
-				werr.Error() != errVolumeNotFound.Error() &&
-				werr.Error() != errDiskNotFound.Error() &&
-				werr.Error() != errUnformattedDisk.Error() &&
-				!errors.Is(werr, context.Canceled) {
-				logger.LogIf(ctx, werr)
-			}
 		}()
 	}
 

cmd/metacache-walk.go

@@ -93,7 +93,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		if opts.Limit <= 0 {
 			return
 		}
-		if m, _, _ := isIndexedMetaV2(metadata); m != nil && !m.IsLatestDeleteMarker() {
+		if m, _, _ := isIndexedMetaV2(metadata); m != nil && !m.AllHidden(true) {
 			objsReturned++
 		}
 	}
@@ -324,7 +324,12 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			case isSysErrNotDir(err):
 				// skip
 			default:
-				logger.LogIf(ctx, err)
+				// It is totally possible that xl.meta was overwritten
+				// while being concurrently listed at the same time in
+				// such scenarios the 'xl.meta' might get truncated
+				if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
+					logger.LogIf(ctx, err)
+				}
 			}
 		}
 

cmd/metrics-v2.go

@@ -27,7 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
@@ -1385,6 +1385,7 @@ func getIAMNodeMetrics() *MetricsGroup {
 			sinceLastSyncMillis = (uint64(time.Now().UnixNano()) - lastSyncTime) / uint64(time.Millisecond)
 		}
 
+		pluginAuthNMetrics := globalAuthNPlugin.Metrics()
 		metrics = []Metric{
 			{
 				Description: MetricDescription{
@@ -1426,7 +1427,68 @@ func getIAMNodeMetrics() *MetricsGroup {
 				},
 				Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures)),
 			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "plugin_authn_service_last_succ_seconds",
+					Help:      "When plugin authentication is configured, returns time (in seconds) since the last successful request to the service",
+					Type:      gaugeMetric,
+				},
+				Value: pluginAuthNMetrics.LastReachableSecs,
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "plugin_authn_service_last_fail_seconds",
+					Help:      "When plugin authentication is configured, returns time (in seconds) since the last failed request to the service",
+					Type:      gaugeMetric,
+				},
+				Value: pluginAuthNMetrics.LastUnreachableSecs,
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "plugin_authn_service_total_requests_minute",
+					Help:      "When plugin authentication is configured, returns total requests count in the last full minute",
+					Type:      gaugeMetric,
+				},
+				Value: float64(pluginAuthNMetrics.TotalRequests),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "plugin_authn_service_failed_requests_minute",
+					Help:      "When plugin authentication is configured, returns failed requests count in the last full minute",
+					Type:      gaugeMetric,
+				},
+				Value: float64(pluginAuthNMetrics.FailedRequests),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "plugin_authn_service_succ_avg_rtt_ms_minute",
+					Help:      "When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute",
+					Type:      gaugeMetric,
+				},
+				Value: pluginAuthNMetrics.AvgSuccRTTMs,
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "plugin_authn_service_succ_max_rtt_ms_minute",
+					Help:      "When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute",
+					Type:      gaugeMetric,
+				},
+				Value: pluginAuthNMetrics.MaxSuccRTTMs,
+			},
 		}
+
 		return metrics
 	})
 	return mg

cmd/metrics.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -616,14 +617,21 @@ func AuthMiddleware(h http.Handler) http.Handler {
 			writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), errAuthentication), r.URL)
 			return
 		}
+
+		cred := auth.Credentials{
+			AccessKey: claims.AccessKey,
+			Claims:    claims.Map(),
+			Groups:    groups,
+		}
+
 		// For authenticated users apply IAM policy.
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     claims.AccessKey,
-			Groups:          groups,
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
 			Action:          iampolicy.PrometheusAdminAction,
-			ConditionValues: getConditionValues(r, "", claims.AccessKey, claims.Map()),
+			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
-			Claims:          claims.Map(),
+			Claims:          cred.Claims,
 		}) {
 			if ok {
 				tc.FuncName = "handler.MetricsAuth"

cmd/notification-summary.go

@@ -32,6 +32,11 @@ func GetTotalCapacity(diskInfo []madmin.Disk) (capacity uint64) {
 // GetTotalUsableCapacity gets the total usable capacity in the cluster.
 func GetTotalUsableCapacity(diskInfo []madmin.Disk, s StorageInfo) (capacity uint64) {
 	for _, disk := range diskInfo {
+		// Ignore invalid.
+		if disk.PoolIndex < 0 || len(s.Backend.StandardSCData) <= disk.PoolIndex {
+			// https://github.com/minio/minio/issues/16500
+			continue
+		}
 		// Ignore parity disks
 		if disk.DiskIndex < s.Backend.StandardSCData[disk.PoolIndex] {
 			capacity += disk.TotalSpace
@@ -51,6 +56,11 @@ func GetTotalCapacityFree(diskInfo []madmin.Disk) (capacity uint64) {
 // GetTotalUsableCapacityFree gets the total usable capacity free in the cluster.
 func GetTotalUsableCapacityFree(diskInfo []madmin.Disk, s StorageInfo) (capacity uint64) {
 	for _, disk := range diskInfo {
+		// Ignore invalid.
+		if disk.PoolIndex < 0 || len(s.Backend.StandardSCData) <= disk.PoolIndex {
+			// https://github.com/minio/minio/issues/16500
+			continue
+		}
 		// Ignore parity disks
 		if disk.DiskIndex < s.Backend.StandardSCData[disk.PoolIndex] {
 			capacity += disk.AvailableSpace

cmd/notification.go

@@ -18,9 +18,7 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -29,7 +27,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/bits-and-blooms/bloom/v3"
 	"github.com/cespare/xxhash/v2"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v2"
@@ -384,75 +381,6 @@ func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerE
 	return ng.Wait()
 }
 
-// updateBloomFilter will cycle all servers to the current index and
-// return a merged bloom filter if a complete one can be retrieved.
-func (sys *NotificationSys) updateBloomFilter(ctx context.Context, current uint64) (*bloomFilter, error) {
-	req := bloomFilterRequest{
-		Current: current,
-		Oldest:  current - dataUsageUpdateDirCycles,
-	}
-	if current < dataUsageUpdateDirCycles {
-		req.Oldest = 0
-	}
-
-	// Load initial state from local...
-	var bf *bloomFilter
-	bfr, err := intDataUpdateTracker.cycleFilter(ctx, req)
-	logger.LogIf(ctx, err)
-	if err == nil && bfr.Complete {
-		nbf := intDataUpdateTracker.newBloomFilter()
-		bf = &nbf
-		_, err = bf.ReadFrom(bytes.NewReader(bfr.Filter))
-		logger.LogIf(ctx, err)
-	}
-
-	var mu sync.Mutex
-	g := errgroup.WithNErrs(len(sys.peerClients))
-	for idx, client := range sys.peerClients {
-		if client == nil {
-			continue
-		}
-		client := client
-		g.Go(func() error {
-			serverBF, err := client.cycleServerBloomFilter(ctx, req)
-			if false && intDataUpdateTracker.debug {
-				b, _ := json.MarshalIndent(serverBF, "", "  ")
-				logger.Info("Drive %v, Bloom filter: %v", client.host.Name, string(b))
-			}
-			// Keep lock while checking result.
-			mu.Lock()
-			defer mu.Unlock()
-
-			if err != nil || !serverBF.Complete || bf == nil {
-				logger.LogOnceIf(ctx, err, client.host.String(), client.cycleServerBloomFilter)
-				bf = nil
-				return nil
-			}
-
-			var tmp bloom.BloomFilter
-			_, err = tmp.ReadFrom(bytes.NewReader(serverBF.Filter))
-			if err != nil {
-				logger.LogIf(ctx, err)
-				bf = nil
-				return nil
-			}
-			if bf.BloomFilter == nil {
-				bf.BloomFilter = &tmp
-			} else {
-				err = bf.Merge(&tmp)
-				if err != nil {
-					logger.LogIf(ctx, err)
-					bf = nil
-					return nil
-				}
-			}
-			return nil
-		}, idx)
-	}
-	g.Wait()
-	return bf, nil
-}
-
 var errPeerNotReachable = errors.New("peer is not reachable")
 
 // GetLocks - makes GetLocks RPC call on all peers.

cmd/object-api-interface.go

@@ -195,7 +195,7 @@ type ObjectLayer interface {
 
 	// Storage operations.
 	Shutdown(context.Context) error
-	NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
+	NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
 	BackendInfo() madmin.BackendInfo
 	StorageInfo(ctx context.Context) StorageInfo
 	LocalStorageInfo(ctx context.Context) StorageInfo

cmd/object-handlers-common.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -165,6 +165,30 @@ func checkPreconditionsPUT(ctx context.Context, w http.ResponseWriter, r *http.R
 		}
 	}
 
+	// If-Match : Return the object only if its entity tag (ETag) is the same as the one specified;
+	// otherwise return a 412 (precondition failed).
+	ifMatchETagHeader := r.Header.Get(xhttp.IfMatch)
+	if ifMatchETagHeader != "" {
+		if !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
+			// If the object ETag does not match with the specified ETag.
+			writeHeaders()
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL)
+			return true
+		}
+	}
+
+	// If-None-Match : Return the object only if its entity tag (ETag) is different from the
+	// one specified otherwise, return a 304 (not modified).
+	ifNoneMatchETagHeader := r.Header.Get(xhttp.IfNoneMatch)
+	if ifNoneMatchETagHeader != "" {
+		if isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
+			// If the object ETag matches with the specified ETag.
+			writeHeaders()
+			w.WriteHeader(http.StatusNotModified)
+			return true
+		}
+	}
+
 	etagMatch := opts.PreserveETag != "" && isETagEqual(objInfo.ETag, opts.PreserveETag)
 	vidMatch := opts.VersionID != "" && opts.VersionID == objInfo.VersionID
 	mtimeMatch := !opts.MTime.IsZero() && objInfo.ModTime.Unix() >= opts.MTime.Unix()
@@ -350,7 +374,7 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 					Name:      dobj.ObjectName,
 					VersionID: dobj.VersionID,
 				},
-				Host: "Internal: [ILM-EXPIRY]",
+				Host: "Internal: [ILM-Expiry]",
 			})
 		}
 	}

cmd/object-handlers.go

@@ -40,6 +40,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/amztime"
+	"github.com/minio/minio/internal/auth"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
@@ -155,7 +156,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 			if globalPolicySys.IsAllowed(policy.Args{
 				Action:          policy.ListBucketAction,
 				BucketName:      bucket,
-				ConditionValues: getConditionValues(r, "", "", nil),
+				ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 				IsOwner:         false,
 			}) {
 				_, err = getObjectInfo(ctx, bucket, object, opts)
@@ -241,7 +242,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		},
 		actualSize,
 	)
-
+	defer objectRSC.Close()
 	s3Select, err := s3select.NewS3Select(r.Body)
 	if err != nil {
 		if serr, ok := err.(s3select.SelectError); ok {
@@ -346,7 +347,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			if globalPolicySys.IsAllowed(policy.Args{
 				Action:          policy.ListBucketAction,
 				BucketName:      bucket,
-				ConditionValues: getConditionValues(r, "", "", nil),
+				ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 				IsOwner:         false,
 			}) {
 				getObjectInfo := objectAPI.GetObjectInfo
@@ -388,9 +389,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRange), r.URL)
 			return
 		}
-		if rangeErr != nil {
-			logger.LogIf(ctx, rangeErr, logger.Application)
-		}
 	}
 
 	// Validate pre-conditions if any.
@@ -416,7 +414,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
 			if perr != nil {
 				proxyGetErr := ErrorRespToObjectError(perr, bucket, object)
-				if !isErrObjectNotFound(proxyGetErr) && !isErrVersionNotFound(proxyGetErr) &&
+				if !isErrBucketNotFound(proxyGetErr) && !isErrObjectNotFound(proxyGetErr) && !isErrVersionNotFound(proxyGetErr) &&
 					!isErrPreconditionFailed(proxyGetErr) && !isErrInvalidRange(proxyGetErr) {
 					logger.LogIf(ctx, fmt.Errorf("Proxying request (replication) failed for %s/%s(%s) - %w", bucket, object, opts.VersionID, perr))
 				}
@@ -647,7 +645,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 			if globalPolicySys.IsAllowed(policy.Args{
 				Action:          policy.ListBucketAction,
 				BucketName:      bucket,
-				ConditionValues: getConditionValues(r, "", "", nil),
+				ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 				IsOwner:         false,
 			}) {
 				getObjectInfo := objectAPI.GetObjectInfo
@@ -939,11 +937,16 @@ var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core,
 	cred := getReqAccessCred(r, globalSite.Region)
 	// In a federated deployment, all the instances share config files
 	// and hence expected to have same credentials.
-	return miniogo.NewCore(host, &miniogo.Options{
+	core, err := miniogo.NewCore(host, &miniogo.Options{
 		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, ""),
 		Secure:    globalIsTLS,
 		Transport: getRemoteInstanceTransport,
 	})
+	if err != nil {
+		return nil, err
+	}
+	core.SetAppInfo("minio-federated", ReleaseTag)
+	return core, nil
 }
 
 // Check if the destination bucket is on a remote site, this code only gets executed
@@ -1771,7 +1774,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 	opts.IndexCB = idxCb
 
-	if !opts.MTime.IsZero() && opts.PreserveETag != "" {
+	if (!opts.MTime.IsZero() && opts.PreserveETag != "") ||
+		r.Header.Get(xhttp.IfMatch) != "" ||
+		r.Header.Get(xhttp.IfNoneMatch) != "" {
 		opts.CheckPrecondFn = func(oi ObjectInfo) bool {
 			if _, err := DecryptObjectInfo(&oi, r); err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1913,7 +1918,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseHeadersOnly(w)
 
 	// Notify object created event.
-	sendEvent(eventArgs{
+	evt := eventArgs{
 		EventName:    event.ObjectCreatedPut,
 		BucketName:   bucket,
 		Object:       objInfo,
@@ -1921,7 +1926,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		RespElements: extractRespElements(w),
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
-	})
+	}
+	sendEvent(evt)
+	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
+		evt.EventName = event.ObjectManyVersions
+		sendEvent(evt)
+	}
 
 	// Remove the transitioned object whose object version is being overwritten.
 	if !globalTierConfigMgr.Empty() {
@@ -3054,7 +3064,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 		}
 	}
 	// set or upgrade restore expiry
-	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now(), rreq.Days)
+	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
 	metadata := cloneMSS(objInfo.UserDefined)
 
 	// update self with restore metadata
@@ -3119,7 +3129,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 				},
 				actualSize,
 			)
-
+			defer objectRSC.Close()
 			if err = rreq.SelectParameters.Open(objectRSC); err != nil {
 				if serr, ok := err.(s3select.SelectError); ok {
 					encodedErrorResponse := encodeResponse(APIErrorResponse{

cmd/object-multipart-handlers.go

@@ -1019,7 +1019,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
 	// Notify object created event.
-	sendEvent(eventArgs{
+	evt := eventArgs{
 		EventName:    event.ObjectCreatedCompleteMultipartUpload,
 		BucketName:   bucket,
 		Object:       objInfo,
@@ -1027,7 +1027,13 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		RespElements: extractRespElements(w),
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
-	})
+	}
+	sendEvent(evt)
+
+	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
+		evt.EventName = event.ObjectManyVersions
+		sendEvent(evt)
+	}
 
 	// Remove the transitioned object whose object version is being overwritten.
 	if !globalTierConfigMgr.Empty() {

cmd/peer-rest-client.go

@@ -34,8 +34,8 @@ import (
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/rest"
+	"github.com/minio/pkg/logger/message/log"
 	xnet "github.com/minio/pkg/net"
 	"github.com/tinylib/msgp/msgp"
 )
@@ -309,25 +309,6 @@ func (client *peerRESTClient) DeleteBucketMetadata(bucket string) error {
 	return nil
 }
 
-// cycleServerBloomFilter will cycle the bloom filter to start recording to index y if not already.
-// The response will contain a bloom filter starting at index x up to, but not including index y.
-// If y is 0, the response will not update y, but return the currently recorded information
-// from the current x to y-1.
-func (client *peerRESTClient) cycleServerBloomFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
-	var reader bytes.Buffer
-	err := gob.NewEncoder(&reader).Encode(req)
-	if err != nil {
-		return nil, err
-	}
-	respBody, err := client.callWithContext(ctx, peerRESTMethodCycleBloom, nil, &reader, -1)
-	if err != nil {
-		return nil, err
-	}
-	var resp bloomFilterResponse
-	defer xhttp.DrainBody(respBody)
-	return &resp, gob.NewDecoder(respBody).Decode(&resp)
-}
-
 // DeletePolicy - delete a specific canned policy.
 func (client *peerRESTClient) DeletePolicy(policyName string) (err error) {
 	values := make(url.Values)
@@ -838,14 +819,20 @@ func (client *peerRESTClient) GetPeerMetrics(ctx context.Context) (<-chan Metric
 	dec := gob.NewDecoder(respBody)
 	ch := make(chan Metric)
 	go func(ch chan<- Metric) {
+		defer func() {
+			xhttp.DrainBody(respBody)
+			close(ch)
+		}()
 		for {
 			var metric Metric
 			if err := dec.Decode(&metric); err != nil {
-				xhttp.DrainBody(respBody)
-				close(ch)
 				return
 			}
-			ch <- metric
+			select {
+			case <-ctx.Done():
+				return
+			case ch <- metric:
+			}
 		}
 	}(ch)
 	return ch, nil

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion = "v29" // Added LocalStorageInfo peer API
+	peerRESTVersion = "v30" // Removed bloom filter
 
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"

cmd/peer-rest-server.go

@@ -34,9 +34,9 @@ import (
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/mux"
+	"github.com/minio/pkg/logger/message/log"
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -631,30 +631,6 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
 	}
 }
 
-// CycleServerBloomFilterHandler cycles bloom filter on server.
-func (s *peerRESTServer) CycleServerBloomFilterHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		s.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	ctx := newContext(r, w, "CycleServerBloomFilter")
-
-	var req bloomFilterRequest
-	err := gob.NewDecoder(r.Body).Decode(&req)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-	bf, err := intDataUpdateTracker.cycleFilter(ctx, req)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(bf))
-}
-
 func (s *peerRESTServer) GetMetacacheListingHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
@@ -1390,7 +1366,6 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCycleBloom).HandlerFunc(httpTraceHdrs(server.CycleServerBloomFilterHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(httpTraceHdrs(server.GetAllBucketStatsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(httpTraceHdrs(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(httpTraceHdrs(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)

cmd/peer-s3-client.go

@@ -158,7 +158,7 @@ func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts Buc
 			if err != nil {
 				return err
 			}
-			bucketInfos[idx] = bucketInfo
+			bucketInfos[idx+1] = bucketInfo
 			return nil
 		}, idx)
 	}

cmd/peer-s3-server.go

@@ -33,7 +33,7 @@ const (
 	peerS3Version = "v1" // First implementation
 
 	peerS3VersionPrefix = SlashSeparator + peerS3Version
-	peerS3Prefix        = minioReservedBucketPath + "/peer"
+	peerS3Prefix        = minioReservedBucketPath + "/peer-s3"
 	peerS3Path          = peerS3Prefix + peerS3VersionPrefix
 )
 

cmd/prepare-storage.go

@@ -198,8 +198,15 @@ func connectLoadInitFormats(verboseLogging bool, firstDisk bool, endpoints Endpo
 		return nil, nil, err
 	}
 
+	// Return error when quorum unformatted disks - indicating we are
+	// waiting for first server to be online.
+	unformattedDisks := quorumUnformattedDisks(sErrs)
+	if unformattedDisks && !firstDisk {
+		return nil, nil, errNotFirstDisk
+	}
+
 	// All disks report unformatted we should initialized everyone.
-	if shouldInitErasureDisks(sErrs) && firstDisk {
+	if unformattedDisks && firstDisk {
 		logger.Info("Formatting %s pool, %v set(s), %v drives per set.",
 			humanize.Ordinal(poolCount), setCount, setDriveCount)
 
@@ -215,19 +222,6 @@ func connectLoadInitFormats(verboseLogging bool, firstDisk bool, endpoints Endpo
 		return storageDisks, format, nil
 	}
 
-	// Return error when quorum unformatted disks - indicating we are
-	// waiting for first server to be online.
-	unformattedDisks := quorumUnformattedDisks(sErrs)
-	if unformattedDisks && !firstDisk {
-		return nil, nil, errNotFirstDisk
-	}
-
-	// Return error when quorum unformatted disks but waiting for rest
-	// of the servers to be online.
-	if unformattedDisks && firstDisk {
-		return nil, nil, errFirstDiskWait
-	}
-
 	// Mark all root disks down
 	markRootDisksAsDown(storageDisks, sErrs)
 

cmd/rebalance-admin.go

@@ -61,6 +61,11 @@ func rebalanceStatus(ctx context.Context, z *erasureServerPools) (r rebalanceAdm
 		TotalSpace     uint64
 	}, len(z.serverPools))
 	for _, disk := range si.Disks {
+		// Ignore invalid.
+		if disk.PoolIndex < 0 || len(diskStats) <= disk.PoolIndex {
+			// https://github.com/minio/minio/issues/16500
+			continue
+		}
 		diskStats[disk.PoolIndex].AvailableSpace += disk.AvailableSpace
 		diskStats[disk.PoolIndex].TotalSpace += disk.TotalSpace
 	}

cmd/s3-zip-handlers.go

@@ -27,6 +27,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
@@ -105,7 +106,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 			if globalPolicySys.IsAllowed(policy.Args{
 				Action:          policy.ListBucketAction,
 				BucketName:      bucket,
-				ConditionValues: getConditionValues(r, "", "", nil),
+				ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 				IsOwner:         false,
 			}) {
 				_, err = getObjectInfo(ctx, bucket, zipPath, opts)
@@ -411,7 +412,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
 			if globalPolicySys.IsAllowed(policy.Args{
 				Action:          policy.ListBucketAction,
 				BucketName:      bucket,
-				ConditionValues: getConditionValues(r, "", "", nil),
+				ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 				IsOwner:         false,
 			}) {
 				_, err = getObjectInfo(ctx, bucket, zipPath, opts)

cmd/server-main.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -42,6 +43,7 @@ import (
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -221,6 +223,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	logger.FatalIf(err, "Invalid command line arguments")
 
 	globalLocalNodeName = GetLocalPeer(globalEndpoints, globalMinioHost, globalMinioPort)
+	nodeNameSum := sha256.Sum256([]byte(globalLocalNodeNameHex))
+	globalLocalNodeNameHex = hex.EncodeToString(nodeNameSum[:])
 
 	globalRemoteEndpoints = make(map[string]Endpoint)
 	for _, z := range globalEndpoints {
@@ -319,6 +323,7 @@ func initAllSubsystems(ctx context.Context) {
 	// Create new ILM tier configuration subsystem
 	globalTierConfigMgr = NewTierConfigMgr()
 
+	globalTransitionState = newTransitionState(GlobalContext)
 	globalSiteResyncMetrics = newSiteResyncMetrics(GlobalContext)
 }
 
@@ -461,7 +466,7 @@ func initConfigSubsystem(ctx context.Context, newObject ObjectLayer) error {
 		}
 
 		// Any other config errors we simply print a message and proceed forward.
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize config, some features may be missing %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize config, some features may be missing: %w", err))
 	}
 
 	return nil
@@ -664,50 +669,34 @@ func serverMain(ctx *cli.Context) {
 
 	// Background all other operations such as initializing bucket metadata etc.
 	go func() {
-		// Initialize transition tier configuration manager
-		initBackgroundReplication(GlobalContext, newObject)
-		initBackgroundTransition(GlobalContext, newObject)
+		// Initialize data scanner.
+		initDataScanner(GlobalContext, newObject)
 
+		// Initialize background replication
+		initBackgroundReplication(GlobalContext, newObject)
+
+		globalTransitionState.Init(newObject)
+
+		// Initialize batch job pool.
 		globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100)
 
+		// Initialize the license update job
+		initLicenseUpdateJob(GlobalContext, newObject)
+
 		go func() {
+			// Initialize transition tier configuration manager
 			err := globalTierConfigMgr.Init(GlobalContext, newObject)
 			if err != nil {
 				logger.LogIf(GlobalContext, err)
-			}
-
-			globalTierJournal, err = initTierDeletionJournal(GlobalContext)
-			if err != nil {
-				logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
+			} else {
+				globalTierJournal, err = initTierDeletionJournal(GlobalContext)
+				if err != nil {
+					logger.FatalIf(err, "Unable to initialize remote tier pending deletes journal")
+				}
 			}
 		}()
 
-		// Initialize quota manager.
-		globalBucketQuotaSys.Init(newObject)
-
-		initDataScanner(GlobalContext, newObject)
-
-		// List buckets to heal, and be re-used for loading configs.
-		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
-		if err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to heal: %w", err))
-		}
-		// initialize replication resync state.
-		go globalReplicationPool.initResync(GlobalContext, buckets, newObject)
-
-		// Populate existing buckets to the etcd backend
-		if globalDNSConfig != nil {
-			// Background this operation.
-			go initFederatorBackend(buckets, newObject)
-		}
-
-		// Initialize bucket metadata sub-system.
-		globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
-
-		// Initialize site replication manager.
-		globalSiteReplicationSys.Init(GlobalContext, newObject)
-
-		// Initialize bucket notification system
+		// Initialize bucket notification system first before loading bucket metadata.
 		logger.LogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject))
 
 		// initialize the new disk cache objects.
@@ -720,8 +709,28 @@ func serverMain(ctx *cli.Context) {
 			setCacheObjectLayer(cacheAPI)
 		}
 
-		// Initialize the license update job
-		initLicenseUpdateJob(GlobalContext, newObject)
+		// List buckets to heal, and be re-used for loading configs.
+		buckets, err := newObject.ListBuckets(GlobalContext, BucketOptions{})
+		if err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("Unable to list buckets to heal: %w", err))
+		}
+		// initialize replication resync state.
+		go globalReplicationPool.initResync(GlobalContext, buckets, newObject)
+
+		// Initialize bucket metadata sub-system.
+		globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
+
+		// Initialize site replication manager after bucket metadat
+		globalSiteReplicationSys.Init(GlobalContext, newObject)
+
+		// Initialize quota manager.
+		globalBucketQuotaSys.Init(newObject)
+
+		// Populate existing buckets to the etcd backend
+		if globalDNSConfig != nil {
+			// Background this operation.
+			go initFederatorBackend(buckets, newObject)
+		}
 
 		// Prints the formatted startup message, if err is not nil then it prints additional information as well.
 		printStartupMessage(getAPIEndpoints(), err)
@@ -744,6 +753,9 @@ func serverMain(ctx *cli.Context) {
 	})
 	logger.FatalIf(err, "Unable to initialize MinIO client")
 
+	// Add User-Agent to differentiate the requests.
+	globalMinioClient.SetAppInfo("minio-perf-test", ReleaseTag)
+
 	if serverDebugLog {
 		logger.Info("== DEBUG Mode enabled ==")
 		logger.Info("Currently set environment settings:")

cmd/service.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"os/exec"
+	"runtime"
 	"syscall"
 )
 
@@ -56,6 +57,19 @@ func initGlobalContext() {
 // deployed binary to be started. It returns the pid of the newly started
 // process when successful.
 func restartProcess() error {
+	if runtime.GOOS == globalWindowsOSName {
+		cmd := exec.Command(os.Args[0], os.Args[1:]...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Stdin = os.Stdin
+		cmd.Env = os.Environ()
+		err := cmd.Run()
+		if err == nil {
+			os.Exit(0)
+		}
+		return err
+	}
+
 	// Use the original binary location. This works with symlinks such that if
 	// the file it points to has been changed we will use the updated symlink.
 	argv0, err := exec.LookPath(os.Args[0])

cmd/shared-lock.go

@@ -63,6 +63,8 @@ func mergeContext(ctx1, ctx2 context.Context) (context.Context, context.CancelFu
 		select {
 		case <-ctx1.Done():
 		case <-ctx2.Done():
+		// The lock acquirer decides to cancel, exit this goroutine
+		case <-ctx.Done():
 		}
 
 		cancel()

cmd/signature-v4-parser.go

@@ -280,7 +280,7 @@ func parseSignV4(v4Auth string, region string, stype serviceType) (sv signValues
 	signV4Values := signValues{}
 
 	var s3Err APIErrorCode
-	// Save credentail values.
+	// Save credential values.
 	signV4Values.Credential, s3Err = parseCredentialHeader(strings.TrimSpace(credElement), region, stype)
 	if s3Err != ErrNone {
 		return sv, s3Err

cmd/storage-rest-client.go

@@ -50,9 +50,18 @@ func isNetworkError(err error) bool {
 		return xnet.IsNetworkOrHostDown(nerr.Err, false)
 	}
 
+	switch {
 	// A peer node can be in shut down phase and proactively
 	// return 503 server closed error,consider it as an offline node
-	return err.Error() == http.ErrServerClosed.Error()
+	case strings.Contains(err.Error(), http.ErrServerClosed.Error()):
+		return true
+	// Corner case, the server closed the connection with a keep-alive timeout
+	// some requests are not retried internally, such as POST request with written body
+	case strings.Contains(err.Error(), "server closed idle connection"):
+		return true
+	}
+
+	return false
 }
 
 // Converts network error to storageErr. This function is

cmd/sts-errors.go

@@ -33,7 +33,7 @@ func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, isErrCode
 		err = stsErrCodes.ToSTSErr(errCode)
 	}
 	if err.Code == "InternalError" || !isErrCodeSTS {
-		aerr := getAPIError(APIErrorCode(errCode))
+		aerr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, errCtxt))
 		if aerr.Code != "InternalError" {
 			err.Code = aerr.Code
 			err.Description = aerr.Description

cmd/sts-handlers.go

@@ -282,7 +282,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 
 	// Call hook for site replication.
 	if cred.ParentUser != globalActiveCred.AccessKey {
-		if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSTSAcc,
 			STSCredential: &madmin.SRSTSCredential{
 				AccessKey:    cred.AccessKey,
@@ -291,9 +291,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 				ParentUser:   cred.ParentUser,
 			},
 			UpdatedAt: updatedAt,
-		}); err != nil {
-			logger.LogIf(ctx, err)
-		}
+		}))
 	}
 
 	assumeRoleResponse := &AssumeRoleResponse{
@@ -363,7 +361,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Validate JWT; check clientID in claims matches the one associated with the roleArn
-	if err := globalOpenIDConfig.Validate(roleArn, token, accessToken, r.Form.Get(stsDurationSeconds), claims); err != nil {
+	if err := globalIAMSys.OpenIDConfig.Validate(r.Context(), roleArn, token, accessToken, r.Form.Get(stsDurationSeconds), claims); err != nil {
 		switch err {
 		case openid.ErrTokenExpired:
 			switch action {
@@ -484,7 +482,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Call hook for site replication.
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemSTSAcc,
 		STSCredential: &madmin.SRSTSCredential{
 			AccessKey:           cred.AccessKey,
@@ -494,9 +492,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 			ParentPolicyMapping: policyName,
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		logger.LogIf(ctx, err)
-	}
+	}))
 
 	var encodedSuccessResponse []byte
 	switch action {
@@ -602,7 +598,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		}
 	}
 
-	ldapUserDN, groupDistNames, err := globalLDAPConfig.Bind(ldapUsername, ldapPassword)
+	ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(ldapUsername, ldapPassword)
 	if err != nil {
 		err = fmt.Errorf("LDAP server error: %w", err)
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
@@ -618,7 +614,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		return
 	}
 
-	expiryDur, err := globalLDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
+	expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
 		return
@@ -657,7 +653,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	}
 
 	// Call hook for site replication.
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemSTSAcc,
 		STSCredential: &madmin.SRSTSCredential{
 			AccessKey:    cred.AccessKey,
@@ -666,9 +662,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 			ParentUser:   cred.ParentUser,
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		logger.LogIf(ctx, err)
-	}
+	}))
 
 	ldapIdentityResponse := &AssumeRoleWithLDAPResponse{
 		Result: LDAPIdentityResult{
@@ -692,7 +686,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	claims := make(map[string]interface{})
 	defer logger.AuditLog(ctx, w, r, claims)
 
-	if !globalSTSTLSConfig.Enabled {
+	if !globalIAMSys.STSTLSConfig.Enabled {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCertificate' is disabled"))
 		return
 	}
@@ -733,7 +727,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	}
 
 	certificate := r.TLS.PeerCertificates[0]
-	if !globalSTSTLSConfig.InsecureSkipVerify { // Verify whether the client certificate has been issued by a trusted CA.
+	if !globalIAMSys.STSTLSConfig.InsecureSkipVerify { // Verify whether the client certificate has been issued by a trusted CA.
 		_, err := certificate.Verify(x509.VerifyOptions{
 			KeyUsages: []x509.ExtKeyUsage{
 				x509.ExtKeyUsageClientAuth,
@@ -782,7 +776,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 		return
 	}
 
-	expiry, err := globalSTSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
+	expiry, err := globalIAMSys.STSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, err)
 		return
@@ -820,7 +814,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	}
 
 	// Call hook for site replication.
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemSTSAcc,
 		STSCredential: &madmin.SRSTSCredential{
 			AccessKey:           tmpCredentials.AccessKey,
@@ -830,9 +824,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 			ParentPolicyMapping: policyName,
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		logger.LogIf(ctx, err)
-	}
+	}))
 
 	response := new(AssumeRoleWithCertificateResponse)
 	response.Result.Credentials = tmpCredentials
@@ -943,7 +935,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 	}
 
 	// Call hook for site replication.
-	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemSTSAcc,
 		STSCredential: &madmin.SRSTSCredential{
 			AccessKey:    tmpCredentials.AccessKey,
@@ -952,10 +944,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 			ParentUser:   tmpCredentials.ParentUser,
 		},
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	response := new(AssumeRoleWithCustomTokenResponse)
 	response.Result.Credentials = tmpCredentials

cmd/sts-handlers_test.go

@@ -36,7 +36,9 @@ func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) {
 	// The STS for root test needs to be the first one after setup.
 	suite.TestSTSForRoot(c)
 	suite.TestSTS(c)
+	suite.TestSTSWithDenyDeleteVersion(c)
 	suite.TestSTSWithTags(c)
+	suite.TestSTSServiceAccountsWithUsername(c)
 	suite.TestSTSWithGroupPolicy(c)
 	suite.TearDownSuite(c)
 }
@@ -73,6 +75,194 @@ func TestIAMInternalIDPSTSServerSuite(t *testing.T) {
 	}
 }
 
+func (s *TestSuiteIAM) TestSTSServiceAccountsWithUsername(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	bucket := "dillon-bucket"
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket create error: %v", err)
+	}
+
+	// Create policy
+	policy := "mypolicy-username"
+	policyBytes := []byte(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:*"
+   ],
+   "Resource": [
+    "arn:aws:s3:::${aws:username}-*"
+   ]
+  }
+ ]
+}`)
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	if err = s.adm.AddUser(ctx, "dillon", "dillon-123"); err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, policy, "dillon", false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	assumeRole := cr.STSAssumeRole{
+		Client:      s.TestSuiteCommon.client,
+		STSEndpoint: s.endPoint,
+		Options: cr.STSAssumeRoleOptions{
+			AccessKey: "dillon",
+			SecretKey: "dillon-123",
+			Location:  "",
+		},
+	}
+
+	value, err := assumeRole.Retrieve()
+	if err != nil {
+		c.Fatalf("Expected to generate STS creds, got err: %#v", err)
+	}
+
+	// Check that the LDAP sts cred is actually working.
+	minioClient, err := minio.New(s.endpoint, &minio.Options{
+		Creds:     cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("Error initializing client: %v", err)
+	}
+
+	// Validate that the client from sts creds can access the bucket.
+	c.mustListObjects(ctx, minioClient, bucket)
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Create svc acc
+	cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient)
+
+	svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+
+	// 1. Check S3 access for service account ListObjects()
+	c.mustListObjects(ctx, svcClient, bucket)
+
+	// 2. Check S3 access for upload
+	c.mustUpload(ctx, svcClient, bucket)
+
+	// 3. Check S3 access for download
+	c.mustDownload(ctx, svcClient, bucket)
+}
+
+func (s *TestSuiteIAM) TestSTSWithDenyDeleteVersion(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ObjectLocking: true})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Create policy, user and associate policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+  "Version": "2012-10-17",
+  "Statement": [
+   {
+    "Sid": "ObjectActionsRW",
+    "Effect": "Allow",
+    "Action": [
+     "s3:PutObject",
+     "s3:PutObjectTagging",
+     "s3:AbortMultipartUpload",
+     "s3:DeleteObject",
+     "s3:GetObject",
+     "s3:GetObjectTagging",
+     "s3:GetObjectVersion",
+     "s3:ListMultipartUploadParts"
+    ],
+    "Resource": [
+     "arn:aws:s3:::%s/*"
+    ]
+   },
+   {
+    "Sid": "DenyDeleteVersionAction",
+    "Effect": "Deny",
+    "Action": [
+     "s3:DeleteObjectVersion"
+    ],
+    "Resource": [
+     "arn:aws:s3:::%s/*"
+    ]
+   }
+  ]
+ }
+`, bucket, bucket))
+
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	// confirm that the user is able to access the bucket
+	uClient := s.getUserClient(c, accessKey, secretKey, "")
+	versions := c.mustUploadReturnVersions(ctx, uClient, bucket)
+	c.mustNotDelete(ctx, uClient, bucket, versions[0])
+
+	assumeRole := cr.STSAssumeRole{
+		Client:      s.TestSuiteCommon.client,
+		STSEndpoint: s.endPoint,
+		Options: cr.STSAssumeRoleOptions{
+			AccessKey: accessKey,
+			SecretKey: secretKey,
+			Location:  "",
+		},
+	}
+
+	value, err := assumeRole.Retrieve()
+	if err != nil {
+		c.Fatalf("err calling assumeRole: %v", err)
+	}
+
+	minioClient, err := minio.New(s.endpoint, &minio.Options{
+		Creds:     cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("Error initializing client: %v", err)
+	}
+
+	versions = c.mustUploadReturnVersions(ctx, minioClient, bucket)
+	c.mustNotDelete(ctx, minioClient, bucket, versions[0])
+}
+
 func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
@@ -171,9 +361,9 @@ func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
 	}
 
 	// Validate sts creds can access the object
-	c.mustPutObjectWithTags(ctx, uClient, bucket, object)
-	c.mustGetObject(ctx, uClient, bucket, object)
-	c.mustHeadObject(ctx, uClient, bucket, object, 2)
+	c.mustPutObjectWithTags(ctx, minioClient, bucket, object)
+	c.mustGetObject(ctx, minioClient, bucket, object)
+	c.mustHeadObject(ctx, minioClient, bucket, object, 2)
 
 	// Validate that the client can remove objects
 	if err = minioClient.RemoveObjectTagging(ctx, bucket, object, minio.RemoveObjectTaggingOptions{}); err != nil {
@@ -485,6 +675,7 @@ func TestIAMWithLDAPServerSuite(t *testing.T) {
 				suite.SetUpLDAP(c, ldapServer)
 				suite.TestLDAPSTS(c)
 				suite.TestLDAPSTSServiceAccounts(c)
+				suite.TestLDAPSTSServiceAccountsWithUsername(c)
 				suite.TestLDAPSTSServiceAccountsWithGroups(c)
 				suite.TearDownSuite(c)
 			},
@@ -726,6 +917,95 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccounts(c *check) {
 	c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
 }
 
+func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithUsername(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	bucket := "dillon"
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket create error: %v", err)
+	}
+
+	// Create policy
+	policy := "mypolicy-username"
+	policyBytes := []byte(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::${ldap:username}/*"
+   ]
+  }
+ ]
+}`)
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
+	err = s.adm.SetPolicy(ctx, policy, userDN, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	ldapID := cr.LDAPIdentity{
+		Client:       s.TestSuiteCommon.client,
+		STSEndpoint:  s.endPoint,
+		LDAPUsername: "dillon",
+		LDAPPassword: "dillon",
+	}
+
+	value, err := ldapID.Retrieve()
+	if err != nil {
+		c.Fatalf("Expected to generate STS creds, got err: %#v", err)
+	}
+
+	// Check that the LDAP sts cred is actually working.
+	minioClient, err := minio.New(s.endpoint, &minio.Options{
+		Creds:     cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure:    s.secure,
+		Transport: s.TestSuiteCommon.client.Transport,
+	})
+	if err != nil {
+		c.Fatalf("Error initializing client: %v", err)
+	}
+
+	// Validate that the client from sts creds can access the bucket.
+	c.mustListObjects(ctx, minioClient, bucket)
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Create svc acc
+	cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient)
+
+	svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+
+	// 1. Check S3 access for service account ListObjects()
+	c.mustListObjects(ctx, svcClient, bucket)
+
+	// 2. Check S3 access for upload
+	c.mustUpload(ctx, svcClient, bucket)
+
+	// 3. Check S3 access for download
+	c.mustDownload(ctx, svcClient, bucket)
+}
+
 // In this test, the parent users gets their permissions from a group, rather
 // than having a policy set directly on them.
 func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithGroups(c *check) {

cmd/tier-handlers.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"strconv"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go/v2"
@@ -94,6 +95,11 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	var ignoreInUse bool
+	if forceStr := r.Form.Get("force"); forceStr != "" {
+		ignoreInUse, _ = strconv.ParseBool(forceStr)
+	}
+
 	// Disallow remote tiers with internal storage class names
 	switch cfg.Name {
 	case storageclass.STANDARD, storageclass.RRS:
@@ -107,7 +113,7 @@ func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	err = globalTierConfigMgr.Add(ctx, cfg)
+	err = globalTierConfigMgr.Add(ctx, cfg, ignoreInUse)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return