fix panic in batch replicate (#20014)

Fixes:

```
panic: send on closed channel
	panic: close of closed channel

goroutine 878 [running]:
github.com/minio/minio/internal/ioutil.SafeClose[...](...)
	/Users/kp/code/src/github.com/minio/minio/internal/ioutil/ioutil.go:407
github.com/minio/minio/cmd.(*erasureServerPools).Walk.func2.2()
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:2229 +0xc0
panic({0x108c25e60?, 0x1090b28d0?})
	/usr/local/go/src/runtime/panic.go:770 +0x124
github.com/minio/minio/cmd.(*erasureServerPools).Walk.func2.3({{0x1400e397316, 0x5}, {0x1400d88b8a8, 0x8}, {0x1f99d80, 0xede101c42, 0x0}, 0x3bc, 0x0, 0x0, ...})
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:2235 +0xb4
github.com/minio/minio/cmd.(*erasureServerPools).Walk.func2()
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:2277 +0xabc
created by github.com/minio/minio/cmd.(*erasureServerPools).Walk in goroutine 575
	/Users/kp/code/src/github.com/minio/minio/cmd/erasure-server-pool.go:2210 +0x33c
```

Makefile

@@ -86,9 +86,9 @@ test-race: verifiers build ## builds minio, runs linters, tests (race)
 
 test-iam: install-race ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue,dev -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -timeout 15m -tags kqueue,dev -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue,dev -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 15m -race -tags kqueue,dev -v -run TestIAM* ./cmd
 
 test-iam-ldap-upgrade-import: install-race ## verify IAM (external LDAP IDP)
 	@echo "Running upgrade tests for IAM (LDAP backend)"

cmd/admin-handlers-pools.go

@@ -374,6 +374,7 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
 	globalNotificationSys.StopRebalance(r.Context())
 	writeSuccessResponseHeadersOnly(w)
 	adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	globalNotificationSys.LoadRebalanceMeta(ctx, false)
 }
 
 func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w http.ResponseWriter, r *http.Request) (proxy bool) {

cmd/admin-handlers-users-race_test.go

@@ -120,9 +120,12 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
 			c.Fatalf("Unable to set user: %v", err)
 		}
 
-		err = s.adm.SetPolicy(ctx, policy, accessKey, false)
-		if err != nil {
-			c.Fatalf("Unable to set policy: %v", err)
+		userReq := madmin.PolicyAssociationReq{
+			Policies: []string{policy},
+			User:     accessKey,
+		}
+		if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil {
+			c.Fatalf("Unable to attach policy: %v", err)
 		}
 
 		accessKeys[i] = accessKey

cmd/admin-handlers-users_test.go

@@ -239,9 +239,12 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	c.Assert(v.Status, madmin.AccountEnabled)
 
 	// 3. Associate policy and check that user can access
-	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{"readwrite"},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	client := s.getUserClient(c, accessKey, secretKey, "")
@@ -348,9 +351,12 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
 	}
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 2.3 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -470,9 +476,12 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 	c.mustNotListObjects(ctx, uClient, "testbucket")
 
 	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy1, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy1},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	admClnt := s.getAdminClient(c, accessKey, secretKey, "")
@@ -490,10 +499,22 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 		c.Fatalf("policy was missing!")
 	}
 
-	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy2, accessKey, false)
+	// Detach policy1 to set up for policy2
+	_, err = s.adm.DetachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy1},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to detach policy: %v", err)
+	}
+
+	// 3.2 associate policy to user
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy2},
+		User:     accessKey,
+	})
+	if err != nil {
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// 3.3 check user can create service account implicitly.
@@ -571,9 +592,12 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
 	c.mustNotListObjects(ctx, uClient, bucket)
 
 	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 3.3 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -726,9 +750,12 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
 	c.mustNotListObjects(ctx, uClient, bucket)
 
 	// 3. Associate policy to group and check user got access.
-	err = s.adm.SetPolicy(ctx, policy, group, true)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		Group:    group,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 3.1 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -871,9 +898,12 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// Create an madmin client with user creds
@@ -952,9 +982,12 @@ func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// Create an madmin client with user creds
@@ -1031,9 +1064,12 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// 1. Create a service account for the user

cmd/api-response.go

@@ -946,10 +946,20 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 
 // writeErrorResponse writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
-	if err.HTTPStatusCode == http.StatusServiceUnavailable {
-		// Set retry-after header to indicate user-agents to retry request after 120secs.
+	switch err.HTTPStatusCode {
+	case http.StatusServiceUnavailable:
+		// Set retry-after header to indicate user-agents to retry request after 60 seconds.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set(xhttp.RetryAfter, "120")
+		w.Header().Set(xhttp.RetryAfter, "60")
+	case http.StatusTooManyRequests:
+		_, deadline := globalAPIConfig.getRequestsPool()
+		if deadline <= 0 {
+			// Set retry-after header to indicate user-agents to retry request after 10 seconds.
+			// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+			w.Header().Set(xhttp.RetryAfter, "10")
+		} else {
+			w.Header().Set(xhttp.RetryAfter, strconv.Itoa(int(deadline.Seconds())))
+		}
 	}
 
 	switch err.Code {

cmd/batch-handlers.go

@@ -1071,86 +1071,84 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
-	var (
-		walkCh = make(chan itemOrErr[ObjectInfo], 100)
-		slowCh = make(chan itemOrErr[ObjectInfo], 100)
-	)
-
-	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
-		go func() {
-			// Snowball currently needs the high level minio-go Client, not the Core one
-			cl, err := miniogo.New(u.Host, &miniogo.Options{
-				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
-				Secure:       u.Scheme == "https",
-				Transport:    getRemoteInstanceTransport(),
-				BucketLookup: lookupStyle(r.Target.Path),
-			})
-			if err != nil {
-				batchLogIf(ctx, err)
-				return
-			}
-
-			// Already validated before arriving here
-			smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
-
-			batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
-			writeFn := func(batch []ObjectInfo) {
-				if len(batch) > 0 {
-					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
-						batchLogIf(ctx, err)
-						for _, b := range batch {
-							slowCh <- itemOrErr[ObjectInfo]{Item: b}
-						}
-					} else {
-						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
-						globalBatchJobsMetrics.save(job.ID, ri)
-						// persist in-memory state to disk after every 10secs.
-						batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-					}
-				}
-			}
-			for obj := range walkCh {
-				if obj.Item.DeleteMarker || !obj.Item.VersionPurgeStatus.Empty() || obj.Item.Size >= int64(smallerThan) {
-					slowCh <- obj
-					continue
-				}
-
-				batch = append(batch, obj.Item)
-
-				if len(batch) < *r.Source.Snowball.Batch {
-					continue
-				}
-				writeFn(batch)
-				batch = batch[:0]
-			}
-			writeFn(batch)
-			xioutil.SafeClose(slowCh)
-		}()
-	} else {
-		slowCh = walkCh
-	}
-
-	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
-	if err != nil {
-		return err
-	}
-
-	wk, err := workers.New(workerSize)
-	if err != nil {
-		// invalid worker size.
-		return err
-	}
-
-	walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict")
-	if walkQuorum == "" {
-		walkQuorum = "strict"
-	}
-
 	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
+		var (
+			walkCh = make(chan itemOrErr[ObjectInfo], 100)
+			slowCh = make(chan itemOrErr[ObjectInfo], 100)
+		)
 
+		if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
+			go func() {
+				// Snowball currently needs the high level minio-go Client, not the Core one
+				cl, err := miniogo.New(u.Host, &miniogo.Options{
+					Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+					Secure:       u.Scheme == "https",
+					Transport:    getRemoteInstanceTransport(),
+					BucketLookup: lookupStyle(r.Target.Path),
+				})
+				if err != nil {
+					batchLogIf(ctx, err)
+					return
+				}
+
+				// Already validated before arriving here
+				smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
+
+				batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
+				writeFn := func(batch []ObjectInfo) {
+					if len(batch) > 0 {
+						if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
+							batchLogIf(ctx, err)
+							for _, b := range batch {
+								slowCh <- itemOrErr[ObjectInfo]{Item: b}
+							}
+						} else {
+							ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
+							globalBatchJobsMetrics.save(job.ID, ri)
+							// persist in-memory state to disk after every 10secs.
+							batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+						}
+					}
+				}
+				for obj := range walkCh {
+					if obj.Item.DeleteMarker || !obj.Item.VersionPurgeStatus.Empty() || obj.Item.Size >= int64(smallerThan) {
+						slowCh <- obj
+						continue
+					}
+
+					batch = append(batch, obj.Item)
+
+					if len(batch) < *r.Source.Snowball.Batch {
+						continue
+					}
+					writeFn(batch)
+					batch = batch[:0]
+				}
+				writeFn(batch)
+				xioutil.SafeClose(slowCh)
+			}()
+		} else {
+			slowCh = walkCh
+		}
+
+		workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+		if err != nil {
+			return err
+		}
+
+		wk, err := workers.New(workerSize)
+		if err != nil {
+			// invalid worker size.
+			return err
+		}
+
+		walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict")
+		if walkQuorum == "" {
+			walkQuorum = "strict"
+		}
 		ctx, cancel := context.WithCancel(ctx)
 		// one of source/target is s3, skip delete marker and all versions under the same object name.
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3

cmd/bootstrap-peer-server.go

@@ -106,14 +106,17 @@ func (s1 *ServerSystemConfig) Diff(s2 *ServerSystemConfig) error {
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":          {},
-	"MINIO_CERT_PASSWD":   {},
-	"MINIO_SERVER_DEBUG":  {},
-	"MINIO_DSYNC_TRACE":   {},
-	"MINIO_ROOT_USER":     {},
-	"MINIO_ROOT_PASSWORD": {},
-	"MINIO_ACCESS_KEY":    {},
-	"MINIO_SECRET_KEY":    {},
+	"MINIO_OPTS":                   {},
+	"MINIO_CERT_PASSWD":            {},
+	"MINIO_SERVER_DEBUG":           {},
+	"MINIO_DSYNC_TRACE":            {},
+	"MINIO_ROOT_USER":              {},
+	"MINIO_ROOT_PASSWORD":          {},
+	"MINIO_ACCESS_KEY":             {},
+	"MINIO_SECRET_KEY":             {},
+	"MINIO_OPERATOR_VERSION":       {},
+	"MINIO_VSPHERE_PLUGIN_VERSION": {},
+	"MINIO_CI_CD":                  {},
 }
 
 func getServerSystemCfg() *ServerSystemConfig {

cmd/common-main.go

@@ -834,7 +834,7 @@ func serverHandleEnvVars() {
 		}
 	}
 
-	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
+	globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
 }
 
 func loadRootCredentials() {

cmd/encryption-v1.go

@@ -134,11 +134,16 @@ func DecryptETags(ctx context.Context, k *kms.KMS, objects []ObjectInfo) error {
 		SSES3SinglePartObjects := make(map[int]bool)
 		for i, object := range batch {
 			if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
-				SSES3SinglePartObjects[i] = true
-
-				metadata = append(metadata, object.UserDefined)
-				buckets = append(buckets, object.Bucket)
-				names = append(names, object.Name)
+				ETag, err := etag.Parse(object.ETag)
+				if err != nil {
+					continue
+				}
+				if ETag.IsEncrypted() {
+					SSES3SinglePartObjects[i] = true
+					metadata = append(metadata, object.UserDefined)
+					buckets = append(buckets, object.Bucket)
+					names = append(names, object.Name)
+				}
 			}
 		}
 
@@ -190,7 +195,7 @@ func DecryptETags(ctx context.Context, k *kms.KMS, objects []ObjectInfo) error {
 				if err != nil {
 					return err
 				}
-				if SSES3SinglePartObjects[i] && ETag.IsEncrypted() {
+				if SSES3SinglePartObjects[i] {
 					ETag, err = etag.Decrypt(keys[0][:], ETag)
 					if err != nil {
 						return err

cmd/erasure-server-pool-rebalance.go

@@ -119,11 +119,8 @@ func (z *erasureServerPools) loadRebalanceMeta(ctx context.Context) error {
 	}
 
 	z.rebalMu.Lock()
-	if len(r.PoolStats) == len(z.serverPools) {
-		z.rebalMeta = r
-	} else {
-		z.updateRebalanceStats(ctx)
-	}
+	z.rebalMeta = r
+	z.updateRebalanceStats(ctx)
 	z.rebalMu.Unlock()
 
 	return nil
@@ -147,24 +144,16 @@ func (z *erasureServerPools) updateRebalanceStats(ctx context.Context) error {
 		}
 	}
 	if ok {
-		lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
-		lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
-		if err != nil {
-			rebalanceLogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
-			return err
-		}
-		defer lock.Unlock(lkCtx)
-
-		ctx = lkCtx.Context()
-
-		noLockOpts := ObjectOptions{NoLock: true}
-		return z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
+		return z.rebalMeta.save(ctx, z.serverPools[0])
 	}
 
 	return nil
 }
 
 func (z *erasureServerPools) findIndex(index int) int {
+	if z.rebalMeta == nil {
+		return 0
+	}
 	for i := 0; i < len(z.rebalMeta.PoolStats); i++ {
 		if i == index {
 			return index
@@ -277,6 +266,10 @@ func (z *erasureServerPools) bucketRebalanceDone(bucket string, poolIdx int) {
 	z.rebalMu.Lock()
 	defer z.rebalMu.Unlock()
 
+	if z.rebalMeta == nil {
+		return
+	}
+
 	ps := z.rebalMeta.PoolStats[poolIdx]
 	if ps == nil {
 		return
@@ -331,6 +324,10 @@ func (r *rebalanceMeta) loadWithOpts(ctx context.Context, store objectIO, opts O
 }
 
 func (r *rebalanceMeta) saveWithOpts(ctx context.Context, store objectIO, opts ObjectOptions) error {
+	if r == nil {
+		return nil
+	}
+
 	data := make([]byte, 4, r.Msgsize()+4)
 
 	// Initialize the header.
@@ -353,8 +350,15 @@ func (z *erasureServerPools) IsRebalanceStarted() bool {
 	z.rebalMu.RLock()
 	defer z.rebalMu.RUnlock()
 
-	if r := z.rebalMeta; r != nil {
-		if r.StoppedAt.IsZero() {
+	r := z.rebalMeta
+	if r == nil {
+		return false
+	}
+	if !r.StoppedAt.IsZero() {
+		return false
+	}
+	for _, ps := range r.PoolStats {
+		if ps.Participating && ps.Info.Status != rebalCompleted {
 			return true
 		}
 	}
@@ -369,7 +373,7 @@ func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool {
 		if !r.StoppedAt.IsZero() {
 			return false
 		}
-		ps := z.rebalMeta.PoolStats[poolIndex]
+		ps := r.PoolStats[poolIndex]
 		return ps.Participating && ps.Info.Status == rebalStarted
 	}
 	return false
@@ -794,7 +798,9 @@ func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int
 	case rebalSaveStoppedAt:
 		r.StoppedAt = time.Now()
 	case rebalSaveStats:
-		r.PoolStats[poolIdx] = z.rebalMeta.PoolStats[poolIdx]
+		if z.rebalMeta != nil {
+			r.PoolStats[poolIdx] = z.rebalMeta.PoolStats[poolIdx]
+		}
 	}
 	z.rebalMeta = r
 

cmd/globals.go

@@ -449,8 +449,8 @@ var (
 	// dynamic sleeper for multipart expiration routine
 	deleteMultipartCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false)
 
-	// Is _MINIO_DISABLE_API_FREEZE_ON_BOOT set?
-	globalDisableFreezeOnBoot bool
+	// Is MINIO_SYNC_BOOT set?
+	globalEnableSyncBoot bool
 
 	// Contains NIC interface name used for internode communication
 	globalInternodeInterface     string

cmd/healthcheck-handler.go

@@ -29,14 +29,35 @@ import (
 
 const unavailable = "offline"
 
-// ClusterCheckHandler returns if the server is ready for requests.
-func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ClusterCheckHandler")
-
+func checkHealth(w http.ResponseWriter) ObjectLayer {
 	objLayer := newObjectLayerFn()
 	if objLayer == nil {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	if !globalBucketMetadataSys.Initialized() {
+		w.Header().Set(xhttp.MinIOServerStatus, "bucket-metadata-offline")
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	if !globalIAMSys.Initialized() {
+		w.Header().Set(xhttp.MinIOServerStatus, "iam-offline")
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	return objLayer
+}
+
+// ClusterCheckHandler returns if the server is ready for requests.
+func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterCheckHandler")
+
+	objLayer := checkHealth(w)
+	if objLayer == nil {
 		return
 	}
 
@@ -72,10 +93,8 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
-	objLayer := newObjectLayerFn()
+	objLayer := checkHealth(w)
 	if objLayer == nil {
-		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
-		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
 

cmd/iam-object-store.go

@@ -457,6 +457,8 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 
 	bootstrapTraceMsgFirstTime("loading all IAM items")
 
+	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+
 	listStartTime := UTCNow()
 	listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
 	if err != nil {
@@ -485,7 +487,6 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	if took := time.Since(policyLoadStartTime); took > maxIAMLoadOpTime {
 		logger.Info("Policy docs load took %.2fs (for %d items)", took.Seconds(), len(policiesList))
 	}
-	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
 
 	if iamOS.usersSysType == MinIOUsersSysType {
 		bootstrapTraceMsgFirstTime("loading regular IAM users")

cmd/iam-store.go

@@ -431,8 +431,41 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		}
 	}
 
-	// returned policy could be empty
-	policies := mp.toSlice()
+	// returned policy could be empty, we use set to de-duplicate.
+	policies := set.CreateStringSet(mp.toSlice()...)
+
+	for _, group := range u.Credentials.Groups {
+		if store.getUsersSysType() == MinIOUsersSysType {
+			g, ok := c.iamGroupsMap[group]
+			if !ok {
+				if err := store.loadGroup(context.Background(), group, c.iamGroupsMap); err != nil {
+					return nil, time.Time{}, err
+				}
+				g, ok = c.iamGroupsMap[group]
+				if !ok {
+					return nil, time.Time{}, errNoSuchGroup
+				}
+			}
+
+			// Group is disabled, so we return no policy - this
+			// ensures the request is denied.
+			if g.Status == statusDisabled {
+				return nil, time.Time{}, nil
+			}
+		}
+
+		policy, ok := c.iamGroupPolicyMap.Load(group)
+		if !ok {
+			if err := store.loadMappedPolicyWithRetry(context.TODO(), group, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return nil, time.Time{}, err
+			}
+			policy, _ = c.iamGroupPolicyMap.Load(group)
+		}
+
+		for _, p := range policy.toSlice() {
+			policies.Add(p)
+		}
+	}
 
 	for _, group := range c.iamUserGroupMemberships[name].ToSlice() {
 		if store.getUsersSysType() == MinIOUsersSysType {
@@ -462,10 +495,12 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 			policy, _ = c.iamGroupPolicyMap.Load(group)
 		}
 
-		policies = append(policies, policy.toSlice()...)
+		for _, p := range policy.toSlice() {
+			policies.Add(p)
+		}
 	}
 
-	return policies, mp.UpdatedAt, nil
+	return policies.ToSlice(), mp.UpdatedAt, nil
 }
 
 func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
@@ -937,12 +972,7 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	}, nil
 }
 
-// ListGroups - lists groups. Since this is not going to be a frequent
-// operation, we fetch this info from storage, and refresh the cache as well.
-func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err error) {
-	cache := store.lock()
-	defer store.unlock()
-
+func (store *IAMStoreSys) updateGroups(ctx context.Context, cache *iamCache) (res []string, err error) {
 	if store.getUsersSysType() == MinIOUsersSysType {
 		m := map[string]GroupInfo{}
 		err = store.loadGroups(ctx, m)
@@ -970,7 +1000,16 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 		})
 	}
 
-	return
+	return res, nil
+}
+
+// ListGroups - lists groups. Since this is not going to be a frequent
+// operation, we fetch this info from storage, and refresh the cache as well.
+func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err error) {
+	cache := store.lock()
+	defer store.unlock()
+
+	return store.updateGroups(ctx, cache)
 }
 
 // listGroups - lists groups - fetch groups from cache
@@ -1445,16 +1484,51 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 	return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
 }
 
-// FilterPolicies - accepts a comma separated list of policy names as a string
-// and bucket and returns only policies that currently exist in MinIO. If
-// bucketName is non-empty, additionally filters policies matching the bucket.
-// The first returned value is the list of currently existing policies, and the
-// second is their combined policy definition.
-func (store *IAMStoreSys) FilterPolicies(policyName string, bucketName string) (string, policy.Policy) {
-	cache := store.rlock()
-	defer store.runlock()
+// MergePolicies - accepts a comma separated list of policy names as a string
+// and returns only policies that currently exist in MinIO. It includes hot loading
+// of policies if not in the memory
+func (store *IAMStoreSys) MergePolicies(policyName string) (string, policy.Policy) {
+	var policies []string
+	var missingPolicies []string
+	var toMerge []policy.Policy
 
-	return filterPolicies(cache, policyName, bucketName)
+	cache := store.rlock()
+	for _, policy := range newMappedPolicy(policyName).toSlice() {
+		if policy == "" {
+			continue
+		}
+		p, found := cache.iamPolicyDocsMap[policy]
+		if !found {
+			missingPolicies = append(missingPolicies, policy)
+			continue
+		}
+		policies = append(policies, policy)
+		toMerge = append(toMerge, p.Policy)
+	}
+	store.runlock()
+
+	if len(missingPolicies) > 0 {
+		m := make(map[string]PolicyDoc)
+		for _, policy := range missingPolicies {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			_ = store.loadPolicyDoc(ctx, policy, m)
+			cancel()
+		}
+
+		cache := store.lock()
+		for policy, p := range m {
+			cache.iamPolicyDocsMap[policy] = p
+		}
+		store.unlock()
+
+		for policy, p := range m {
+			policies = append(policies, policy)
+			toMerge = append(toMerge, p.Policy)
+		}
+
+	}
+
+	return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
 }
 
 // GetBucketUsers - returns users (not STS or service accounts) that have access
@@ -2675,6 +2749,18 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error
 			}
 		}
 
+		load := len(cache.iamGroupsMap) == 0
+		if store.getUsersSysType() == LDAPUsersSysType && cache.iamGroupPolicyMap.Size() == 0 {
+			load = true
+		}
+		if load {
+			if _, err = store.updateGroups(ctx, cache); err != nil {
+				return "done", err
+			}
+		}
+
+		cache.buildUserGroupMemberships()
+
 		return "done", err
 	})
 

cmd/iam.go

@@ -315,6 +315,24 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
+	cache := sys.store.lock()
+	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+	sys.store.unlock()
+
+	// Load RoleARNs
+	sys.rolesMap = make(map[arn.ARN]string)
+
+	// From OpenID
+	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
+	// From AuthN plugin if enabled.
+	if authn := newGlobalAuthNPluginFn(); authn != nil {
+		riMap := authn.GetRoleInfo()
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx, true); err != nil {
@@ -334,20 +352,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	go sys.periodicRoutines(ctx, refreshInterval)
 
-	// Load RoleARNs
-	sys.rolesMap = make(map[arn.ARN]string)
-
-	// From OpenID
-	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
-	// From AuthN plugin if enabled.
-	if authn := newGlobalAuthNPluginFn(); authn != nil {
-		riMap := authn.GetRoleInfo()
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
 	sys.printIAMRoles()
 
 	bootstrapTraceMsg("finishing IAM loading")
@@ -436,7 +440,7 @@ func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[a
 	// running server by creating the policies after start up.
 	for arn, rolePolicies := range m {
 		specifiedPoliciesSet := newMappedPolicy(rolePolicies).policySet()
-		validPolicies, _ := sys.store.FilterPolicies(rolePolicies, "")
+		validPolicies, _ := sys.store.MergePolicies(rolePolicies)
 		knownPoliciesSet := newMappedPolicy(validPolicies).policySet()
 		unknownPoliciesSet := specifiedPoliciesSet.Difference(knownPoliciesSet)
 		if len(unknownPoliciesSet) > 0 {
@@ -672,7 +676,7 @@ func (sys *IAMSys) CurrentPolicies(policyName string) string {
 		return ""
 	}
 
-	policies, _ := sys.store.FilterPolicies(policyName, "")
+	policies, _ := sys.store.MergePolicies(policyName)
 	return policies
 }
 
@@ -2122,7 +2126,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string)
 	var combinedPolicy policy.Policy
 	// Policies were found, evaluate all of them.
 	if !isOwnerDerived {
-		availablePoliciesStr, c := sys.store.FilterPolicies(strings.Join(svcPolicies, ","), "")
+		availablePoliciesStr, c := sys.store.MergePolicies(strings.Join(svcPolicies, ","))
 		if availablePoliciesStr == "" {
 			return false
 		}
@@ -2214,22 +2218,16 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 	// 2. Combine the mapped policies into a single combined policy.
 
 	var combinedPolicy policy.Policy
+	// Policies were found, evaluate all of them.
 	if !isOwnerDerived {
-		var err error
-		combinedPolicy, err = sys.store.GetPolicy(strings.Join(policies, ","))
-		if errors.Is(err, errNoSuchPolicy) {
-			for _, pname := range policies {
-				_, err := sys.store.GetPolicy(pname)
-				if errors.Is(err, errNoSuchPolicy) {
-					// all policies presented in the claim should exist
-					iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
-					return false
-				}
-			}
-			iamLogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
+		availablePoliciesStr, c := sys.store.MergePolicies(strings.Join(policies, ","))
+		if availablePoliciesStr == "" {
+			// all policies presented in the claim should exist
+			iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", policies, iamPolicyClaimNameOpenID()))
+
 			return false
 		}
-
+		combinedPolicy = c
 	}
 
 	// 3. If an inline session-policy is present, evaluate it.
@@ -2350,7 +2348,7 @@ func isAllowedBySessionPolicy(args policy.Args) (hasSessionPolicy bool, isAllowe
 
 // GetCombinedPolicy returns a combined policy combining all policies
 func (sys *IAMSys) GetCombinedPolicy(policies ...string) policy.Policy {
-	_, policy := sys.store.FilterPolicies(strings.Join(policies, ","), "")
+	_, policy := sys.store.MergePolicies(strings.Join(policies, ","))
 	return policy
 }
 

cmd/server-main.go

@@ -897,7 +897,7 @@ func serverMain(ctx *cli.Context) {
 		})
 	}
 
-	if !globalDisableFreezeOnBoot {
+	if globalEnableSyncBoot {
 		// Freeze the services until the bucket notification subsystem gets initialized.
 		bootstrapTrace("freezeServices", freezeServices)
 	}
@@ -1000,10 +1000,11 @@ func serverMain(ctx *cli.Context) {
 	}()
 
 	go func() {
-		if !globalDisableFreezeOnBoot {
+		if globalEnableSyncBoot {
 			defer bootstrapTrace("unfreezeServices", unfreezeServices)
 			t := time.AfterFunc(5*time.Minute, func() {
-				warnings = append(warnings, color.YellowBold("- Initializing the config subsystem is taking longer than 5 minutes. Please set '_MINIO_DISABLE_API_FREEZE_ON_BOOT=true' to not freeze the APIs"))
+				warnings = append(warnings,
+					color.YellowBold("- Initializing the config subsystem is taking longer than 5 minutes. Please remove 'MINIO_SYNC_BOOT=on' to not freeze the APIs"))
 			})
 			defer t.Stop()
 		}

cmd/sftp-server_test.go

@@ -194,9 +194,12 @@ func (s *TestSuiteIAM) SFTPInvalidServiceAccountPassword(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
-	if err != nil {
-		c.Fatalf("unable to set policy: %v", err)
+	userReq := madmin.PolicyAssociationReq{
+		Policies: []string{"readwrite"},
+		User:     accessKey,
+	}
+	if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	newSSHCon := newSSHConnMock(accessKey + "=svc")
@@ -222,9 +225,12 @@ func (s *TestSuiteIAM) SFTPServiceAccountLogin(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
-	if err != nil {
-		c.Fatalf("unable to set policy: %v", err)
+	userReq := madmin.PolicyAssociationReq{
+		Policies: []string{"readwrite"},
+		User:     accessKey,
+	}
+	if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	newSSHCon := newSSHConnMock(accessKey + "=svc")
@@ -270,9 +276,12 @@ func (s *TestSuiteIAM) SFTPValidLDAPLoginWithPassword(c *check) {
 	}
 
 	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
-	err = s.adm.SetPolicy(ctx, policy, userDN, false)
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+	userReq := madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     userDN,
+	}
+	if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	newSSHCon := newSSHConnMock("dillon=ldap")

cmd/sts-handlers_test.go

@@ -116,9 +116,12 @@ func (s *TestSuiteIAM) TestSTSServiceAccountsWithUsername(c *check) {
 		c.Fatalf("policy add error: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, "dillon", false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     "dillon",
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	assumeRole := cr.STSAssumeRole{
@@ -231,9 +234,12 @@ func (s *TestSuiteIAM) TestSTSWithDenyDeleteVersion(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	// confirm that the user is able to access the bucket
@@ -332,9 +338,12 @@ func (s *TestSuiteIAM) TestSTSWithTags(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	// confirm that the user is able to access the bucket
@@ -420,9 +429,12 @@ func (s *TestSuiteIAM) TestSTS(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	// confirm that the user is able to access the bucket
@@ -515,9 +527,12 @@ func (s *TestSuiteIAM) TestSTSWithGroupPolicy(c *check) {
 		c.Fatalf("unable to add user to group: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, "test-group", true)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		Group:    "test-group",
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("Unable to attach policy: %v", err)
 	}
 
 	// confirm that the user is able to access the bucket - permission comes
@@ -984,6 +999,7 @@ func (s *TestSuiteIAM) TestIAMExport(c *check, caseNum int, content iamTestConte
 	}
 
 	for userDN, policies := range content.ldapUserPolicyMappings {
+		// No need to detach, we are starting from a clean slate after exporting.
 		_, err := s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{
 			Policies: policies,
 			User:     userDN,
@@ -1194,14 +1210,21 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
 
 	// Attempting to set a non-existent policy should fail.
 	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
-	err = s.adm.SetPolicy(ctx, policy+"x", userDN, false)
+	_, err = s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy + "x"},
+		User:     userDN,
+	})
 	if err == nil {
-		c.Fatalf("should not be able to set non-existent policy")
+		c.Fatalf("should not be able to attach non-existent policy")
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, userDN, false)
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+	userReq := madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     userDN,
+	}
+
+	if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach user policy: %v", err)
 	}
 
 	value, err := ldapID.Retrieve()
@@ -1240,10 +1263,8 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
 		c.Fatalf("unexpected non-access-denied err: %v", err)
 	}
 
-	// Remove the policy assignment on the user DN:
-	err = s.adm.SetPolicy(ctx, "", userDN, false)
-	if err != nil {
-		c.Fatalf("Unable to remove policy setting: %v", err)
+	if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to detach user policy: %v", err)
 	}
 
 	_, err = ldapID.Retrieve()
@@ -1253,9 +1274,13 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
 
 	// Set policy via group and validate policy assignment.
 	groupDN := "cn=projectb,ou=groups,ou=swengg,dc=min,dc=io"
-	err = s.adm.SetPolicy(ctx, policy, groupDN, true)
-	if err != nil {
-		c.Fatalf("Unable to set group policy: %v", err)
+	groupReq := madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		Group:    groupDN,
+	}
+
+	if _, err = s.adm.AttachPolicyLDAP(ctx, groupReq); err != nil {
+		c.Fatalf("Unable to attach group policy: %v", err)
 	}
 
 	value, err = ldapID.Retrieve()
@@ -1278,6 +1303,10 @@ func (s *TestSuiteIAM) TestLDAPSTS(c *check) {
 	// Validate that the client cannot remove any objects
 	err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{})
 	c.Assert(err.Error(), "Access Denied.")
+
+	if _, err = s.adm.DetachPolicyLDAP(ctx, groupReq); err != nil {
+		c.Fatalf("Unable to detach group policy: %v", err)
+	}
 }
 
 func (s *TestSuiteIAM) TestLDAPUnicodeVariationsLegacyAPI(c *check) {
@@ -1490,12 +1519,13 @@ func (s *TestSuiteIAM) TestLDAPUnicodeVariations(c *check) {
 	// \uFE52 is the unicode dot SMALL FULL STOP used below:
 	userDNWithUnicodeDot := "uid=svc﹒algorithm,OU=swengg,DC=min,DC=io"
 
-	_, err = s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{
+	userReq := madmin.PolicyAssociationReq{
 		Policies: []string{policy},
 		User:     userDNWithUnicodeDot,
-	})
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach user policy: %v", err)
 	}
 
 	value, err := ldapID.Retrieve()
@@ -1534,12 +1564,9 @@ func (s *TestSuiteIAM) TestLDAPUnicodeVariations(c *check) {
 	}
 
 	// Remove the policy assignment on the user DN:
-	_, err = s.adm.DetachPolicyLDAP(ctx, madmin.PolicyAssociationReq{
-		Policies: []string{policy},
-		User:     userDNWithUnicodeDot,
-	})
-	if err != nil {
-		c.Fatalf("Unable to remove policy setting: %v", err)
+
+	if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to detach user policy: %v", err)
 	}
 
 	_, err = ldapID.Retrieve()
@@ -1550,11 +1577,12 @@ func (s *TestSuiteIAM) TestLDAPUnicodeVariations(c *check) {
 	// Set policy via group and validate policy assignment.
 	actualGroupDN := mustNormalizeDN("cn=project.c,ou=groups,ou=swengg,dc=min,dc=io")
 	groupDNWithUnicodeDot := "cn=project﹒c,ou=groups,ou=swengg,dc=min,dc=io"
-	_, err = s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{
+	groupReq := madmin.PolicyAssociationReq{
 		Policies: []string{policy},
 		Group:    groupDNWithUnicodeDot,
-	})
-	if err != nil {
+	}
+
+	if _, err = s.adm.AttachPolicyLDAP(ctx, groupReq); err != nil {
 		c.Fatalf("Unable to attach group policy: %v", err)
 	}
 
@@ -1594,6 +1622,10 @@ func (s *TestSuiteIAM) TestLDAPUnicodeVariations(c *check) {
 	// Validate that the client cannot remove any objects
 	err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{})
 	c.Assert(err.Error(), "Access Denied.")
+
+	if _, err = s.adm.DetachPolicyLDAP(ctx, groupReq); err != nil {
+		c.Fatalf("Unable to detach group policy: %v", err)
+	}
 }
 
 func (s *TestSuiteIAM) TestLDAPSTSServiceAccounts(c *check) {
@@ -1630,9 +1662,13 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccounts(c *check) {
 	}
 
 	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
-	err = s.adm.SetPolicy(ctx, policy, userDN, false)
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+	userReq := madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     userDN,
+	}
+
+	if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach user policy: %v", err)
 	}
 
 	ldapID := cr.LDAPIdentity{
@@ -1687,6 +1723,11 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccounts(c *check) {
 
 	// 6. Check that service account cannot be created for some other user.
 	c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
+
+	// Detach the policy from the user
+	if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to detach user policy: %v", err)
+	}
 }
 
 func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithUsername(c *check) {
@@ -1707,12 +1748,12 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithUsername(c *check) {
   {
    "Effect": "Allow",
    "Action": [
-    "s3:PutObject",
-    "s3:GetObject",
-    "s3:ListBucket"
+	"s3:PutObject",
+	"s3:GetObject",
+	"s3:ListBucket"
    ],
    "Resource": [
-    "arn:aws:s3:::${ldap:username}/*"
+	"arn:aws:s3:::${ldap:username}/*"
    ]
   }
  ]
@@ -1723,9 +1764,14 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithUsername(c *check) {
 	}
 
 	userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
-	err = s.adm.SetPolicy(ctx, policy, userDN, false)
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+
+	userReq := madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     userDN,
+	}
+
+	if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach user policy: %v", err)
 	}
 
 	ldapID := cr.LDAPIdentity{
@@ -1776,6 +1822,10 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithUsername(c *check) {
 
 	// 3. Check S3 access for download
 	c.mustDownload(ctx, svcClient, bucket)
+
+	if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to detach user policy: %v", err)
+	}
 }
 
 // In this test, the parent users gets their permissions from a group, rather
@@ -1814,9 +1864,13 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithGroups(c *check) {
 	}
 
 	groupDN := "cn=projecta,ou=groups,ou=swengg,dc=min,dc=io"
-	err = s.adm.SetPolicy(ctx, policy, groupDN, true)
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+	userReq := madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		Group:    groupDN,
+	}
+
+	if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach user policy: %v", err)
 	}
 
 	ldapID := cr.LDAPIdentity{
@@ -1871,18 +1925,24 @@ func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithGroups(c *check) {
 
 	// 6. Check that service account cannot be created for some other user.
 	c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
+
+	// Detach the user policy
+	if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to detach user policy: %v", err)
+	}
 }
 
 func (s *TestSuiteIAM) TestLDAPCyrillicUser(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
-	_, err := s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{
+	userReq := madmin.PolicyAssociationReq{
 		Policies: []string{"readwrite"},
 		User:     "uid=Пользователь,ou=people,ou=swengg,dc=min,dc=io",
-	})
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	if _, err := s.adm.AttachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to attach user policy: %v", err)
 	}
 
 	cases := []struct {
@@ -1940,6 +2000,10 @@ func (s *TestSuiteIAM) TestLDAPCyrillicUser(c *check) {
 			c.Fatalf("Test %d: unexpected dn claim: %s", i+1, dnClaim)
 		}
 	}
+
+	if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil {
+		c.Fatalf("Unable to detach user policy: %v", err)
+	}
 }
 
 func (s *TestSuiteIAM) TestLDAPAttributesLookup(c *check) {
@@ -1947,12 +2011,13 @@ func (s *TestSuiteIAM) TestLDAPAttributesLookup(c *check) {
 	defer cancel()
 
 	groupDN := "cn=projectb,ou=groups,ou=swengg,dc=min,dc=io"
-	_, err := s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{
+	groupReq := madmin.PolicyAssociationReq{
 		Policies: []string{"readwrite"},
 		Group:    groupDN,
-	})
-	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	if _, err := s.adm.AttachPolicyLDAP(ctx, groupReq); err != nil {
+		c.Fatalf("Unable to attach user policy: %v", err)
 	}
 
 	cases := []struct {
@@ -2025,6 +2090,10 @@ func (s *TestSuiteIAM) TestLDAPAttributesLookup(c *check) {
 			c.Fatalf("Test %d: unexpected sshPublicKey type: %s", i+1, parts[0])
 		}
 	}
+
+	if _, err = s.adm.DetachPolicyLDAP(ctx, groupReq); err != nil {
+		c.Fatalf("Unable to detach group policy: %v", err)
+	}
 }
 
 func (s *TestSuiteIAM) TestOpenIDSTS(c *check) {

docs/distributed/distributed-from-config-file.sh

@@ -22,6 +22,12 @@ export MINIO_CI_CD=1
 if [ ! -f ./mc ]; then
 	os="$(uname -s)"
 	arch="$(uname -m)"
+	case "${arch}" in
+	"x86_64")
+		arch="amd64"
+		;;
+	esac
+
 	wget -O mc https://dl.minio.io/client/mc/release/${os,,}-${arch,,}/mc &&
 		chmod +x mc
 fi

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2024-06-22T05-26-45Z
+  image: quay.io/minio/minio:RELEASE.2024-06-28T09-06-49Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/tuning/README.md

@@ -0,0 +1,26 @@
+# How to enable 'minio' performance profile with tuned?
+
+## Prerequisites
+
+Please make sure the following packages are already installed via `dnf` or `apt`: 
+
+- `tuned`
+- `curl`
+
+### Install `tuned.conf` performance profile
+
+#### Step 1 - download `tuned.conf` from the referenced link
+```
+wget https://raw.githubusercontent.com/minio/minio/master/docs/tuning/tuned.conf
+```
+
+#### Step 2 - install tuned.conf as supported performance profile on all nodes
+```
+sudo mkdir -p /usr/lib/tuned/minio/
+sudo mv tuned.conf /usr/lib/tuned/minio
+```
+
+#### Step 3 - to enable minio performance profile on all the nodes
+```
+sudo tuned-admin profile minio
+```

docs/tuning/tuned.conf

@@ -0,0 +1,81 @@
+[main]
+summary=Maximum server performance for MinIO
+
+[vm]
+transparent_hugepage=madvise
+transparent_hugepage.defrag=defer+madvise
+transparent_hugepage.khugepaged.max_ptes_none=0
+
+[cpu]
+force_latency=1
+governor=performance
+energy_perf_bias=performance
+min_perf_pct=100
+
+[sysctl]
+fs.xfs.xfssyncd_centisecs=72000
+net.core.busy_read=50
+net.core.busy_poll=50
+kernel.numa_balancing=1
+
+# Do not use swap at all
+vm.swappiness=0
+vm.vfs_cache_pressure=50
+
+# Start writeback at 3% memory
+vm.dirty_background_ratio=3
+# Force writeback at 10% memory
+vm.dirty_ratio=10
+
+# Quite a few memory map
+# areas may be consumed
+vm.max_map_count=524288
+
+# Default is 500000 = 0.5ms
+kernel.sched_migration_cost_ns=5000000
+
+# stalled hdd io threads
+kernel.hung_task_timeout_secs=85
+
+# network tuning for bigger throughput
+net.core.netdev_max_backlog=250000
+net.core.somaxconn=16384
+net.ipv4.tcp_syncookies=0
+net.ipv4.tcp_max_syn_backlog=16384
+net.core.wmem_max=4194304
+net.core.rmem_max=4194304
+net.core.rmem_default=4194304
+net.core.wmem_default=4194304
+net.ipv4.tcp_rmem="4096 87380 4194304"
+net.ipv4.tcp_wmem="4096 65536 4194304"
+
+# Reduce CPU utilization
+net.ipv4.tcp_timestamps=0
+
+# Increase throughput
+net.ipv4.tcp_sack=1
+
+# Low latency mode for TCP
+net.ipv4.tcp_low_latency=1
+
+# The following variable is used to tell the kernel how 
+# much of the socket buffer space should be used for TCP 
+# window size, and how much to save for an application buffer.
+net.ipv4.tcp_adv_win_scale=1
+
+# disable RFC2861 behavior
+net.ipv4.tcp_slow_start_after_idle = 0
+
+# Fix faulty network setups
+net.ipv4.tcp_mtu_probing=1
+net.ipv4.tcp_base_mss=1280
+
+# Disable ipv6
+net.ipv6.conf.all.disable_ipv6=1
+net.ipv6.conf.default.disable_ipv6=1
+net.ipv6.conf.lo.disable_ipv6=1
+
+[bootloader]
+# Avoid firing timers for all CPUs at the same time. This is irrelevant for
+# full nohz systems
+cmdline=skew_tick=1

internal/bucket/object/lock/lock.go

@@ -572,6 +572,7 @@ func FilterObjectLockMetadata(metadata map[string]string, filterRetention, filte
 	dst := metadata
 	var copied bool
 	delKey := func(key string) {
+		key = strings.ToLower(key)
 		if _, ok := metadata[key]; !ok {
 			return
 		}

internal/bucket/object/lock/lock_test.go

@@ -606,7 +606,7 @@ func TestFilterObjectLockMetadata(t *testing.T) {
 
 	for i, tt := range tests {
 		o := FilterObjectLockMetadata(tt.metadata, tt.filterRetention, tt.filterLegalHold)
-		if !reflect.DeepEqual(o, tt.metadata) {
+		if !reflect.DeepEqual(o, tt.expected) {
 			t.Fatalf("Case %d expected %v, got %v", i, tt.metadata, o)
 		}
 	}

internal/config/etcd/etcd.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/fips"
 	"github.com/minio/pkg/v3/env"
 	xnet "github.com/minio/pkg/v3/net"
 	clientv3 "go.etcd.io/etcd/client/v3"
@@ -159,7 +160,13 @@ func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) {
 	cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, kvs.Get(PathPrefix))
 	if etcdSecure {
 		cfg.TLS = &tls.Config{
-			RootCAs: rootCAs,
+			RootCAs:                  rootCAs,
+			PreferServerCipherSuites: true,
+			MinVersion:               tls.VersionTLS12,
+			NextProtos:               []string{"http/1.1", "h2"},
+			ClientSessionCache:       tls.NewLRUClientSessionCache(64),
+			CipherSuites:             fips.TLSCiphersBackwardCompatible(),
+			CurvePreferences:         fips.TLSCurveIDs(),
 		}
 		// This is only to support client side certificate authentication
 		// https://coreos.com/etcd/docs/latest/op-guide/security.html