fix: refactor locks to apply them uniquely per node (#11052)

This refactor is done for few reasons below

- to avoid deadlocks in scenarios when number
  of nodes are smaller < actual erasure stripe
  count where in N participating local lockers
  can lead to deadlocks across systems.

- avoids expiry routines to run 1000 of separate
  network operations and routes per disk where
  as each of them are still accessing one single
  local entity.

- it is ideal to have since globalLockServer
  per instance.

- In a 32node deployment however, each server
  group is still concentrated towards the
  same set of lockers that partipicate during
  the write/read phase, unlike previous minio/dsync
  implementation - this potentially avoids send
  32 requests instead we will still send at max
  requests of unique nodes participating in a
  write/read phase.

- reduces overall chattiness on smaller setups.

cmd/erasure-sets.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/dchest/siphash"
 	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
@@ -278,10 +279,20 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 
 // GetAllLockers return a list of all lockers for all sets.
 func (s *erasureSets) GetAllLockers() []dsync.NetLocker {
-	allLockers := make([]dsync.NetLocker, s.setDriveCount*s.setCount)
-	for i, lockers := range s.erasureLockers {
-		for j, locker := range lockers {
-			allLockers[i*s.setDriveCount+j] = locker
+	var allLockers []dsync.NetLocker
+	lockEpSet := set.NewStringSet()
+	for _, lockers := range s.erasureLockers {
+		for _, locker := range lockers {
+			if locker == nil || !locker.IsOnline() {
+				// Skip any offline lockers.
+				continue
+			}
+			if lockEpSet.Contains(locker.String()) {
+				// Skip duplicate lockers.
+				continue
+			}
+			lockEpSet.Add(locker.String())
+			allLockers = append(allLockers, locker)
 		}
 	}
 	return allLockers
@@ -289,15 +300,8 @@ func (s *erasureSets) GetAllLockers() []dsync.NetLocker {
 
 func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) {
 	return func() ([]dsync.NetLocker, string) {
-		lockers := make([]dsync.NetLocker, s.setDriveCount)
+		lockers := make([]dsync.NetLocker, len(s.erasureLockers[setIndex]))
 		copy(lockers, s.erasureLockers[setIndex])
-		sort.Slice(lockers, func(i, j int) bool {
-			// re-order lockers with affinity for
-			// - non-local lockers
-			// - online lockers
-			// are used first
-			return !lockers[i].IsLocal() && lockers[i].IsOnline()
-		})
 		return lockers, s.erasureLockOwner
 	}
 }
@@ -362,14 +366,24 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 
 	for i := 0; i < setCount; i++ {
 		s.erasureDisks[i] = make([]StorageAPI, setDriveCount)
-		s.erasureLockers[i] = make([]dsync.NetLocker, setDriveCount)
+	}
+
+	var erasureLockers = map[string]dsync.NetLocker{}
+	for _, endpoint := range endpoints {
+		if _, ok := erasureLockers[endpoint.Host]; !ok {
+			erasureLockers[endpoint.Host] = newLockAPI(endpoint)
+		}
 	}
 
 	for i := 0; i < setCount; i++ {
+		var lockerEpSet = set.NewStringSet()
 		for j := 0; j < setDriveCount; j++ {
 			endpoint := endpoints[i*setDriveCount+j]
-			// Rely on endpoints list to initialize, init lockers and available disks.
-			s.erasureLockers[i][j] = newLockAPI(endpoint)
+			// Only add lockers only one per endpoint and per erasure set.
+			if locker, ok := erasureLockers[endpoint.Host]; ok && !lockerEpSet.Contains(endpoint.Host) {
+				lockerEpSet.Add(endpoint.Host)
+				s.erasureLockers[i] = append(s.erasureLockers[i], locker)
+			}
 			disk := storageDisks[i*setDriveCount+j]
 			if disk == nil {
 				continue
@@ -396,11 +410,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 			bp:            bp,
 			mrfOpCh:       make(chan partialOperation, 10000),
 		}
-
-		go s.sets[i].cleanupStaleUploads(ctx,
-			GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
 	}
 
+	// start cleanup stale uploads go-routine.
+	go s.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
+
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval)
 	go s.maintainMRFList()
@@ -409,6 +423,22 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	return s, nil
 }
 
+func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
+	ticker := time.NewTicker(cleanupInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			for _, set := range s.sets {
+				set.cleanupStaleUploads(ctx, expiry)
+			}
+		}
+	}
+}
+
 // NewNSLock - initialize a new namespace RWLocker instance.
 func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
 	if len(objects) == 1 {