Support MinIO to be deployed on more than 32 nodes (#8492)

This PR implements locking from a global entity into
a more localized set level entity, allowing for locks
to be held only on the resources which are writing
to a collection of disks rather than a global level.

In this process this PR also removes the top-level
limit of 32 nodes to an unlimited number of nodes. This
is a precursor change before bring in bucket expansion.

cmd/admin-handlers.go

@@ -476,7 +476,13 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 }
 
 func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
-	entry := &madmin.LockEntry{Timestamp: l.Timestamp, Resource: resource, ServerList: []string{server}, Owner: l.Node, Source: l.Source, ID: l.UID}
+	entry := &madmin.LockEntry{
+		Timestamp:  l.Timestamp,
+		Resource:   resource,
+		ServerList: []string{server},
+		Source:     l.Source,
+		ID:         l.UID,
+	}
 	if l.Writer {
 		entry.Type = "Write"
 	} else {
@@ -491,12 +497,14 @@ func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
 		if peerLock == nil {
 			continue
 		}
-		for k, v := range peerLock.Locks {
-			for _, lockReqInfo := range v {
-				if val, ok := entryMap[lockReqInfo.UID]; ok {
-					val.ServerList = append(val.ServerList, peerLock.Addr)
-				} else {
-					entryMap[lockReqInfo.UID] = newLockEntry(lockReqInfo, k, peerLock.Addr)
+		for _, locks := range peerLock.Locks {
+			for k, v := range locks {
+				for _, lockReqInfo := range v {
+					if val, ok := entryMap[lockReqInfo.UID]; ok {
+						val.ServerList = append(val.ServerList, peerLock.Addr)
+					} else {
+						entryMap[lockReqInfo.UID] = newLockEntry(lockReqInfo, k, peerLock.Addr)
+					}
 				}
 			}
 		}
@@ -531,10 +539,13 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 	peerLocks := globalNotificationSys.GetLocks(ctx)
 	// Once we have received all the locks currently used from peers
 	// add the local peer locks list as well.
-	localLocks := globalLockServer.ll.DupLockMap()
+	var getRespLocks GetLocksResp
+	for _, llocker := range globalLockServers {
+		getRespLocks = append(getRespLocks, llocker.DupLockMap())
+	}
 	peerLocks = append(peerLocks, &PeerLocks{
 		Addr:  getHostName(r),
-		Locks: localLocks,
+		Locks: getRespLocks,
 	})
 
 	topLocks := topLockEntries(peerLocks)

cmd/admin-handlers_test.go

@@ -27,7 +27,6 @@ import (
 	"net/url"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
@@ -68,10 +67,6 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// code backend.
 	globalIsXL = true
 
-	// initialize NSLock.
-	isDistXL := false
-	initNSLock(isDistXL)
-
 	// Init global heal state
 	if globalIsXL {
 		globalAllHealState = initHealState()
@@ -345,46 +340,6 @@ func TestToAdminAPIErrCode(t *testing.T) {
 	}
 }
 
-func TestTopLockEntries(t *testing.T) {
-	t1 := UTCNow()
-	t2 := UTCNow().Add(10 * time.Second)
-	peerLocks := []*PeerLocks{
-		{
-			Addr: "1",
-			Locks: map[string][]lockRequesterInfo{
-				"1": {
-					{false, "node2", "ep2", "2", t2, t2, ""},
-					{true, "node1", "ep1", "1", t1, t1, ""},
-				},
-				"2": {
-					{false, "node2", "ep2", "2", t2, t2, ""},
-					{true, "node1", "ep1", "1", t1, t1, ""},
-				},
-			},
-		},
-		{
-			Addr: "2",
-			Locks: map[string][]lockRequesterInfo{
-				"1": {
-					{false, "node2", "ep2", "2", t2, t2, ""},
-					{true, "node1", "ep1", "1", t1, t1, ""},
-				},
-				"2": {
-					{false, "node2", "ep2", "2", t2, t2, ""},
-					{true, "node1", "ep1", "1", t1, t1, ""},
-				},
-			},
-		},
-	}
-	les := topLockEntries(peerLocks)
-	if len(les) != 2 {
-		t.Fatalf("Did not get 2 results")
-	}
-	if les[0].Timestamp.After(les[1].Timestamp) {
-		t.Fatalf("Got wrong sorted value")
-	}
-}
-
 func TestExtractHealInitParams(t *testing.T) {
 	mkParams := func(clientToken string, forceStart, forceStop bool) url.Values {
 		v := url.Values{}

cmd/config-encrypted.go

@@ -52,7 +52,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 	// at a given time, this big transaction lock ensures this
 	// appropriately. This is also true for rotation of encrypted
 	// content.
-	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigPrefix)
+	objLock := objAPI.NewNSLock(context.Background(), minioMetaBucket, transactionConfigPrefix)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/config.go

@@ -294,7 +294,7 @@ func initConfig(objAPI ObjectLayer) error {
 	// all the config as necessary, this is to ensure that
 	// redundant locks are not held for each migration - this allows
 	// for a more predictable behavior while debugging.
-	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigPrefix)
+	objLock := objAPI.NewNSLock(context.Background(), minioMetaBucket, transactionConfigPrefix)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/daily-lifecycle-ops.go

@@ -108,7 +108,7 @@ var lifecycleTimeout = newDynamicTimeout(60*time.Second, time.Second)
 
 func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 	// Lock to avoid concurrent lifecycle ops from other nodes
-	sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-lifecycle-ops")
+	sweepLock := objAPI.NewNSLock(ctx, "system", "daily-lifecycle-ops")
 	if err := sweepLock.GetLock(lifecycleTimeout); err != nil {
 		return err
 	}

cmd/disk-cache.go

@@ -1,3 +1,19 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package cmd
 
 import (
@@ -49,8 +65,6 @@ type cacheObjects struct {
 	cache []*diskCache
 	// file path patterns to exclude from cache
 	exclude []string
-	// to manage cache namespace locks
-	nsMutex *nsLockMap
 
 	// if true migration is in progress from v1 to v2
 	migrating bool
@@ -58,6 +72,7 @@ type cacheObjects struct {
 	migMutex sync.Mutex
 
 	// Object functions pointing to the corresponding functions of backend implementation.
+	NewNSLockFn      func(ctx context.Context, bucket, object string) RWLocker
 	GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn   func(ctx context.Context, bucket, object string) error
@@ -66,7 +81,7 @@ type cacheObjects struct {
 }
 
 func (c *cacheObjects) delete(ctx context.Context, dcache *diskCache, bucket, object string) (err error) {
-	cLock := c.nsMutex.NewNSLock(ctx, bucket, object)
+	cLock := c.NewNSLockFn(ctx, bucket, object)
 	if err := cLock.GetLock(globalObjectTimeout); err != nil {
 		return err
 	}
@@ -75,7 +90,7 @@ func (c *cacheObjects) delete(ctx context.Context, dcache *diskCache, bucket, ob
 }
 
 func (c *cacheObjects) put(ctx context.Context, dcache *diskCache, bucket, object string, data io.Reader, size int64, opts ObjectOptions) error {
-	cLock := c.nsMutex.NewNSLock(ctx, bucket, object)
+	cLock := c.NewNSLockFn(ctx, bucket, object)
 	if err := cLock.GetLock(globalObjectTimeout); err != nil {
 		return err
 	}
@@ -84,7 +99,7 @@ func (c *cacheObjects) put(ctx context.Context, dcache *diskCache, bucket, objec
 }
 
 func (c *cacheObjects) get(ctx context.Context, dcache *diskCache, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	cLock := c.nsMutex.NewNSLock(ctx, bucket, object)
+	cLock := c.NewNSLockFn(ctx, bucket, object)
 	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
 		return nil, err
 	}
@@ -94,7 +109,7 @@ func (c *cacheObjects) get(ctx context.Context, dcache *diskCache, bucket, objec
 }
 
 func (c *cacheObjects) stat(ctx context.Context, dcache *diskCache, bucket, object string) (oi ObjectInfo, err error) {
-	cLock := c.nsMutex.NewNSLock(ctx, bucket, object)
+	cLock := c.NewNSLockFn(ctx, bucket, object)
 	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}
@@ -549,9 +564,11 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 	c := &cacheObjects{
 		cache:     cache,
 		exclude:   config.Exclude,
-		nsMutex:   newNSLock(false),
 		migrating: migrateSw,
 		migMutex:  sync.Mutex{},
+		NewNSLockFn: func(ctx context.Context, bucket, object string) RWLocker {
+			return globalObjectAPI.NewNSLock(ctx, bucket, object)
+		},
 		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},

cmd/endpoint-ellipses.go

@@ -30,11 +30,6 @@ import (
 // This file implements and supports ellipses pattern for
 // `minio server` command line arguments.
 
-// Maximum number of unique args supported on the command line.
-const (
-	serverCommandLineArgsMax = 32
-)
-
 // Endpoint set represents parsed ellipses values, also provides
 // methods to get the sets of endpoints.
 type endpointSet struct {

cmd/endpoint.go

@@ -577,12 +577,6 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		uniqueArgs.Add(endpoint.Host)
 	}
 
-	// Error out if we have more than serverCommandLineArgsMax unique servers.
-	if len(uniqueArgs.ToSlice()) > serverCommandLineArgsMax {
-		err := fmt.Errorf("Unsupported number of endpoints (%s), total number of servers cannot be more than %d", endpoints, serverCommandLineArgsMax)
-		return serverAddr, endpoints, setupType, err
-	}
-
 	// Error out if we have less than 2 unique servers.
 	if len(uniqueArgs.ToSlice()) < 2 && setupType == DistXLSetupType {
 		err := fmt.Errorf("Unsupported number of endpoints (%s), minimum number of servers cannot be less than 2 in distributed setup", endpoints)

cmd/format-xl.go

@@ -471,15 +471,7 @@ func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (stri
 }
 
 // formatXLFixDeploymentID - Add deployment id if it is not present.
-func formatXLFixDeploymentID(ctx context.Context, endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
-	// Acquire lock on format.json
-	mutex := newNSLock(globalIsDistXL)
-	formatLock := mutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
-	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
-		return err
-	}
-	defer formatLock.Unlock()
-
+func formatXLFixDeploymentID(endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
 	// Attempt to load all `format.json` from all disks.
 	var sErrs []error
 	formats, sErrs := loadFormatXLAll(storageDisks)
@@ -518,12 +510,12 @@ func formatXLFixDeploymentID(ctx context.Context, endpoints EndpointList, storag
 	}
 	// Deployment ID needs to be set on all the disks.
 	// Save `format.json` across all disks.
-	return saveFormatXLAll(ctx, storageDisks, formats)
+	return saveFormatXLAll(context.Background(), storageDisks, formats)
 
 }
 
 // Update only the valid local disks which have not been updated before.
-func formatXLFixLocalDeploymentID(ctx context.Context, endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) error {
+func formatXLFixLocalDeploymentID(endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) error {
 	// If this server was down when the deploymentID was updated
 	// then we make sure that we update the local disks with the deploymentID.
 	for index, storageDisk := range storageDisks {
@@ -542,7 +534,7 @@ func formatXLFixLocalDeploymentID(ctx context.Context, endpoints EndpointList, s
 			}
 			format.ID = refFormat.ID
 			if err := saveFormatXL(storageDisk, format); err != nil {
-				logger.LogIf(ctx, err)
+				logger.LogIf(context.Background(), err)
 				return fmt.Errorf("Unable to save format.json, %s", err)
 			}
 		}

cmd/fs-v1-multipart.go

@@ -646,7 +646,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Hold write lock on the object.
-	destLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
+	destLock := fs.NewNSLock(ctx, bucket, object)
 	if err = destLock.GetLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}

cmd/fs-v1.go

@@ -165,6 +165,12 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	return fs, nil
 }
 
+// NewNSLock - initialize a new namespace RWLocker instance.
+func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+	// lockers are explicitly 'nil' for FS mode since there are only local lockers
+	return fs.nsMutex.NewNSLock(ctx, nil, bucket, object)
+}
+
 // Shutdown - should be called when process shuts down.
 func (fs *FSObjects) Shutdown(ctx context.Context) error {
 	fs.fsFormatRlk.Close()
@@ -290,7 +296,7 @@ func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileI
 // MakeBucketWithLocation - create a new bucket, returns if it
 // already exists.
 func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
-	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
+	bucketLock := fs.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
 		return err
 	}
@@ -313,7 +319,7 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 
 // GetBucketInfo - fetch bucket metadata info.
 func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
+	bucketLock := fs.NewNSLock(ctx, bucket, "")
 	if e := bucketLock.GetRLock(globalObjectTimeout); e != nil {
 		return bi, e
 	}
@@ -376,7 +382,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 // DeleteBucket - delete a bucket and all the metadata associated
 // with the bucket including pending multipart, object metadata.
 func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
-	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
+	bucketLock := fs.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
@@ -412,7 +418,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
 func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 	if !cpSrcDstSame {
-		objectDWLock := fs.nsMutex.NewNSLock(ctx, dstBucket, dstObject)
+		objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject)
 		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, err
 		}
@@ -484,7 +490,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	if lockType != noLock {
 		// Lock the object before reading.
-		lock := fs.nsMutex.NewNSLock(ctx, bucket, object)
+		lock := fs.NewNSLock(ctx, bucket, object)
 		switch lockType {
 		case writeLock:
 			if err = lock.GetLock(globalObjectTimeout); err != nil {
@@ -571,7 +577,7 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
 	}
 
 	// Lock the object before reading.
-	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := fs.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
@@ -739,7 +745,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 // getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
 func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
-	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := fs.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}
@@ -764,7 +770,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
-		objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
+		objectLock := fs.NewNSLock(ctx, bucket, object)
 		if err = objectLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
@@ -810,7 +816,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return ObjectInfo{}, err
 	}
 	// Lock the object.
-	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := fs.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
@@ -965,7 +971,7 @@ func (fs *FSObjects) DeleteObjects(ctx context.Context, bucket string, objects [
 // and there are no rollbacks supported.
 func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) error {
 	// Acquire a write lock before deleting the object.
-	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := fs.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/gateway-main.go

@@ -142,8 +142,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Set system resources to maximum.
 	logger.LogIf(context.Background(), setMaxResources())
 
-	initNSLock(false) // Enable local namespace lock.
-
 	// Set when gateway is enabled
 	globalIsGateway = true
 

cmd/gateway-unsupported.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lifecycle"
@@ -28,6 +29,12 @@ import (
 // GatewayUnsupported list of unsupported call stubs for gateway.
 type GatewayUnsupported struct{}
 
+// NewNSLock is a dummy stub for gateway.
+func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+	logger.CriticalIf(ctx, errors.New("not implemented"))
+	return nil
+}
+
 // ListMultipartUploads lists all multipart uploads.
 func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
 	return lmi, NotImplemented{}

cmd/global-heal.go

@@ -83,7 +83,8 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
 	// Hold a lock for healing the erasure set
 	zeroDuration := time.Millisecond
 	zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
-	erasureSetHealLock := globalNSMutex.NewNSLock(ctx, "system", fmt.Sprintf("erasure-set-heal-%d", setIndex))
+	erasureSetHealLock := xlObj.nsMutex.NewNSLock(ctx, xlObj.getLockers(),
+		"system", fmt.Sprintf("erasure-set-heal-%d", setIndex))
 	if err := erasureSetHealLock.GetLock(zeroDynamicTimeout); err != nil {
 		return err
 	}
@@ -126,7 +127,7 @@ func execLeaderTasks(sets *xlSets) {
 	ctx := context.Background()
 
 	// Hold a lock so only one server performs auto-healing
-	leaderLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, "leader")
+	leaderLock := sets.NewNSLock(ctx, minioMetaBucket, "leader")
 	for {
 		err := leaderLock.GetLock(leaderLockTimeout)
 		if err == nil {

cmd/iam.go

@@ -338,7 +338,7 @@ func (sys *IAMSys) Load() error {
 func (sys *IAMSys) doIAMConfigMigration(objAPI ObjectLayer) error {
 	// Take IAM configuration migration lock
 	lockPath := iamConfigPrefix + "/migration.lock"
-	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, lockPath)
+	objLock := objAPI.NewNSLock(context.Background(), minioMetaBucket, lockPath)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/local-locker.go

@@ -17,22 +17,21 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 	"time"
 
-	"github.com/minio/dsync/v2"
+	"github.com/minio/minio/pkg/dsync"
 )
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
-	Writer          bool      // Bool whether write or read lock.
-	Node            string    // Network address of client claiming lock.
-	ServiceEndpoint string    // RPC path of client claiming lock.
-	UID             string    // UID to uniquely identify request of client.
-	Timestamp       time.Time // Timestamp set at the time of initialization.
-	TimeLastCheck   time.Time // Timestamp for last check of validity of lock.
-	Source          string    // Contains line, function and filename reqesting the lock.
+	Writer        bool      // Bool whether write or read lock.
+	UID           string    // UID to uniquely identify request of client.
+	Timestamp     time.Time // Timestamp set at the time of initialization.
+	TimeLastCheck time.Time // Timestamp for last check of validity of lock.
+	Source        string    // Contains line, function and filename reqesting the lock.
 }
 
 // isWriteLock returns whether the lock is a write or read lock.
@@ -40,20 +39,41 @@ func isWriteLock(lri []lockRequesterInfo) bool {
 	return len(lri) == 1 && lri[0].Writer
 }
 
+type errorLocker struct{}
+
+func (d *errorLocker) String() string {
+	return ""
+}
+
+func (d *errorLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
+	return false, errors.New("unable to lock")
+}
+
+func (d *errorLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
+	return false, errors.New("unable to unlock")
+}
+
+func (d *errorLocker) RLock(args dsync.LockArgs) (reply bool, err error) {
+	return false, errors.New("unable to rlock")
+}
+
+func (d *errorLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
+	return false, errors.New("unable to runlock")
+}
+
+func (d *errorLocker) Close() error {
+	return nil
+}
+
 // localLocker implements Dsync.NetLocker
 type localLocker struct {
-	mutex           sync.Mutex
-	serviceEndpoint string
-	serverAddr      string
-	lockMap         map[string][]lockRequesterInfo
+	mutex    sync.Mutex
+	endpoint Endpoint
+	lockMap  map[string][]lockRequesterInfo
 }
 
-func (l *localLocker) ServerAddr() string {
-	return l.serverAddr
-}
-
-func (l *localLocker) ServiceEndpoint() string {
-	return l.serviceEndpoint
+func (l *localLocker) String() string {
+	return l.endpoint.String()
 }
 
 func (l *localLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
@@ -63,13 +83,11 @@ func (l *localLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
 	if !isLockTaken { // No locks held on the given name, so claim write lock
 		l.lockMap[args.Resource] = []lockRequesterInfo{
 			{
-				Writer:          true,
-				Node:            args.ServerAddr,
-				ServiceEndpoint: args.ServiceEndpoint,
-				Source:          args.Source,
-				UID:             args.UID,
-				Timestamp:       UTCNow(),
-				TimeLastCheck:   UTCNow(),
+				Writer:        true,
+				Source:        args.Source,
+				UID:           args.UID,
+				Timestamp:     UTCNow(),
+				TimeLastCheck: UTCNow(),
 			},
 		}
 	}
@@ -96,17 +114,38 @@ func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
 
 }
 
+// removeEntry based on the uid of the lock message, removes a single entry from the
+// lockRequesterInfo array or the whole array from the map (in case of a write lock
+// or last read lock)
+func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
+	// Find correct entry to remove based on uid.
+	for index, entry := range *lri {
+		if entry.UID == uid {
+			if len(*lri) == 1 {
+				// Remove the write lock.
+				delete(l.lockMap, name)
+			} else {
+				// Remove the appropriate read lock.
+				*lri = append((*lri)[:index], (*lri)[index+1:]...)
+				l.lockMap[name] = *lri
+			}
+			return true
+		}
+	}
+
+	// None found return false, perhaps entry removed in previous run.
+	return false
+}
+
 func (l *localLocker) RLock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	lrInfo := lockRequesterInfo{
-		Writer:          false,
-		Node:            args.ServerAddr,
-		ServiceEndpoint: args.ServiceEndpoint,
-		Source:          args.Source,
-		UID:             args.UID,
-		Timestamp:       UTCNow(),
-		TimeLastCheck:   UTCNow(),
+		Writer:        false,
+		Source:        args.Source,
+		UID:           args.UID,
+		Timestamp:     UTCNow(),
+		TimeLastCheck: UTCNow(),
 	}
 	if lri, ok := l.lockMap[args.Resource]; ok {
 		if reply = !isWriteLock(lri); reply {
@@ -139,18 +178,6 @@ func (l *localLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
 	return reply, nil
 }
 
-func (l *localLocker) ForceUnlock(args dsync.LockArgs) (reply bool, err error) {
-	l.mutex.Lock()
-	defer l.mutex.Unlock()
-	if len(args.UID) != 0 {
-		return false, fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID)
-	}
-	// Only clear lock when it is taken
-	// Remove the lock (irrespective of write or read lock)
-	delete(l.lockMap, args.Resource)
-	return true, nil
-}
-
 func (l *localLocker) DupLockMap() map[string][]lockRequesterInfo {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
@@ -161,3 +188,14 @@ func (l *localLocker) DupLockMap() map[string][]lockRequesterInfo {
 	}
 	return lockCopy
 }
+
+func (l *localLocker) Close() error {
+	return nil
+}
+
+func newLocker(endpoint Endpoint) *localLocker {
+	return &localLocker{
+		endpoint: endpoint,
+		lockMap:  make(map[string][]lockRequesterInfo),
+	}
+}

cmd/lock-rest-client.go

@@ -26,18 +26,16 @@ import (
 
 	"net/url"
 
-	"github.com/minio/dsync/v2"
 	"github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/dsync"
 )
 
 // lockRESTClient is authenticable lock REST client
 type lockRESTClient struct {
-	host       *xnet.Host
 	restClient *rest.Client
-	serverURL  *url.URL
+	endpoint   Endpoint
 	connected  int32
 }
 
@@ -49,20 +47,13 @@ func toLockError(err error) error {
 	switch err.Error() {
 	case errLockConflict.Error():
 		return errLockConflict
-	case errLockNotExpired.Error():
-		return errLockNotExpired
 	}
 	return err
 }
 
-// ServerAddr - dsync.NetLocker interface compatible method.
-func (client *lockRESTClient) ServerAddr() string {
-	return client.serverURL.Host
-}
-
-// ServiceEndpoint - dsync.NetLocker interface compatible method.
-func (client *lockRESTClient) ServiceEndpoint() string {
-	return client.serverURL.Path
+// String stringer *dsync.NetLocker* interface compatible method.
+func (client *lockRESTClient) String() string {
+	return client.endpoint.String()
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
@@ -83,22 +74,17 @@ func (client *lockRESTClient) call(method string, values url.Values, body io.Rea
 	}
 
 	if isNetworkError(err) {
-		time.AfterFunc(defaultRetryUnit*5, func() {
-			// After 5 seconds, take this lock client
-			// online for a retry.
+		time.AfterFunc(defaultRetryUnit*3, func() {
+			// After 3 seconds, take this lock client online for a retry.
 			atomic.StoreInt32(&client.connected, 1)
 		})
+
 		atomic.StoreInt32(&client.connected, 0)
 	}
 
 	return nil, toLockError(err)
 }
 
-// Stringer provides a canonicalized representation of node.
-func (client *lockRESTClient) String() string {
-	return client.host.String()
-}
-
 // IsOnline - returns whether REST client failed to connect or not.
 func (client *lockRESTClient) IsOnline() bool {
 	return atomic.LoadInt32(&client.connected) == 1
@@ -117,15 +103,13 @@ func (client *lockRESTClient) restCall(call string, args dsync.LockArgs) (reply
 	values.Set(lockRESTUID, args.UID)
 	values.Set(lockRESTSource, args.Source)
 	values.Set(lockRESTResource, args.Resource)
-	values.Set(lockRESTServerAddr, args.ServerAddr)
-	values.Set(lockRESTServerEndpoint, args.ServiceEndpoint)
 
 	respBody, err := client.call(call, values, nil, -1)
 	defer http.DrainBody(respBody)
 	switch err {
 	case nil:
 		return true, nil
-	case errLockConflict, errLockNotExpired:
+	case errLockConflict:
 		return false, nil
 	default:
 		return false, err
@@ -152,45 +136,42 @@ func (client *lockRESTClient) Unlock(args dsync.LockArgs) (reply bool, err error
 	return client.restCall(lockRESTMethodUnlock, args)
 }
 
-// ForceUnlock calls force unlock RPC.
-func (client *lockRESTClient) ForceUnlock(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(lockRESTMethodForceUnlock, args)
+func closeLockers(lockers []dsync.NetLocker) {
+	for _, locker := range lockers {
+		locker.Close()
+	}
 }
 
-// Expired calls expired RPC.
-func (client *lockRESTClient) Expired(args dsync.LockArgs) (reply bool, err error) {
-	return client.restCall(lockRESTMethodExpired, args)
+func newLockAPI(endpoint Endpoint) dsync.NetLocker {
+	if endpoint.IsLocal {
+		return globalLockServers[endpoint]
+	}
+	return newlockRESTClient(endpoint)
 }
 
 // Returns a lock rest client.
-func newlockRESTClient(peer *xnet.Host) *lockRESTClient {
-
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
+func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 	serverURL := &url.URL{
-		Scheme: scheme,
-		Host:   peer.String(),
-		Path:   lockRESTPath,
+		Scheme: endpoint.Scheme,
+		Host:   endpoint.Host,
+		Path:   pathJoin(lockRESTPrefix, endpoint.Path, lockRESTVersion),
 	}
 
 	var tlsConfig *tls.Config
 	if globalIsSSL {
 		tlsConfig = &tls.Config{
-			ServerName: peer.Name,
+			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
 			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 
-	restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
-
+	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)
+	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
 		logger.LogIf(context.Background(), err)
-		return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: 0}
+		return &lockRESTClient{endpoint: endpoint, restClient: restClient, connected: 0}
 	}
 
-	return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: 1}
+	return &lockRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}
 }

cmd/lock-rest-client_test.go

@@ -19,17 +19,16 @@ package cmd
 import (
 	"testing"
 
-	"github.com/minio/dsync/v2"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/dsync"
 )
 
 // Tests lock rpc client.
 func TestLockRESTlient(t *testing.T) {
-	host, err := xnet.ParseHost("localhost:9000")
+	endpoint, err := NewEndpoint("http://localhost:9000")
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
-	lkClient := newlockRESTClient(host)
+	lkClient := newlockRESTClient(endpoint)
 	if lkClient.connected == 0 {
 		t.Fatalf("unexpected error. connection failed")
 	}
@@ -54,14 +53,4 @@ func TestLockRESTlient(t *testing.T) {
 	if err == nil {
 		t.Fatal("Expected for Unlock to fail")
 	}
-
-	_, err = lkClient.ForceUnlock(dsync.LockArgs{})
-	if err == nil {
-		t.Fatal("Expected for ForceUnlock to fail")
-	}
-
-	_, err = lkClient.Expired(dsync.LockArgs{})
-	if err == nil {
-		t.Fatal("Expected for Expired to fail")
-	}
 }

cmd/lock-rest-server-common.go

@@ -18,23 +18,19 @@ package cmd
 
 import (
 	"errors"
-	"time"
 )
 
 const (
 	lockRESTVersion       = "v2"
 	lockRESTVersionPrefix = SlashSeparator + "v2"
 	lockRESTPrefix        = minioReservedBucketPath + "/lock"
-	lockRESTPath          = lockRESTPrefix + lockRESTVersionPrefix
 )
 
 const (
-	lockRESTMethodLock        = "/lock"
-	lockRESTMethodRLock       = "/rlock"
-	lockRESTMethodUnlock      = "/unlock"
-	lockRESTMethodRUnlock     = "/runlock"
-	lockRESTMethodForceUnlock = "/forceunlock"
-	lockRESTMethodExpired     = "/expired"
+	lockRESTMethodLock    = "/lock"
+	lockRESTMethodRLock   = "/rlock"
+	lockRESTMethodUnlock  = "/unlock"
+	lockRESTMethodRUnlock = "/runlock"
 
 	// Unique ID of lock/unlock request.
 	lockRESTUID = "uid"
@@ -43,67 +39,6 @@ const (
 	lockRESTSource = "source"
 	// Resource contains a entity to be locked/unlocked.
 	lockRESTResource = "resource"
-	// ServerAddr contains the address of the server who requested lock/unlock of the above resource.
-	lockRESTServerAddr = "serverAddr"
-	// ServiceEndpoint contains the network path of above server to do lock/unlock.
-	lockRESTServerEndpoint = "serverEndpoint"
 )
 
-// nameLockRequesterInfoPair is a helper type for lock maintenance
-type nameLockRequesterInfoPair struct {
-	name string
-	lri  lockRequesterInfo
-}
-
 var errLockConflict = errors.New("lock conflict")
-var errLockNotExpired = errors.New("lock not expired")
-
-// Similar to removeEntry but only removes an entry only if the lock entry exists in map.
-func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
-	// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
-	if lri, ok := l.lockMap[nlrip.name]; ok {
-		// Even if the entry exists, it may not be the same entry which was
-		// considered as expired, so we simply an attempt to remove it if its
-		// not possible there is nothing we need to do.
-		l.removeEntry(nlrip.name, nlrip.lri.UID, &lri)
-	}
-}
-
-// removeEntry based on the uid of the lock message, removes a single entry from the
-// lockRequesterInfo array or the whole array from the map (in case of a write lock
-// or last read lock)
-func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
-	// Find correct entry to remove based on uid.
-	for index, entry := range *lri {
-		if entry.UID == uid {
-			if len(*lri) == 1 {
-				// Remove the write lock.
-				delete(l.lockMap, name)
-			} else {
-				// Remove the appropriate read lock.
-				*lri = append((*lri)[:index], (*lri)[index+1:]...)
-				l.lockMap[name] = *lri
-			}
-			return true
-		}
-	}
-
-	// None found return false, perhaps entry removed in previous run.
-	return false
-}
-
-// getLongLivedLocks returns locks that are older than a certain time and
-// have not been 'checked' for validity too soon enough
-func getLongLivedLocks(m map[string][]lockRequesterInfo, interval time.Duration) []nameLockRequesterInfoPair {
-	rslt := []nameLockRequesterInfoPair{}
-	for name, lriArray := range m {
-		for idx := range lriArray {
-			// Check whether enough time has gone by since last check
-			if time.Since(lriArray[idx].TimeLastCheck) >= interval {
-				rslt = append(rslt, nameLockRequesterInfoPair{name: name, lri: lriArray[idx]})
-				lriArray[idx].TimeLastCheck = UTCNow()
-			}
-		}
-	}
-	return rslt
-}

cmd/lock-rest-server-common_test.go

@@ -21,7 +21,6 @@ import (
 	"reflect"
 	"sync"
 	"testing"
-	"time"
 )
 
 // Helper function to create a lock server for testing
@@ -36,9 +35,8 @@ func createLockTestServer(t *testing.T) (string, *lockRESTServer, string) {
 
 	locker := &lockRESTServer{
 		ll: &localLocker{
-			mutex:           sync.Mutex{},
-			serviceEndpoint: "rpc-path",
-			lockMap:         make(map[string][]lockRequesterInfo),
+			mutex:   sync.Mutex{},
+			lockMap: make(map[string][]lockRequesterInfo),
 		},
 	}
 	creds := globalActiveCred
@@ -49,63 +47,22 @@ func createLockTestServer(t *testing.T) (string, *lockRESTServer, string) {
 	return fsDir, locker, token
 }
 
-// Test function to remove lock entries from map only in case they still exist based on name & uid combination
-func TestLockRpcServerRemoveEntryIfExists(t *testing.T) {
-	testPath, locker, _ := createLockTestServer(t)
-	defer os.RemoveAll(testPath)
-
-	lri := lockRequesterInfo{
-		Writer:          false,
-		Node:            "host",
-		ServiceEndpoint: "rpc-path",
-		UID:             "0123-4567",
-		Timestamp:       UTCNow(),
-		TimeLastCheck:   UTCNow(),
-	}
-	nlrip := nameLockRequesterInfoPair{name: "name", lri: lri}
-
-	// first test by simulating item has already been deleted
-	locker.ll.removeEntryIfExists(nlrip)
-	{
-		gotLri := locker.ll.lockMap["name"]
-		expectedLri := []lockRequesterInfo(nil)
-		if !reflect.DeepEqual(expectedLri, gotLri) {
-			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
-		}
-	}
-
-	// then test normal deletion
-	locker.ll.lockMap["name"] = []lockRequesterInfo{lri} // add item
-	locker.ll.removeEntryIfExists(nlrip)
-	{
-		gotLri := locker.ll.lockMap["name"]
-		expectedLri := []lockRequesterInfo(nil)
-		if !reflect.DeepEqual(expectedLri, gotLri) {
-			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
-		}
-	}
-}
-
 // Test function to remove lock entries from map based on name & uid combination
 func TestLockRpcServerRemoveEntry(t *testing.T) {
 	testPath, locker, _ := createLockTestServer(t)
 	defer os.RemoveAll(testPath)
 
 	lockRequesterInfo1 := lockRequesterInfo{
-		Writer:          true,
-		Node:            "host",
-		ServiceEndpoint: "rpc-path",
-		UID:             "0123-4567",
-		Timestamp:       UTCNow(),
-		TimeLastCheck:   UTCNow(),
+		Writer:        true,
+		UID:           "0123-4567",
+		Timestamp:     UTCNow(),
+		TimeLastCheck: UTCNow(),
 	}
 	lockRequesterInfo2 := lockRequesterInfo{
-		Writer:          true,
-		Node:            "host",
-		ServiceEndpoint: "rpc-path",
-		UID:             "89ab-cdef",
-		Timestamp:       UTCNow(),
-		TimeLastCheck:   UTCNow(),
+		Writer:        true,
+		UID:           "89ab-cdef",
+		Timestamp:     UTCNow(),
+		TimeLastCheck: UTCNow(),
 	}
 
 	locker.ll.lockMap["name"] = []lockRequesterInfo{
@@ -140,64 +97,3 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 		}
 	}
 }
-
-// Tests function returning long lived locks.
-func TestLockRpcServerGetLongLivedLocks(t *testing.T) {
-	ut := UTCNow()
-	// Collection of test cases for verifying returning valid long lived locks.
-	testCases := []struct {
-		lockMap      map[string][]lockRequesterInfo
-		lockInterval time.Duration
-		expectedNSLR []nameLockRequesterInfoPair
-	}{
-		// Testcase - 1 validates long lived locks, returns empty list.
-		{
-			lockMap: map[string][]lockRequesterInfo{
-				"test": {{
-					Writer:          true,
-					Node:            "10.1.10.21",
-					ServiceEndpoint: "/lock/mnt/disk1",
-					UID:             "10000112",
-					Timestamp:       ut,
-					TimeLastCheck:   ut,
-				}},
-			},
-			lockInterval: 1 * time.Minute,
-			expectedNSLR: []nameLockRequesterInfoPair{},
-		},
-		// Testcase - 2 validates long lived locks, returns at least one list.
-		{
-			lockMap: map[string][]lockRequesterInfo{
-				"test": {{
-					Writer:          true,
-					Node:            "10.1.10.21",
-					ServiceEndpoint: "/lock/mnt/disk1",
-					UID:             "10000112",
-					Timestamp:       ut,
-					TimeLastCheck:   ut.Add(-2 * time.Minute),
-				}},
-			},
-			lockInterval: 1 * time.Minute,
-			expectedNSLR: []nameLockRequesterInfoPair{
-				{
-					name: "test",
-					lri: lockRequesterInfo{
-						Writer:          true,
-						Node:            "10.1.10.21",
-						ServiceEndpoint: "/lock/mnt/disk1",
-						UID:             "10000112",
-						Timestamp:       ut,
-						TimeLastCheck:   ut.Add(-2 * time.Minute),
-					},
-				},
-			},
-		},
-	}
-	// Validates all test cases here.
-	for i, testCase := range testCases {
-		nsLR := getLongLivedLocks(testCase.lockMap, testCase.lockInterval)
-		if !reflect.DeepEqual(testCase.expectedNSLR, nsLR) {
-			t.Errorf("Test %d: Expected %#v, got %#v", i+1, testCase.expectedNSLR, nsLR)
-		}
-	}
-}

cmd/lock-rest-server.go

@@ -17,16 +17,13 @@
 package cmd
 
 import (
-	"context"
 	"errors"
-	"math/rand"
 	"net/http"
+	"path"
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/dsync/v2"
-	"github.com/minio/minio/cmd/logger"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/dsync"
 )
 
 const (
@@ -58,11 +55,9 @@ func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 
 func getLockArgs(r *http.Request) dsync.LockArgs {
 	return dsync.LockArgs{
-		UID:             r.URL.Query().Get(lockRESTUID),
-		Source:          r.URL.Query().Get(lockRESTSource),
-		Resource:        r.URL.Query().Get(lockRESTResource),
-		ServerAddr:      r.URL.Query().Get(lockRESTServerAddr),
-		ServiceEndpoint: r.URL.Query().Get(lockRESTServerEndpoint),
+		UID:      r.URL.Query().Get(lockRESTUID),
+		Source:   r.URL.Query().Get(lockRESTSource),
+		Resource: r.URL.Query().Get(lockRESTResource),
 	}
 }
 
@@ -132,130 +127,28 @@ func (l *lockRESTServer) RUnlockHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// ForceUnlockHandler - force releases the acquired lock.
-func (l *lockRESTServer) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
-	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	// Ignore the ForceUnlock() "reply" return value because if err == nil, "reply" is always true
-	// Consequently, if err != nil, reply is always false
-	if _, err := l.ll.ForceUnlock(getLockArgs(r)); err != nil {
-		l.writeErrorResponse(w, err)
-		return
-	}
-}
-
-// ExpiredHandler - query expired lock status.
-func (l *lockRESTServer) ExpiredHandler(w http.ResponseWriter, r *http.Request) {
-	if !l.IsValid(w, r) {
-		l.writeErrorResponse(w, errors.New("Invalid request"))
-		return
-	}
-
-	lockArgs := getLockArgs(r)
-
-	l.ll.mutex.Lock()
-	defer l.ll.mutex.Unlock()
-	// Lock found, proceed to verify if belongs to given uid.
-	if lri, ok := l.ll.lockMap[lockArgs.Resource]; ok {
-		// Check whether uid is still active
-		for _, entry := range lri {
-			if entry.UID == lockArgs.UID {
-				l.writeErrorResponse(w, errLockNotExpired)
-				return
-			}
-		}
-	}
-}
-
-// lockMaintenance loops over locks that have been active for some time and checks back
-// with the original server whether it is still alive or not
-//
-// Following logic inside ignores the errors generated for Dsync.Active operation.
-// - server at client down
-// - some network error (and server is up normally)
-//
-// We will ignore the error, and we will retry later to get a resolve on this lock
-func (l *lockRESTServer) lockMaintenance(interval time.Duration) {
-	l.ll.mutex.Lock()
-	// Get list of long lived locks to check for staleness.
-	nlripLongLived := getLongLivedLocks(l.ll.lockMap, interval)
-	l.ll.mutex.Unlock()
-
-	// Validate if long lived locks are indeed clean.
-	for _, nlrip := range nlripLongLived {
-		// Initialize client based on the long live locks.
-		host, err := xnet.ParseHost(nlrip.lri.Node)
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			continue
-		}
-		c := newlockRESTClient(host)
-		if !c.IsOnline() {
-			continue
-		}
-
-		// Call back to original server verify whether the lock is still active (based on name & uid)
-		expired, err := c.Expired(dsync.LockArgs{
-			UID:      nlrip.lri.UID,
-			Resource: nlrip.name,
-		})
-
-		if err != nil {
-			continue
-		}
-
-		// For successful response, verify if lock was indeed active or stale.
-		if expired {
-			// The lock is no longer active at server that originated
-			// the lock, attempt to remove the lock.
-			l.ll.mutex.Lock()
-			l.ll.removeEntryIfExists(nlrip) // Purge the stale entry if it exists.
-			l.ll.mutex.Unlock()
-		}
-
-		// Close the connection regardless of the call response.
-		c.Close()
-	}
-}
-
-// Start lock maintenance from all lock servers.
-func startLockMaintenance(lkSrv *lockRESTServer) {
-	// Initialize a new ticker with a minute between each ticks.
-	ticker := time.NewTicker(lockMaintenanceInterval)
-	// Stop the timer upon service closure and cleanup the go-routine.
-	defer ticker.Stop()
-
-	// Start with random sleep time, so as to avoid "synchronous checks" between servers
-	time.Sleep(time.Duration(rand.Float64() * float64(lockMaintenanceInterval)))
-	for {
-		// Verifies every minute for locks held more than 2 minutes.
-		select {
-		case <-GlobalServiceDoneCh:
-			return
-		case <-ticker.C:
-			lkSrv.lockMaintenance(lockValidityCheckInterval)
-		}
-	}
-}
-
 // registerLockRESTHandlers - register lock rest router.
-func registerLockRESTHandlers(router *mux.Router) {
-	subrouter := router.PathPrefix(lockRESTPrefix).Subrouter()
-	queries := restQueries(lockRESTUID, lockRESTSource, lockRESTResource, lockRESTServerAddr, lockRESTServerEndpoint)
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(globalLockServer.LockHandler)).Queries(queries...)
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(globalLockServer.RLockHandler)).Queries(queries...)
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(globalLockServer.UnlockHandler)).Queries(queries...)
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(globalLockServer.RUnlockHandler)).Queries(queries...)
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodForceUnlock).HandlerFunc(httpTraceHdrs(globalLockServer.ForceUnlockHandler)).Queries(queries...)
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodExpired).HandlerFunc(httpTraceAll(globalLockServer.ExpiredHandler)).Queries(queries...)
+func registerLockRESTHandlers(router *mux.Router, endpoints EndpointList) {
+	queries := restQueries(lockRESTUID, lockRESTSource, lockRESTResource)
+	for _, endpoint := range endpoints {
+		if !endpoint.IsLocal {
+			continue
+		}
+
+		lockServer := &lockRESTServer{
+			ll: newLocker(endpoint),
+		}
+
+		subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
+		subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler)).Queries(queries...)
+		subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler)).Queries(queries...)
+		subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler)).Queries(queries...)
+		subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler)).Queries(queries...)
+
+		globalLockServers[endpoint] = lockServer.ll
+	}
 
 	// If none of the routes match add default error handler routes
 	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
 	router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
-
-	// Start lock maintenance from all lock servers.
-	go startLockMaintenance(globalLockServer)
 }

cmd/namespace-lock.go

@@ -27,21 +27,13 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/minio/dsync/v2"
 	"github.com/minio/lsync"
-	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/dsync"
 )
 
-// Global name space lock.
-var globalNSMutex *nsLockMap
-
-// Global lock server one per server.
-var globalLockServer *lockRESTServer
-
-// Instance of dsync for distributed clients.
-var globalDsync *dsync.Dsync
+// local lock servers
+var globalLockServers = make(map[Endpoint]*localLocker)
 
 // RWLocker - locker interface to introduce GetRLock, RUnlock.
 type RWLocker interface {
@@ -51,45 +43,6 @@ type RWLocker interface {
 	RUnlock()
 }
 
-// Initialize distributed locking only in case of distributed setup.
-// Returns lock clients and the node index for the current server.
-func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int, err error) {
-	myNode = -1
-
-	seenHosts := set.NewStringSet()
-	for _, endpoint := range endpoints {
-		if seenHosts.Contains(endpoint.Host) {
-			continue
-		}
-		seenHosts.Add(endpoint.Host)
-
-		var locker dsync.NetLocker
-		if endpoint.IsLocal {
-			myNode = len(clnts)
-
-			globalLockServer = &lockRESTServer{
-				ll: &localLocker{
-					serverAddr:      endpoint.Host,
-					serviceEndpoint: lockRESTPrefix,
-					lockMap:         make(map[string][]lockRequesterInfo),
-				},
-			}
-			locker = globalLockServer.ll
-		} else {
-			var host *xnet.Host
-			host, err = xnet.ParseHost(endpoint.Host)
-			locker = newlockRESTClient(host)
-		}
-
-		clnts = append(clnts, locker)
-	}
-
-	if myNode == -1 {
-		return clnts, myNode, errors.New("no endpoint pointing to the local machine is found")
-	}
-	return clnts, myNode, err
-}
-
 // newNSLock - return a new name space lock map.
 func newNSLock(isDistXL bool) *nsLockMap {
 	nsMutex := nsLockMap{
@@ -102,11 +55,6 @@ func newNSLock(isDistXL bool) *nsLockMap {
 	return &nsMutex
 }
 
-// initNSLock - initialize name space lock map.
-func initNSLock(isDistXL bool) {
-	globalNSMutex = newNSLock(isDistXL)
-}
-
 // nsParam - carries name space resource.
 type nsParam struct {
 	volume string
@@ -224,32 +172,6 @@ func (n *nsLockMap) RUnlock(volume, path, opsID string) {
 	n.unlock(volume, path, readLock)
 }
 
-// ForceUnlock - forcefully unlock a lock based on name.
-func (n *nsLockMap) ForceUnlock(volume, path string) {
-	n.lockMapMutex.Lock()
-	defer n.lockMapMutex.Unlock()
-
-	// Clarification on operation:
-	// - In case of FS or XL we call ForceUnlock on the local globalNSMutex
-	//   (since there is only a single server) which will cause the 'stuck'
-	//   mutex to be removed from the map. Existing operations for this
-	//   will continue to be blocked (and timeout). New operations on this
-	//   resource will use a new mutex and proceed normally.
-	//
-	// - In case of Distributed setup (using dsync), there is no need to call
-	//   ForceUnlock on the server where the lock was acquired and is presumably
-	//   'stuck'. Instead dsync.ForceUnlock() will release the underlying locks
-	//   that participated in granting the lock. Any pending dsync locks that
-	//   are blocking can now proceed as normal and any new locks will also
-	//   participate normally.
-	if n.isDistXL { // For distributed mode, broadcast ForceUnlock message.
-		dsync.NewDRWMutex(context.Background(), pathJoin(volume, path), globalDsync).ForceUnlock()
-	}
-
-	// Remove lock from the map.
-	delete(n.lockMap, nsParam{volume, path})
-}
-
 // dsync's distributed lock instance.
 type distLockInstance struct {
 	rwMutex             *dsync.DRWMutex
@@ -301,10 +223,14 @@ type localLockInstance struct {
 // NewNSLock - returns a lock instance for a given volume and
 // path. The returned lockInstance object encapsulates the nsLockMap,
 // volume, path and operation ID.
-func (n *nsLockMap) NewNSLock(ctx context.Context, volume, path string) RWLocker {
+func (n *nsLockMap) NewNSLock(ctx context.Context, lockers []dsync.NetLocker, volume, path string) RWLocker {
 	opsID := mustGetUUID()
 	if n.isDistXL {
-		return &distLockInstance{dsync.NewDRWMutex(ctx, pathJoin(volume, path), globalDsync), volume, path, opsID}
+		sync, err := dsync.New(lockers)
+		if err != nil {
+			logger.CriticalIf(ctx, err)
+		}
+		return &distLockInstance{dsync.NewDRWMutex(ctx, pathJoin(volume, path), sync), volume, path, opsID}
 	}
 	return &localLockInstance{ctx, n, volume, path, opsID}
 }

cmd/namespace-lock_test.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"testing"
 	"time"
 )
@@ -32,7 +31,7 @@ func TestGetSource(t *testing.T) {
 	currentSource := func() string { return getSource() }
 	gotSource := currentSource()
 	// Hard coded line number, 32, in the "expectedSource" value
-	expectedSource := "[namespace-lock_test.go:33:TestGetSource()]"
+	expectedSource := "[namespace-lock_test.go:32:TestGetSource()]"
 	if gotSource != expectedSource {
 		t.Errorf("expected : %s, got : %s", expectedSource, gotSource)
 	}
@@ -41,7 +40,8 @@ func TestGetSource(t *testing.T) {
 // Tests functionality provided by namespace lock.
 func TestNamespaceLockTest(t *testing.T) {
 	isDistXL := false
-	initNSLock(isDistXL)
+	nsMutex := newNSLock(isDistXL)
+
 	// List of test cases.
 	testCases := []struct {
 		lk               func(s1, s2, s3 string, t time.Duration) bool
@@ -53,22 +53,22 @@ func TestNamespaceLockTest(t *testing.T) {
 		shouldPass       bool
 	}{
 		{
-			lk:               globalNSMutex.Lock,
-			unlk:             globalNSMutex.Unlock,
+			lk:               nsMutex.Lock,
+			unlk:             nsMutex.Unlock,
 			lockedRefCount:   1,
 			unlockedRefCount: 0,
 			shouldPass:       true,
 		},
 		{
-			rlk:              globalNSMutex.RLock,
-			runlk:            globalNSMutex.RUnlock,
+			rlk:              nsMutex.RLock,
+			runlk:            nsMutex.RUnlock,
 			lockedRefCount:   4,
 			unlockedRefCount: 2,
 			shouldPass:       true,
 		},
 		{
-			rlk:              globalNSMutex.RLock,
-			runlk:            globalNSMutex.RUnlock,
+			rlk:              nsMutex.RLock,
+			runlk:            nsMutex.RUnlock,
 			lockedRefCount:   1,
 			unlockedRefCount: 0,
 			shouldPass:       true,
@@ -82,7 +82,7 @@ func TestNamespaceLockTest(t *testing.T) {
 	if !testCase.lk("a", "b", "c", 60*time.Second) { // lock once.
 		t.Fatalf("Failed to acquire lock")
 	}
-	nsLk, ok := globalNSMutex.lockMap[nsParam{"a", "b"}]
+	nsLk, ok := nsMutex.lockMap[nsParam{"a", "b"}]
 	if !ok && testCase.shouldPass {
 		t.Errorf("Lock in map missing.")
 	}
@@ -94,7 +94,7 @@ func TestNamespaceLockTest(t *testing.T) {
 	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
 		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 1, testCase.unlockedRefCount, nsLk.ref)
 	}
-	_, ok = globalNSMutex.lockMap[nsParam{"a", "b"}]
+	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
 	if ok && !testCase.shouldPass {
 		t.Errorf("Lock map found after unlock.")
 	}
@@ -113,7 +113,7 @@ func TestNamespaceLockTest(t *testing.T) {
 	if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock fourth time.
 		t.Fatalf("Failed to acquire fourth read lock")
 	}
-	nsLk, ok = globalNSMutex.lockMap[nsParam{"a", "b"}]
+	nsLk, ok = nsMutex.lockMap[nsParam{"a", "b"}]
 	if !ok && testCase.shouldPass {
 		t.Errorf("Lock in map missing.")
 	}
@@ -127,7 +127,7 @@ func TestNamespaceLockTest(t *testing.T) {
 	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
 		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 2, testCase.unlockedRefCount, nsLk.ref)
 	}
-	_, ok = globalNSMutex.lockMap[nsParam{"a", "b"}]
+	_, ok = nsMutex.lockMap[nsParam{"a", "b"}]
 	if !ok && testCase.shouldPass {
 		t.Errorf("Lock map not found.")
 	}
@@ -138,7 +138,7 @@ func TestNamespaceLockTest(t *testing.T) {
 		t.Fatalf("Failed to acquire read lock")
 	}
 
-	nsLk, ok = globalNSMutex.lockMap[nsParam{"a", "c"}]
+	nsLk, ok = nsMutex.lockMap[nsParam{"a", "c"}]
 	if !ok && testCase.shouldPass {
 		t.Errorf("Lock in map missing.")
 	}
@@ -150,7 +150,7 @@ func TestNamespaceLockTest(t *testing.T) {
 	if testCase.unlockedRefCount != nsLk.ref && testCase.shouldPass {
 		t.Errorf("Test %d fails, expected to pass. Wanted ref count is %d, got %d", 3, testCase.unlockedRefCount, nsLk.ref)
 	}
-	_, ok = globalNSMutex.lockMap[nsParam{"a", "c"}]
+	_, ok = nsMutex.lockMap[nsParam{"a", "c"}]
 	if ok && !testCase.shouldPass {
 		t.Errorf("Lock map not found.")
 	}
@@ -158,83 +158,44 @@ func TestNamespaceLockTest(t *testing.T) {
 
 func TestNamespaceLockTimedOut(t *testing.T) {
 	isDistXL := false
-	initNSLock(isDistXL)
+	nsMutex := newNSLock(isDistXL)
 	// Get write lock
-	if !globalNSMutex.Lock("my-bucket", "my-object", "abc", 60*time.Second) {
+	if !nsMutex.Lock("my-bucket", "my-object", "abc", 60*time.Second) {
 		t.Fatalf("Failed to acquire lock")
 	}
 
 	// Second attempt for write lock on same resource should time out
-	locked := globalNSMutex.Lock("my-bucket", "my-object", "def", 1*time.Second)
+	locked := nsMutex.Lock("my-bucket", "my-object", "def", 1*time.Second)
 	if locked {
 		t.Fatalf("Should not have acquired lock")
 	}
 
 	// Read lock on same resource should also time out
-	locked = globalNSMutex.RLock("my-bucket", "my-object", "def", 1*time.Second)
+	locked = nsMutex.RLock("my-bucket", "my-object", "def", 1*time.Second)
 	if locked {
 		t.Fatalf("Should not have acquired read lock while write lock is active")
 	}
 
 	// Release write lock
-	globalNSMutex.Unlock("my-bucket", "my-object", "abc")
+	nsMutex.Unlock("my-bucket", "my-object", "abc")
 
 	// Get read lock
-	if !globalNSMutex.RLock("my-bucket", "my-object", "ghi", 60*time.Second) {
+	if !nsMutex.RLock("my-bucket", "my-object", "ghi", 60*time.Second) {
 		t.Fatalf("Failed to acquire read lock")
 	}
 
 	// Write lock on same resource should time out
-	locked = globalNSMutex.Lock("my-bucket", "my-object", "klm", 1*time.Second)
+	locked = nsMutex.Lock("my-bucket", "my-object", "klm", 1*time.Second)
 	if locked {
 		t.Fatalf("Should not have acquired lock")
 	}
 
 	// 2nd read lock should be just fine
-	if !globalNSMutex.RLock("my-bucket", "my-object", "nop", 60*time.Second) {
+	if !nsMutex.RLock("my-bucket", "my-object", "nop", 60*time.Second) {
 		t.Fatalf("Failed to acquire second read lock")
 	}
 
 	// Release both read locks
-	globalNSMutex.RUnlock("my-bucket", "my-object", "ghi")
-	globalNSMutex.RUnlock("my-bucket", "my-object", "nop")
-}
-
-// Tests functionality to forcefully unlock locks.
-func TestNamespaceForceUnlockTest(t *testing.T) {
-	isDistXL := false
-	initNSLock(isDistXL)
-	// Create lock.
-	lock := globalNSMutex.NewNSLock(context.Background(), "bucket", "object")
-	if lock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
-		t.Fatalf("Failed to get lock")
-	}
-	// Forcefully unlock lock.
-	globalNSMutex.ForceUnlock("bucket", "object")
-
-	ch := make(chan struct{}, 1)
-
-	go func() {
-		// Try to claim lock again.
-		anotherLock := globalNSMutex.NewNSLock(context.Background(), "bucket", "object")
-		if anotherLock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
-			t.Errorf("Failed to get lock")
-			return
-		}
-		// And signal success.
-		ch <- struct{}{}
-	}()
-
-	select {
-	case <-ch:
-		// Signaled so all is fine.
-		break
-
-	case <-time.After(100 * time.Millisecond):
-		// In case we hit the time out, the lock has not been cleared.
-		t.Errorf("Lock not cleared.")
-	}
-
-	// Clean up lock.
-	globalNSMutex.ForceUnlock("bucket", "object")
+	nsMutex.RUnlock("my-bucket", "my-object", "ghi")
+	nsMutex.RUnlock("my-bucket", "my-object", "nop")
 }

cmd/notification.go

@@ -466,7 +466,6 @@ func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
 
 // GetLocks - makes GetLocks RPC call on all peers.
 func (sys *NotificationSys) GetLocks(ctx context.Context) []*PeerLocks {
-
 	locksResp := make([]*PeerLocks, len(sys.peerClients))
 	g := errgroup.WithNErrs(len(sys.peerClients))
 	for index, client := range sys.peerClients {
@@ -703,7 +702,7 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
+	objLock := objAPI.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
 		return err
 	}
@@ -1344,7 +1343,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
+	objLock := objAPI.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
@@ -1395,7 +1394,7 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
+	objLock := objAPI.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/object-api-common.go

@@ -106,7 +106,7 @@ func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) {
 		return &posixDiskIDCheck{storage: storage}, nil
 	}
 
-	return newStorageRESTClient(endpoint)
+	return newStorageRESTClient(endpoint), nil
 }
 
 // Cleanup a directory recursively.

cmd/object-api-interface.go

@@ -48,6 +48,9 @@ const (
 
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
+	// Locking operations on object.
+	NewNSLock(ctx context.Context, bucket string, object string) RWLocker
+
 	// Storage operations.
 	Shutdown(context.Context) error
 	StorageInfo(context.Context) StorageInfo

cmd/peer-rest-client.go

@@ -100,7 +100,7 @@ func (client *peerRESTClient) Close() error {
 }
 
 // GetLocksResp stores various info from the client for each lock that is requested.
-type GetLocksResp map[string][]lockRequesterInfo
+type GetLocksResp []map[string][]lockRequesterInfo
 
 // NetReadPerfInfo - fetch network read performance information for a remote node.
 func (client *peerRESTClient) NetReadPerfInfo(size int64) (info ServerNetReadPerfInfo, err error) {
@@ -759,10 +759,10 @@ func newPeerRESTClient(peer *xnet.Host) (*peerRESTClient, error) {
 		}
 	}
 
-	restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
-
+	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)
+	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
-		return &peerRESTClient{host: peer, restClient: restClient, connected: 0}, err
+		return nil, err
 	}
 
 	return &peerRESTClient{host: peer, restClient: restClient, connected: 1}, nil

cmd/peer-rest-server.go

@@ -148,8 +148,12 @@ func (s *peerRESTServer) GetLocksHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	ctx := newContext(r, w, "GetLocks")
-	locks := globalLockServer.ll.DupLockMap()
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(locks))
+
+	var llockers []map[string][]lockRequesterInfo
+	for _, llocker := range globalLockServers {
+		llockers = append(llockers, llocker.DupLockMap())
+	}
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(llockers))
 
 	w.(http.Flusher).Flush()
 

cmd/prepare-storage.go

@@ -268,14 +268,18 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
 	}
 
 	if format.ID == "" {
-		if err = formatXLFixDeploymentID(context.Background(), endpoints, storageDisks, format); err != nil {
+		// Not a first disk, wait until first disk fixes deploymentID
+		if !firstDisk {
+			return nil, errNotFirstDisk
+		}
+		if err = formatXLFixDeploymentID(endpoints, storageDisks, format); err != nil {
 			return nil, err
 		}
 	}
 
 	globalDeploymentID = format.ID
 
-	if err = formatXLFixLocalDeploymentID(context.Background(), endpoints, storageDisks, format); err != nil {
+	if err = formatXLFixLocalDeploymentID(endpoints, storageDisks, format); err != nil {
 		return nil, err
 	}
 	return format, nil

cmd/rest/client.go

@@ -18,11 +18,9 @@ package rest
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"net/url"
 	"time"
@@ -102,32 +100,11 @@ func (c *Client) Close() {
 	}
 }
 
-func newCustomDialContext(timeout time.Duration) func(ctx context.Context, network, addr string) (net.Conn, error) {
-	return func(ctx context.Context, network, addr string) (net.Conn, error) {
-		dialer := &net.Dialer{
-			Timeout:   timeout,
-			KeepAlive: timeout,
-			DualStack: true,
-		}
-
-		return dialer.DialContext(ctx, network, addr)
-	}
-}
-
 // NewClient - returns new REST client.
-func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAuthToken func() string) (*Client, error) {
+func NewClient(url *url.URL, newCustomTransport func() *http.Transport, newAuthToken func() string) (*Client, error) {
 	// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
 	// except custom DialContext and TLSClientConfig.
-	tr := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           newCustomDialContext(timeout),
-		MaxIdleConnsPerHost:   256,
-		IdleConnTimeout:       60 * time.Second,
-		TLSHandshakeTimeout:   30 * time.Second,
-		ExpectContinueTimeout: 10 * time.Second,
-		TLSClientConfig:       tlsConfig,
-		DisableCompression:    true,
-	}
+	tr := newCustomTransport()
 	return &Client{
 		httpClient:          &http.Client{Transport: tr},
 		httpIdleConnsCloser: tr.CloseIdleConnections,

cmd/routers.go

@@ -31,7 +31,7 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 	registerPeerRESTHandlers(router)
 
 	// Register distributed namespace lock.
-	registerLockRESTHandlers(router)
+	registerLockRESTHandlers(router, endpoints)
 
 }
 

cmd/server-main.go

@@ -27,7 +27,6 @@ import (
 	"syscall"
 
 	"github.com/minio/cli"
-	"github.com/minio/dsync/v2"
 	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -144,12 +143,6 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	var setupType SetupType
 	var err error
 
-	if len(ctx.Args()) > serverCommandLineArgsMax {
-		uErr := config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Invalid total number of endpoints (%d) passed, supported upto 32 unique arguments",
-			len(ctx.Args())))
-		logger.FatalIf(uErr, "Unable to validate passed endpoints")
-	}
-
 	endpoints := strings.Fields(env.Get(config.EnvEndpoints, ""))
 	if len(endpoints) > 0 {
 		globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
@@ -329,21 +322,6 @@ func serverMain(ctx *cli.Context) {
 	// Set system resources to maximum.
 	logger.LogIf(context.Background(), setMaxResources())
 
-	// Set nodes for dsync for distributed setup.
-	if globalIsDistXL {
-		clnts, myNode, err := newDsyncNodes(globalEndpoints)
-		if err != nil {
-			logger.Fatal(err, "Unable to initialize distributed locking on %s", globalEndpoints)
-		}
-		globalDsync, err = dsync.New(clnts, myNode)
-		if err != nil {
-			logger.Fatal(err, "Unable to initialize distributed locking on %s", globalEndpoints)
-		}
-	}
-
-	// Initialize name space lock.
-	initNSLock(globalIsDistXL)
-
 	if globalIsXL {
 		// Init global heal state
 		globalAllHealState = initHealState()

cmd/storage-rest-client.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"crypto/tls"
 	"encoding/gob"
 	"encoding/hex"
@@ -30,6 +31,7 @@ import (
 	"sync/atomic"
 
 	"github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	xnet "github.com/minio/minio/pkg/net"
 )
@@ -446,19 +448,9 @@ func (client *storageRESTClient) Close() error {
 }
 
 // Returns a storage rest client.
-func newStorageRESTClient(endpoint Endpoint) (*storageRESTClient, error) {
-	host, err := xnet.ParseHost(endpoint.Host)
-	if err != nil {
-		return nil, err
-	}
-
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
+func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
 	serverURL := &url.URL{
-		Scheme: scheme,
+		Scheme: endpoint.Scheme,
 		Host:   endpoint.Host,
 		Path:   path.Join(storageRESTPrefix, endpoint.Path, storageRESTVersion),
 	}
@@ -466,16 +458,17 @@ func newStorageRESTClient(endpoint Endpoint) (*storageRESTClient, error) {
 	var tlsConfig *tls.Config
 	if globalIsSSL {
 		tlsConfig = &tls.Config{
-			ServerName: host.Name,
+			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
 			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 
-	restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
+	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)
+	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
-		return nil, err
+		logger.LogIf(context.Background(), err)
+		return &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: 0}
 	}
-	client := &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}
-	return client, nil
+	return &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}
 }

cmd/storage-rest_test.go

@@ -514,11 +514,7 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES
 	}
 
 	registerStorageRESTHandlers(router, EndpointList{endpoint})
-	restClient, err := newStorageRESTClient(endpoint)
-	if err != nil {
-		t.Fatalf("newStorageRESTClient failed for %v, with error %s", endpoint, err)
-	}
-
+	restClient := newStorageRESTClient(endpoint)
 	prevGlobalServerConfig := globalServerConfig
 	globalServerConfig = newServerConfig()
 

cmd/test-utils_test.go

@@ -60,6 +60,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -69,9 +70,6 @@ func init() {
 	// Set as non-distributed.
 	globalIsDistXL = false
 
-	// Initialize name space lock.
-	initNSLock(globalIsDistXL)
-
 	// Disable printing console messages during tests.
 	color.Output = ioutil.Discard
 
@@ -451,13 +449,6 @@ func resetGlobalConfig() {
 	globalServerConfigMu.Unlock()
 }
 
-// reset global NSLock.
-func resetGlobalNSLock() {
-	if globalNSMutex != nil {
-		globalNSMutex = nil
-	}
-}
-
 func resetGlobalEndpoints() {
 	globalEndpoints = EndpointList{}
 }
@@ -497,8 +488,6 @@ func resetTestGlobals() {
 	resetGlobalConfigPath()
 	// Reset Global server config.
 	resetGlobalConfig()
-	// Reset global NSLock.
-	resetGlobalNSLock()
 	// Reset global endpoints.
 	resetGlobalEndpoints()
 	// Reset global isXL flag.
@@ -1629,6 +1618,9 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
 	xl.getDisks = func() []StorageAPI {
 		return xl.storageDisks
 	}
+	xl.getLockers = func() []dsync.NetLocker {
+		return nil
+	}
 
 	globalConfigSys = NewConfigSys()
 
@@ -1918,9 +1910,6 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 	// this is to make sure that the tests are not affected by modified value.
 	resetTestGlobals()
 
-	// initialize NSLock.
-	initNSLock(false)
-
 	objLayer, fsDir, err := prepareFS()
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for single node setup: %s", err)

cmd/utils.go

@@ -332,25 +332,49 @@ func ToS3ETag(etag string) string {
 	return etag
 }
 
+type dialContext func(ctx context.Context, network, address string) (net.Conn, error)
+
+func newCustomDialContext(dialTimeout, dialKeepAlive time.Duration) dialContext {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		dialer := &net.Dialer{
+			Timeout:   dialTimeout,
+			KeepAlive: dialKeepAlive,
+			DualStack: true,
+		}
+
+		return dialer.DialContext(ctx, network, addr)
+	}
+}
+
+func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout, dialKeepAlive time.Duration) func() *http.Transport {
+	// For more details about various values used here refer
+	// https://golang.org/pkg/net/http/#Transport documentation
+	tr := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           newCustomDialContext(dialTimeout, dialKeepAlive),
+		MaxIdleConnsPerHost:   256,
+		IdleConnTimeout:       60 * time.Second,
+		TLSHandshakeTimeout:   30 * time.Second,
+		ExpectContinueTimeout: 10 * time.Second,
+		TLSClientConfig:       tlsConfig,
+		// Go net/http automatically unzip if content-type is
+		// gzip disable this feature, as we are always interested
+		// in raw stream.
+		DisableCompression: true,
+	}
+	return func() *http.Transport {
+		return tr
+	}
+}
+
 // NewCustomHTTPTransport returns a new http configuration
 // used while communicating with the cloud backends.
 // This sets the value for MaxIdleConnsPerHost from 2 (go default)
-// to 100.
+// to 256.
 func NewCustomHTTPTransport() *http.Transport {
-	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   defaultDialTimeout,
-			KeepAlive: defaultDialKeepAlive,
-		}).DialContext,
-		MaxIdleConns:          1024,
-		MaxIdleConnsPerHost:   1024,
-		IdleConnTimeout:       30 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
-		DisableCompression:    true,
-	}
+	return newCustomHTTPTransport(&tls.Config{
+		RootCAs: globalRootCAs,
+	}, defaultDialTimeout, defaultDialKeepAlive)()
 }
 
 // Load the json (typically from disk file).

cmd/xl-sets.go

@@ -29,6 +29,7 @@ import (
 	"github.com/minio/minio/cmd/config/storageclass"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
@@ -38,6 +39,9 @@ import (
 // setsStorageAPI is encapsulated type for Close()
 type setsStorageAPI [][]StorageAPI
 
+// setsDsyncLockers is encapsulated type for Close()
+type setsDsyncLockers [][]dsync.NetLocker
+
 func (s setsStorageAPI) Close() error {
 	for i := 0; i < len(s); i++ {
 		for j, disk := range s[i] {
@@ -51,6 +55,18 @@ func (s setsStorageAPI) Close() error {
 	return nil
 }
 
+func (s setsDsyncLockers) Close() error {
+	for i := 0; i < len(s); i++ {
+		for _, locker := range s[i] {
+			if locker == nil {
+				continue
+			}
+			locker.Close()
+		}
+	}
+	return nil
+}
+
 // xlSets implements ObjectLayer combining a static list of erasure coded
 // object sets. NOTE: There is no dynamic scaling allowed or intended in
 // current design.
@@ -66,6 +82,12 @@ type xlSets struct {
 	// Re-ordered list of disks per set.
 	xlDisks setsStorageAPI
 
+	// Distributed locker clients.
+	xlLockers setsDsyncLockers
+
+	// Lockers map holds dsync lockers for each endpoint
+	lockersMap map[Endpoint]dsync.NetLocker
+
 	// List of endpoints provided on the command line.
 	endpoints EndpointList
 
@@ -101,7 +123,11 @@ func (s *xlSets) isConnected(endpoint Endpoint) bool {
 			if s.xlDisks[i][j].String() != endpointStr {
 				continue
 			}
-			return s.xlDisks[i][j].IsOnline()
+			if s.xlDisks[i][j].IsOnline() {
+				return true
+			}
+			s.xlLockers[i][j].Close()
+			return false
 		}
 	}
 	return false
@@ -147,10 +173,9 @@ func findDiskIndex(refFormat, format *formatXLV3) (int, int, error) {
 	return -1, -1, fmt.Errorf("diskID: %s not found", format.XL.This)
 }
 
-// connectDisksWithQuorum is same as connectDisks but waits
-// for quorum number of formatted disks to be online in
-// any given sets.
-func (s *xlSets) connectDisksWithQuorum() {
+// connectDisksAndLockersWithQuorum is same as connectDisksAndLockers but waits
+// for quorum number of formatted disks to be online in any given sets.
+func (s *xlSets) connectDisksAndLockersWithQuorum() {
 	var onlineDisks int
 	for onlineDisks < len(s.endpoints)/2 {
 		for _, endpoint := range s.endpoints {
@@ -171,6 +196,7 @@ func (s *xlSets) connectDisksWithQuorum() {
 			}
 			disk.SetDiskID(format.XL.This)
 			s.xlDisks[i][j] = disk
+			s.xlLockers[i][j] = s.lockersMap[endpoint]
 			onlineDisks++
 		}
 		// Sleep for a while - so that we don't go into
@@ -179,9 +205,9 @@ func (s *xlSets) connectDisksWithQuorum() {
 	}
 }
 
-// connectDisks - attempt to connect all the endpoints, loads format
+// connectDisksAndLockers - attempt to connect all the endpoints, loads format
 // and re-arranges the disks in proper position.
-func (s *xlSets) connectDisks() {
+func (s *xlSets) connectDisksAndLockers() {
 	for _, endpoint := range s.endpoints {
 		if s.isConnected(endpoint) {
 			continue
@@ -201,6 +227,7 @@ func (s *xlSets) connectDisks() {
 		disk.SetDiskID(format.XL.This)
 		s.xlDisksMu.Lock()
 		s.xlDisks[i][j] = disk
+		s.xlLockers[i][j] = s.lockersMap[endpoint]
 		s.xlDisksMu.Unlock()
 	}
 }
@@ -220,11 +247,27 @@ func (s *xlSets) monitorAndConnectEndpoints(monitorInterval time.Duration) {
 		case <-s.disksConnectDoneCh:
 			return
 		case <-ticker.C:
-			s.connectDisks()
+			s.connectDisksAndLockers()
 		}
 	}
 }
 
+func (s *xlSets) GetLockers(setIndex int) func() []dsync.NetLocker {
+	return func() []dsync.NetLocker {
+		s.xlDisksMu.Lock()
+		defer s.xlDisksMu.Unlock()
+		lockers := make([]dsync.NetLocker, s.drivesPerSet)
+		copy(lockers, s.xlLockers[setIndex])
+		for i, lk := range lockers {
+			// Add error lockers for unavailable locker.
+			if lk == nil {
+				lockers[i] = &errorLocker{}
+			}
+		}
+		return lockers
+	}
+}
+
 // GetDisks returns a closure for a given set, which provides list of disks per set.
 func (s *xlSets) GetDisks(setIndex int) func() []StorageAPI {
 	return func() []StorageAPI {
@@ -241,10 +284,17 @@ const defaultMonitorConnectEndpointInterval = time.Second * 10 // Set to 10 secs
 // Initialize new set of erasure coded sets.
 func newXLSets(endpoints EndpointList, format *formatXLV3, setCount int, drivesPerSet int) (ObjectLayer, error) {
 
+	lockersMap := make(map[Endpoint]dsync.NetLocker)
+	for _, endpoint := range endpoints {
+		lockersMap[endpoint] = newLockAPI(endpoint)
+	}
+
 	// Initialize the XL sets instance.
 	s := &xlSets{
 		sets:               make([]*xlObjects, setCount),
 		xlDisks:            make([][]StorageAPI, setCount),
+		xlLockers:          make([][]dsync.NetLocker, setCount),
+		lockersMap:         lockersMap,
 		endpoints:          endpoints,
 		setCount:           setCount,
 		drivesPerSet:       drivesPerSet,
@@ -262,18 +312,21 @@ func newXLSets(endpoints EndpointList, format *formatXLV3, setCount int, drivesP
 
 	for i := 0; i < len(format.XL.Sets); i++ {
 		s.xlDisks[i] = make([]StorageAPI, drivesPerSet)
+		s.xlLockers[i] = make([]dsync.NetLocker, drivesPerSet)
 
 		// Initialize xl objects for a given set.
 		s.sets[i] = &xlObjects{
-			getDisks: s.GetDisks(i),
-			nsMutex:  mutex,
-			bp:       bp,
+			getDisks:   s.GetDisks(i),
+			getLockers: s.GetLockers(i),
+			nsMutex:    mutex,
+			bp:         bp,
 		}
-		go s.sets[i].cleanupStaleMultipartUploads(context.Background(), GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
+		go s.sets[i].cleanupStaleMultipartUploads(context.Background(),
+			GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
 	}
 
 	// Connect disks right away, but wait until we have `format.json` quorum.
-	s.connectDisksWithQuorum()
+	s.connectDisksAndLockersWithQuorum()
 
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
@@ -281,6 +334,11 @@ func newXLSets(endpoints EndpointList, format *formatXLV3, setCount int, drivesP
 	return s, nil
 }
 
+// NewNSLock - initialize a new namespace RWLocker instance.
+func (s *xlSets) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+	return s.getHashedSet(object).NewNSLock(ctx, bucket, object)
+}
+
 // StorageInfo - combines output of StorageInfo across all erasure coded object sets.
 func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
 	var storageInfo StorageInfo
@@ -406,6 +464,13 @@ func (s *xlSets) Shutdown(ctx context.Context) error {
 // even if one of the sets fail to create buckets, we proceed to undo a
 // successful operation.
 func (s *xlSets) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
+	set := s.getHashedSet(bucket)
+	bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
+	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer bucketLock.Unlock()
+
 	g := errgroup.WithNErrs(len(s.sets))
 
 	// Create buckets in parallel across all sets.
@@ -484,6 +549,13 @@ func (s *xlSets) getHashedSet(input string) (set *xlObjects) {
 
 // GetBucketInfo - returns bucket info from one of the erasure coded set.
 func (s *xlSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
+	set := s.getHashedSet(bucket)
+	bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
+	if err = bucketLock.GetRLock(globalOperationTimeout); err != nil {
+		return bucketInfo, err
+	}
+	defer bucketLock.RUnlock()
+
 	return s.getHashedSet(bucket).GetBucketInfo(ctx, bucket)
 }
 
@@ -563,6 +635,13 @@ func (s *xlSets) IsCompressionSupported() bool {
 // even if one of the sets fail to delete buckets, we proceed to
 // undo a successful operation.
 func (s *xlSets) DeleteBucket(ctx context.Context, bucket string) error {
+	set := s.getHashedSet(bucket)
+	bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
+	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer bucketLock.Unlock()
+
 	g := errgroup.WithNErrs(len(s.sets))
 
 	// Delete buckets in parallel across all sets.
@@ -709,7 +788,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 	}
 
 	if !cpSrcDstSame {
-		objectDWLock := destSet.nsMutex.NewNSLock(ctx, destBucket, destObject)
+		objectDWLock := destSet.nsMutex.NewNSLock(ctx, destSet.getLockers(), destBucket, destObject)
 		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return objInfo, err
 		}
@@ -1239,7 +1318,8 @@ func formatsToDrivesInfo(endpoints EndpointList, formats []*formatXLV3, sErrs []
 // healing in a distributed setup.
 func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
 	// Acquire lock on format.json
-	formatLock := s.getHashedSet(formatConfigFile).nsMutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
+	set := s.getHashedSet(formatConfigFile)
+	formatLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), minioMetaBucket, formatConfigFile)
 	if err = formatLock.GetRLock(globalHealingTimeout); err != nil {
 		return err
 	}
@@ -1285,9 +1365,10 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
 	// Replace the new format.
 	s.format = refFormat
 
-	// Close all existing disks and reconnect all the disks.
+	// Close all existing disks, lockers and reconnect all the disks/lockers.
 	s.xlDisks.Close()
-	s.connectDisks()
+	s.xlLockers.Close()
+	s.connectDisksAndLockers()
 
 	// Restart monitoring loop to monitor reformatted disks again.
 	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
@@ -1356,7 +1437,8 @@ func markRootDisksAsDown(storageDisks []StorageAPI) {
 // coded data in it.
 func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
 	// Acquire lock on format.json
-	formatLock := s.getHashedSet(formatConfigFile).nsMutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
+	set := s.getHashedSet(formatConfigFile)
+	formatLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), minioMetaBucket, formatConfigFile)
 	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
 		return madmin.HealResultItem{}, err
 	}
@@ -1492,9 +1574,10 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 		// Replace with new reference format.
 		s.format = refFormat
 
-		// Disconnect/relinquish all existing disks and reconnect the disks.
+		// Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers.
 		s.xlDisks.Close()
-		s.connectDisks()
+		s.xlLockers.Close()
+		s.connectDisksAndLockers()
 
 		// Restart our monitoring loop to start monitoring newly formatted disks.
 		go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
@@ -1505,8 +1588,9 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 
 // HealBucket - heals inconsistent buckets and bucket metadata on all sets.
 func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (result madmin.HealResultItem, err error) {
-	bucketLock := globalNSMutex.NewNSLock(ctx, bucket, "")
-	if err := bucketLock.GetLock(globalHealingTimeout); err != nil {
+	set := s.getHashedSet(bucket)
+	bucketLock := set.nsMutex.NewNSLock(ctx, set.getLockers(), bucket, "")
+	if err = bucketLock.GetLock(globalOperationTimeout); err != nil {
 		return result, err
 	}
 	defer bucketLock.Unlock()

cmd/xl-v1-bucket.go

@@ -140,11 +140,6 @@ func (xl xlObjects) getBucketInfo(ctx context.Context, bucketName string) (bucke
 
 // GetBucketInfo - returns BucketInfo for a bucket.
 func (xl xlObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketLock := xl.nsMutex.NewNSLock(ctx, bucket, "")
-	if e := bucketLock.GetRLock(globalObjectTimeout); e != nil {
-		return bi, e
-	}
-	defer bucketLock.RUnlock()
 	bucketInfo, err := xl.getBucketInfo(ctx, bucket)
 	if err != nil {
 		return bi, toObjectErr(err, bucket)
@@ -226,12 +221,6 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
 
 // DeleteBucket - deletes a bucket.
 func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
-	bucketLock := xl.nsMutex.NewNSLock(ctx, bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		return err
-	}
-	defer bucketLock.Unlock()
-
 	// Collect if all disks report volume not found.
 	storageDisks := xl.getDisks()
 

cmd/xl-v1-healing.go

@@ -681,7 +681,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 
 	// Lock the object before healing. Use read lock since healing
 	// will only regenerate parts & xl.json of outdated disks.
-	objectLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 	if lerr := objectLock.GetRLock(globalHealingTimeout); lerr != nil {
 		return madmin.HealResultItem{}, lerr
 	}

cmd/xl-v1-multipart.go

@@ -264,7 +264,7 @@ func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object strin
 func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
 	// Hold read locks on source object only if we are
 	// going to read data from source object.
-	objectSRLock := xl.nsMutex.NewNSLock(ctx, srcBucket, srcObject)
+	objectSRLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), srcBucket, srcObject)
 	if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
 		return pi, err
 	}
@@ -306,7 +306,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
 
 	// pre-check upload id lock.
-	preUploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
+	preUploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
 	if err := preUploadIDLock.GetRLock(globalOperationTimeout); err != nil {
 		return pi, err
 	}
@@ -404,7 +404,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	// post-upload check (write) lock
-	postUploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
+	postUploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
 	if err = postUploadIDLock.GetLock(globalOperationTimeout); err != nil {
 		return pi, err
 	}
@@ -499,7 +499,8 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI
 	}
 	// Hold lock so that there is no competing
 	// abort-multipart-upload or complete-multipart-upload.
-	uploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket,
+	uploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(),
+		minioMetaMultipartBucket,
 		xl.getUploadIDLockPath(bucket, object, uploadID))
 	if err := uploadIDLock.GetLock(globalListingTimeout); err != nil {
 		return result, err
@@ -603,7 +604,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, err
 	}
 	// Hold write lock on the object.
-	destLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
+	destLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 	if err := destLock.GetLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}
@@ -618,7 +619,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	//
 	// 2) no one does a parallel complete-multipart-upload on this
 	// multipart upload
-	uploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
+	uploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
 	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
 		return oi, err
 	}
@@ -824,7 +825,7 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
 	uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
 	// Hold lock so that there is no competing
 	// complete-multipart-upload or put-object-part.
-	uploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
+	uploadIDLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), minioMetaMultipartBucket, uploadIDLockPath)
 	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/xl-v1-object.go

@@ -127,7 +127,7 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 
 	// Acquire lock
 	if lockType != noLock {
-		lock := xl.nsMutex.NewNSLock(ctx, bucket, object)
+		lock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 		switch lockType {
 		case writeLock:
 			if err = lock.GetLock(globalObjectTimeout); err != nil {
@@ -190,7 +190,7 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 // length indicates the total length of the object.
 func (xl xlObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
 	// Lock the object before reading.
-	objectLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		return err
 	}
@@ -369,7 +369,7 @@ func (xl xlObjects) getObjectInfoDir(ctx context.Context, bucket, object string)
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
-	objectLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}
@@ -498,7 +498,7 @@ func (xl xlObjects) PutObject(ctx context.Context, bucket string, object string,
 	}
 
 	// Lock the object.
-	objectLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
 		return objInfo, err
 	}
@@ -851,7 +851,7 @@ func (xl xlObjects) deleteObjects(ctx context.Context, bucket string, objects []
 			continue
 		}
 		// Acquire a write lock before deleting the object.
-		objectLocks[i] = xl.nsMutex.NewNSLock(ctx, bucket, object)
+		objectLocks[i] = xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 		if errs[i] = objectLocks[i].GetLock(globalOperationTimeout); errs[i] != nil {
 			continue
 		}
@@ -954,7 +954,7 @@ func (xl xlObjects) DeleteObjects(ctx context.Context, bucket string, objects []
 // response to the client request.
 func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (err error) {
 	// Acquire a write lock before deleting the object.
-	objectLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
+	objectLock := xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
 	if perr := objectLock.GetLock(globalOperationTimeout); perr != nil {
 		return perr
 	}

cmd/xl-v1.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -39,12 +40,15 @@ var OfflineDisk StorageAPI // zero value is nil
 
 // xlObjects - Implements XL object layer.
 type xlObjects struct {
-	// name space mutex for object layer.
-	nsMutex *nsLockMap
-
 	// getDisks returns list of storageAPIs.
 	getDisks func() []StorageAPI
 
+	// getLockers returns list of remote and local lockers.
+	getLockers func() []dsync.NetLocker
+
+	// Locker mutex map.
+	nsMutex *nsLockMap
+
 	// Byte pools used for temporary i/o buffers.
 	bp *bpool.BytePoolCap
 
@@ -55,10 +59,16 @@ type xlObjects struct {
 	listPool *TreeWalkPool
 }
 
+// NewNSLock - initialize a new namespace RWLocker instance.
+func (xl xlObjects) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
+	return xl.nsMutex.NewNSLock(ctx, xl.getLockers(), bucket, object)
+}
+
 // Shutdown function for object storage interface.
 func (xl xlObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
 	closeStorageDisks(xl.getDisks())
+	closeLockers(xl.getLockers())
 	return nil
 }