Add large bucket support for erasure coded backend (#5160)

This PR implements an object layer which
combines input erasure sets of XL layers
into a unified namespace.

This object layer extends the existing
erasure coded implementation, it is assumed
in this design that providing > 16 disks is
a static configuration as well i.e if you started
the setup with 32 disks with 4 sets 8 disks per
pack then you would need to provide 4 sets always.

Some design details and restrictions:

- Objects are distributed using consistent ordering
  to a unique erasure coded layer.
- Each pack has its own dsync so locks are synchronized
  properly at pack (erasure layer).
- Each pack still has a maximum of 16 disks
  requirement, you can start with multiple
  such sets statically.
- Static sets set of disks and cannot be
  changed, there is no elastic expansion allowed.
- Static sets set of disks and cannot be
  changed, there is no elastic removal allowed.
- ListObjects() across sets can be noticeably
  slower since List happens on all servers,
  and is merged at this sets layer.

Fixes #5465
Fixes #5464
Fixes #5461
Fixes #5460
Fixes #5459
Fixes #5458
Fixes #5460
Fixes #5488
Fixes #5489
Fixes #5497
Fixes #5496

cmd/admin-heal-ops.go

@@ -528,58 +528,15 @@ func (h *healSequence) healDiskFormat() error {
 		return errServerNotInitialized
 	}
 
-	// Acquire lock on format.json
-	formatLock := globalNSMutex.NewNSLock(minioMetaBucket, formatConfigFile)
-	if err := formatLock.GetLock(globalHealingTimeout); err != nil {
-		return errFnHealFromAPIErr(err)
-	}
-	defer formatLock.Unlock()
-
-	// Create a new set of storage instances to heal format.json.
-	bootstrapDisks, err := initStorageDisks(globalEndpoints)
+	res, err := objectAPI.HealFormat(h.settings.DryRun)
 	if err != nil {
 		return errFnHealFromAPIErr(err)
 	}
 
-	// Wrap into retrying disks
-	retryingDisks := initRetryableStorageDisks(bootstrapDisks,
-		time.Millisecond, time.Millisecond*5,
-		globalStorageHealthCheckInterval, globalStorageRetryThreshold)
-
-	// Heal format.json on available storage.
-	hres, err := healFormatXL(retryingDisks, h.settings.DryRun)
-	if err != nil {
-		return errFnHealFromAPIErr(err)
-	}
-
-	// reload object layer global only if we healed some disk
-	onlineBefore, onlineAfter := hres.GetOnlineCounts()
-	numHealed := onlineAfter - onlineBefore
-	if numHealed > 0 {
-		// Instantiate new object layer with newly formatted
-		// storage.
-		newObjectAPI, err := newXLObjects(retryingDisks)
-		if err != nil {
-			return errFnHealFromAPIErr(err)
-		}
-
-		// Set object layer with newly formatted storage to
-		// globalObjectAPI.
-		globalObjLayerMutex.Lock()
-		globalObjectAPI = newObjectAPI
-		globalObjLayerMutex.Unlock()
-
-		// Shutdown storage belonging to old object layer
-		// instance.
-		objectAPI.Shutdown()
-
-		// Inform peers to reinitialize storage with newly
-		// formatted storage.
-		reInitPeerDisks(globalAdminPeers)
-	}
+	peersReInitFormat(globalAdminPeers, h.settings.DryRun)
 
 	// Push format heal result
-	return h.pushHealResultItem(hres)
+	return h.pushHealResultItem(res)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
@@ -601,8 +558,7 @@ func (h *healSequence) healBuckets() error {
 	}
 
 	for _, bucket := range buckets {
-		err = h.healBucket(bucket.Name)
-		if err != nil {
+		if err = h.healBucket(bucket.Name); err != nil {
 			return err
 		}
 	}
@@ -615,19 +571,27 @@ func (h *healSequence) healBucket(bucket string) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
+	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	if err := bucketLock.GetLock(globalHealingTimeout); err != nil {
+		return err
+	}
+
 	results, err := objectAPI.HealBucket(bucket, h.settings.DryRun)
 	// push any available results before checking for error
 	for _, result := range results {
 		if perr := h.pushHealResultItem(result); perr != nil {
+			bucketLock.Unlock()
 			return perr
 		}
 	}
+	bucketLock.Unlock()
 	// handle heal-bucket error
 	if err != nil {
 		return err