heal: Add ListBucketsHeal object API (#3563)

ListBucketsHeal will list which buckets that need to be healed:
  * ListBucketsHeal() (buckets []BucketInfo, err error)

cmd/admin-handlers.go

@@ -345,6 +345,34 @@ func (adminAPI adminAPIHandlers) ListObjectsHealHandler(w http.ResponseWriter, r
 	writeSuccessResponseXML(w, encodeResponse(listResponse))
 }
 
+// ListBucketsHealHandler - GET /?heal
+func (adminAPI adminAPIHandlers) ListBucketsHealHandler(w http.ResponseWriter, r *http.Request) {
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Get the list buckets to be healed.
+	bucketsInfo, err := objLayer.ListBucketsHeal()
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	listResponse := generateListBucketsResponse(bucketsInfo)
+	// Write success response.
+	writeSuccessResponseXML(w, encodeResponse(listResponse))
+}
+
 // HealBucketHandler - POST /?heal&bucket=mybucket
 // - bucket is mandatory query parameter
 // Heal a given bucket, if present.

cmd/admin-handlers_test.go

@@ -687,7 +687,7 @@ func TestListObjectsHealHandler(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d - Failed to construct list objects needing heal request - %v", i+1, err)
 		}
-		req.Header.Set(minioAdminOpHeader, "list")
+		req.Header.Set(minioAdminOpHeader, "list-objects")
 
 		cred := serverConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)

cmd/admin-router.go

@@ -49,7 +49,10 @@ func registerAdminRouter(mux *router.Router) {
 	/// Heal operations
 
 	// List Objects needing heal.
-	adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list").HandlerFunc(adminAPI.ListObjectsHealHandler)
+	adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-objects").HandlerFunc(adminAPI.ListObjectsHealHandler)
+	// List Buckets needing heal.
+	adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-buckets").HandlerFunc(adminAPI.ListBucketsHealHandler)
+
 	// Heal Buckets.
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler)
 	// Heal Objects.

cmd/api-response.go

@@ -181,8 +181,9 @@ type CommonPrefix struct {
 
 // Bucket container for bucket metadata
 type Bucket struct {
-	Name         string
-	CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
+	Name           string
+	CreationDate   string          // time string of format "2006-01-02T15:04:05.000Z"
+	HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"`
 }
 
 // Object container for object metadata
@@ -196,8 +197,8 @@ type Object struct {
 	Owner Owner
 
 	// The class of storage used to store the object.
-	StorageClass string
-	HealInfo     *HealInfo `xml:"HealInfo,omitempty"`
+	StorageClass   string
+	HealObjectInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -285,6 +286,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
 		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZLong)
+		listbucket.HealBucketInfo = bucket.HealBucketInfo
 		listbuckets = append(listbuckets, listbucket)
 	}
 
@@ -317,8 +319,8 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		content.Size = object.Size
 		content.StorageClass = globalMinioDefaultStorageClass
 		content.Owner = owner
-		// object.HealInfo is non-empty only when resp is constructed in ListObjectsHeal.
-		content.HealInfo = object.HealInfo
+		// object.HealObjectInfo is non-empty only when resp is constructed in ListObjectsHeal.
+		content.HealObjectInfo = object.HealObjectInfo
 		contents = append(contents, content)
 	}
 	// TODO - support EncodingType in xml decoding

cmd/fs-v1.go

@@ -897,3 +897,8 @@ func (fs fsObjects) HealBucket(bucket string) error {
 func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	return ListObjectsInfo{}, traceError(NotImplemented{})
 }
+
+// ListBucketsHeal - list all buckets to be healed. Valid only for XL
+func (fs fsObjects) ListBucketsHeal() ([]BucketInfo, error) {
+	return []BucketInfo{}, traceError(NotImplemented{})
+}

cmd/object-api-datatypes.go

@@ -50,6 +50,20 @@ type StorageInfo struct {
 	}
 }
 
+type healStatus int
+
+const (
+	healthy           healStatus = iota // Object is healthy
+	canHeal                             // Object can be healed
+	corrupted                           // Object can't be healed
+	quorumUnavailable                   // Object can't be healed until read quorum is available
+)
+
+// HealBucketInfo - represents healing related information of a bucket.
+type HealBucketInfo struct {
+	Status healStatus
+}
+
 // BucketInfo - represents bucket metadata.
 type BucketInfo struct {
 	// Name of the bucket.
@@ -57,18 +71,13 @@ type BucketInfo struct {
 
 	// Date and time when the bucket was created.
 	Created time.Time
+
+	// Healing information
+	HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"`
 }
 
-type healStatus int
-
-const (
-	canHeal           healStatus = iota // Object can be healed
-	corrupted                           // Object can't be healed
-	quorumUnavailable                   // Object can't be healed until read quorum is available
-)
-
-// HealInfo - represents healing related information of an object.
-type HealInfo struct {
+// HealObjectInfo - represents healing related information of an object.
+type HealObjectInfo struct {
 	Status              healStatus
 	MissingDataCount    int
 	MissingPartityCount int
@@ -103,8 +112,8 @@ type ObjectInfo struct {
 	ContentEncoding string
 
 	// User-Defined metadata
-	UserDefined map[string]string
-	HealInfo    *HealInfo `xml:"HealInfo,omitempty"`
+	UserDefined    map[string]string
+	HealObjectInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"`
 }
 
 // ListPartsInfo - represents list of all parts.

cmd/object-api-interface.go

@@ -48,6 +48,7 @@ type ObjectLayer interface {
 
 	// Healing operations.
 	HealBucket(bucket string) error
+	ListBucketsHeal() (buckets []BucketInfo, err error)
 	HealObject(bucket, object string) error
 	ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
 }

cmd/xl-v1-healing-common.go

@@ -141,12 +141,12 @@ func xlShouldHeal(partsMetadata []xlMetaV1, errs []error) bool {
 // xlHealStat - returns a structure which describes how many data,
 // parity erasure blocks are missing and if it is possible to heal
 // with the blocks present.
-func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealInfo {
+func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealObjectInfo {
 	// Less than quorum erasure coded blocks of the object have the same create time.
 	// This object can't be healed with the information we have.
 	modTime, count := commonTime(listObjectModtimes(partsMetadata, errs))
 	if count < xl.readQuorum {
-		return HealInfo{
+		return HealObjectInfo{
 			Status:              quorumUnavailable,
 			MissingDataCount:    0,
 			MissingPartityCount: 0,
@@ -156,7 +156,7 @@ func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealInfo {
 	// If there isn't a valid xlMeta then we can't heal the object.
 	xlMeta, err := pickValidXLMeta(partsMetadata, modTime)
 	if err != nil {
-		return HealInfo{
+		return HealObjectInfo{
 			Status:              corrupted,
 			MissingDataCount:    0,
 			MissingPartityCount: 0,
@@ -183,7 +183,7 @@ func xlHealStat(xl xlObjects, partsMetadata []xlMetaV1, errs []error) HealInfo {
 
 	// This object can be healed. We have enough object metadata
 	// to reconstruct missing erasure coded blocks.
-	return HealInfo{
+	return HealObjectInfo{
 		Status:              canHeal,
 		MissingDataCount:    missingDataCount,
 		MissingPartityCount: missingParityCount,

cmd/xl-v1-healing.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"path"
+	"sort"
 	"sync"
 )
 
@@ -153,9 +154,11 @@ func healBucketMetadata(storageDisks []StorageAPI, bucket string, readQuorum int
 	return healBucketMetaFn(lConfigPath)
 }
 
-// listBucketNames list all bucket names from all disks to heal.
-func listBucketNames(storageDisks []StorageAPI) (bucketNames map[string]struct{}, err error) {
-	bucketNames = make(map[string]struct{})
+// listAllBuckets lists all buckets from all disks. It also
+// returns the occurrence of each buckets in all disks
+func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo, bucketsOcc map[string]int, err error) {
+	buckets = make(map[string]VolInfo)
+	bucketsOcc = make(map[string]int)
 	for _, disk := range storageDisks {
 		if disk == nil {
 			continue
@@ -173,7 +176,10 @@ func listBucketNames(storageDisks []StorageAPI) (bucketNames map[string]struct{}
 				if isMinioMetaBucketName(volInfo.Name) {
 					continue
 				}
-				bucketNames[volInfo.Name] = struct{}{}
+				// Increase counter per bucket name
+				bucketsOcc[volInfo.Name]++
+				// Save volume info under bucket name
+				buckets[volInfo.Name] = volInfo
 			}
 			continue
 		}
@@ -183,7 +189,101 @@ func listBucketNames(storageDisks []StorageAPI) (bucketNames map[string]struct{}
 		}
 		break
 	}
-	return bucketNames, err
+	return buckets, bucketsOcc, err
+}
+
+// reduceHealStatus - fetches the worst heal status in a provided slice
+func reduceHealStatus(status []healStatus) healStatus {
+	worstStatus := healthy
+	for _, st := range status {
+		if st > worstStatus {
+			worstStatus = st
+		}
+	}
+	return worstStatus
+}
+
+// bucketHealStatus - returns the heal status of the provided bucket. Internally,
+// this function lists all object heal status of objects inside meta bucket config
+// directory and returns the worst heal status that can be found
+func (xl xlObjects) bucketHealStatus(bucketName string) (healStatus, error) {
+	// A list of all the bucket config files
+	configFiles := []string{bucketPolicyConfig, bucketNotificationConfig, bucketListenerConfig}
+	// The status of buckets config files
+	configsHealStatus := make([]healStatus, len(configFiles))
+	// The list of errors found during checking heal status of each config file
+	configsErrs := make([]error, len(configFiles))
+	// The path of meta bucket that contains all config files
+	configBucket := path.Join(minioMetaBucket, bucketConfigPrefix, bucketName)
+
+	// Check of config files heal status in go-routines
+	var wg sync.WaitGroup
+	// Loop over config files
+	for idx, configFile := range configFiles {
+		wg.Add(1)
+		// Compute heal status of current config file
+		go func(bucket, object string, index int) {
+			defer wg.Done()
+			// Check
+			listObjectsHeal, err := xl.listObjectsHeal(bucket, object, "", "", 1)
+			// If any error, save and immediately quit
+			if err != nil {
+				configsErrs[index] = err
+				return
+			}
+			// Check if current bucket contains any not healthy config file and save heal status
+			if len(listObjectsHeal.Objects) > 0 {
+				configsHealStatus[index] = listObjectsHeal.Objects[0].HealObjectInfo.Status
+			}
+		}(configBucket, configFile, idx)
+	}
+	wg.Wait()
+
+	// Return any found error
+	for _, err := range configsErrs {
+		if err != nil {
+			return healthy, err
+		}
+	}
+
+	// Reduce and return heal status
+	return reduceHealStatus(configsHealStatus), nil
+}
+
+// ListBucketsHeal - Find all buckets that need to be healed
+func (xl xlObjects) ListBucketsHeal() ([]BucketInfo, error) {
+	listBuckets := []BucketInfo{}
+	// List all buckets that can be found in all disks
+	buckets, occ, err := listAllBuckets(xl.storageDisks)
+	if err != nil {
+		return listBuckets, err
+	}
+	// Iterate over all buckets
+	for _, currBucket := range buckets {
+		// Check the status of bucket metadata
+		bucketHealStatus, err := xl.bucketHealStatus(currBucket.Name)
+		if err != nil {
+			return []BucketInfo{}, err
+		}
+		// If all metadata are sane, check if the bucket directory is present in all disks
+		if bucketHealStatus == healthy && occ[currBucket.Name] != len(xl.storageDisks) {
+			// Current bucket is missing in some of the storage disks
+			bucketHealStatus = canHeal
+		}
+		// Add current bucket to the returned result if not healthy
+		if bucketHealStatus != healthy {
+			listBuckets = append(listBuckets,
+				BucketInfo{
+					Name:           currBucket.Name,
+					Created:        currBucket.Created,
+					HealBucketInfo: &HealBucketInfo{Status: bucketHealStatus},
+				})
+		}
+
+	}
+	// Sort found buckets
+	sort.Sort(byBucketName(listBuckets))
+	return listBuckets, nil
 }
 
 // This function is meant for all the healing that needs to be done
@@ -196,7 +296,7 @@ func listBucketNames(storageDisks []StorageAPI) (bucketNames map[string]struct{}
 // - add support for healing dangling `xl.json`.
 func quickHeal(storageDisks []StorageAPI, writeQuorum int, readQuorum int) error {
 	// List all bucket names from all disks.
-	bucketNames, err := listBucketNames(storageDisks)
+	bucketNames, _, err := listAllBuckets(storageDisks)
 	if err != nil {
 		return err
 	}

cmd/xl-v1-healing_test.go

@@ -423,3 +423,66 @@ func TestQuickHeal(t *testing.T) {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 }
+
+// TestListBucketsHeal lists buckets heal result
+func TestListBucketsHeal(t *testing.T) {
+	root, err := newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeAll(root)
+
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeRoots(fsDirs)
+
+	endpoints, err := parseStorageEndpoints(fsDirs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	obj, _, err := initObjectLayer(endpoints)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a bucket that won't get corrupted
+	saneBucket := "sanebucket"
+	if err = obj.MakeBucket(saneBucket); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a bucket that will be removed in some disks
+	corruptedBucketName := getRandomBucketName()
+	if err = obj.MakeBucket(corruptedBucketName); err != nil {
+		t.Fatal(err)
+	}
+
+	xl := obj.(*xlObjects)
+
+	// Remove bucket in disk 0, 1 and 2
+	for i := 0; i <= 2; i++ {
+		if err = xl.storageDisks[i].DeleteVol(corruptedBucketName); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// List the missing buckets.
+	buckets, err := xl.ListBucketsHeal()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check the number of buckets in list buckets heal result
+	if len(buckets) != 1 {
+		t.Fatalf("Length of missing buckets is incorrect, expected: 1, found: %d", len(buckets))
+	}
+
+	// Check the name of bucket in list buckets heal result
+	if buckets[0].Name != corruptedBucketName {
+		t.Fatalf("Name of missing bucket is incorrect, expected: %s, found: %s", corruptedBucketName, buckets[0].Name)
+	}
+}

cmd/xl-v1-list-objects-heal.go

@@ -159,11 +159,11 @@ func (xl xlObjects) listObjectsHeal(bucket, prefix, marker, delimiter string, ma
 		if xlShouldHeal(partsMetadata, errs) {
 			healStat := xlHealStat(xl, partsMetadata, errs)
 			result.Objects = append(result.Objects, ObjectInfo{
-				Name:     objInfo.Name,
-				ModTime:  objInfo.ModTime,
-				Size:     objInfo.Size,
-				IsDir:    false,
-				HealInfo: &healStat,
+				Name:           objInfo.Name,
+				ModTime:        objInfo.ModTime,
+				Size:           objInfo.Size,
+				IsDir:          false,
+				HealObjectInfo: &healStat,
 			})
 		}
 		objectLock.RUnlock()