fix: refactor background heal for cluster health (#10225)

cmd/admin-handlers.go

@@ -799,6 +799,59 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, r, respCh)
 }
 
+func getAggregatedBackgroundHealState(ctx context.Context, failOnErr bool) (madmin.BgHealState, error) {
+	var bgHealStates []madmin.BgHealState
+
+	localHealState, ok := getLocalBackgroundHealStatus()
+	if !ok {
+		if failOnErr {
+			return madmin.BgHealState{}, errServerNotInitialized
+		}
+	}
+
+	// Get local heal status first
+	bgHealStates = append(bgHealStates, localHealState)
+
+	if globalIsDistErasure {
+		// Get heal status from other peers
+		peersHealStates, nerrs := globalNotificationSys.BackgroundHealStatus()
+		for _, nerr := range nerrs {
+			if nerr.Err != nil {
+				if failOnErr {
+					return madmin.BgHealState{}, nerr.Err
+				}
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+		bgHealStates = append(bgHealStates, peersHealStates...)
+	}
+
+	// Aggregate healing result
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+		HealDisks:         bgHealStates[0].HealDisks,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
+	for _, state := range bgHealStates {
+		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		aggregatedHealStateResult.HealDisks = append(aggregatedHealStateResult.HealDisks, state.HealDisks...)
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
+		}
+	}
+
+	return aggregatedHealStateResult, nil
+}
+
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
@@ -815,39 +868,8 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	var bgHealStates []madmin.BgHealState
-
-	// Get local heal status first
-	bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
-
-	if globalIsDistErasure {
-		// Get heal status from other peers
-		peersHealStates := globalNotificationSys.BackgroundHealStatus()
-		bgHealStates = append(bgHealStates, peersHealStates...)
-	}
-
-	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{
-		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
-		LastHealActivity:  bgHealStates[0].LastHealActivity,
-		NextHealRound:     bgHealStates[0].NextHealRound,
-	}
-
-	bgHealStates = bgHealStates[1:]
-
-	for _, state := range bgHealStates {
-		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
-			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
-			// The node which has the last heal activity means its
-			// is the node that is orchestrating self healing operations,
-			// which also means it is the same node which decides when
-			// the next self healing operation will be done.
-			aggregatedHealStateResult.NextHealRound = state.NextHealRound
-		}
-	}
-
-	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
+	aggregateHealStateResult, _ := getAggregatedBackgroundHealState(r.Context(), false)
+	if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/admin-heal-ops.go

@@ -88,7 +88,8 @@ type allHealState struct {
 	sync.Mutex
 
 	// map of heal path to heal sequence
-	healSeqMap map[string]*healSequence
+	healSeqMap     map[string]*healSequence
+	healLocalDisks []Endpoints
 }
 
 // newHealState - initialize global heal state management
@@ -102,6 +103,22 @@ func newHealState() *allHealState {
 	return healState
 }
 
+func (ahs *allHealState) getHealLocalDisks() []Endpoints {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	healLocalDisks := make([]Endpoints, len(ahs.healLocalDisks))
+	copy(healLocalDisks, ahs.healLocalDisks)
+	return healLocalDisks
+}
+
+func (ahs *allHealState) updateHealLocalDisks(healLocalDisks []Endpoints) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	ahs.healLocalDisks = healLocalDisks
+}
+
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
@@ -485,6 +502,10 @@ func (h *healSequence) isQuitting() bool {
 // check if the heal sequence has ended
 func (h *healSequence) hasEnded() bool {
 	h.mutex.RLock()
+	// background heal never ends
+	if h.clientToken == bgHealingUUID {
+		return false
+	}
 	ended := len(h.currentStatus.Items) == 0 || h.currentStatus.Summary == healStoppedStatus || h.currentStatus.Summary == healFinishedStatus
 	h.mutex.RUnlock()
 	return ended

cmd/background-heal-ops.go

@@ -116,24 +116,6 @@ func newHealRoutine() *healRoutine {
 
 }
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
-	go globalBackgroundHealRoutine.run(ctx, objAPI)
-
-	nh := newBgHealSequence()
-	// Heal any disk format and metadata early, if possible.
-	if err := nh.healDiskMeta(); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(nh.ctx, err)
-		}
-	}
-
-	globalBackgroundHealState.LaunchNewHealSequence(nh)
-}
-
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) {

cmd/background-newdisks-heal-ops.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -26,19 +27,23 @@ import (
 
 const defaultMonitorNewDiskInterval = time.Minute * 3
 
-func initLocalDisksAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	go monitorLocalDisksAndHeal(ctx, objAPI)
-}
-
-// monitorLocalDisksAndHeal - ensures that detected new disks are healed
-//  1. Only the concerned erasure set will be listed and healed
-//  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
+func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	z, ok := objAPI.(*erasureZones)
 	if !ok {
 		return
 	}
 
+	initBackgroundHealing(ctx, objAPI) // start quick background healing
+
+	localDisksInZoneHeal := getLocalDisksToHeal(objAPI)
+	globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
+
+	drivesToHeal := getDrivesToHealCount(localDisksInZoneHeal)
+	if drivesToHeal != 0 {
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+			drivesToHeal, defaultMonitorNewDiskInterval))
+	}
+
 	var bgSeq *healSequence
 	var found bool
 
@@ -50,53 +55,99 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 		time.Sleep(time.Second)
 	}
 
+	if drivesToHeal != 0 {
+		// Heal any disk format and metadata early, if possible.
+		if err := bgSeq.healDiskMeta(); err != nil {
+			if newObjectLayerFn() != nil {
+				// log only in situations, when object layer
+				// has fully initialized.
+				logger.LogIf(bgSeq.ctx, err)
+			}
+		}
+	}
+
+	go monitorLocalDisksAndHeal(ctx, z, drivesToHeal, localDisksInZoneHeal, bgSeq)
+}
+
+func getLocalDisksToHeal(objAPI ObjectLayer) []Endpoints {
+	z, ok := objAPI.(*erasureZones)
+	if !ok {
+		return nil
+	}
+
+	// Attempt a heal as the server starts-up first.
+	localDisksInZoneHeal := make([]Endpoints, len(z.zones))
+	for i, ep := range globalEndpoints {
+		localDisksToHeal := Endpoints{}
+		for _, endpoint := range ep.Endpoints {
+			if !endpoint.IsLocal {
+				continue
+			}
+			// Try to connect to the current endpoint
+			// and reformat if the current disk is not formatted
+			_, _, err := connectEndpoint(endpoint)
+			if err == errUnformattedDisk {
+				localDisksToHeal = append(localDisksToHeal, endpoint)
+			}
+		}
+		if len(localDisksToHeal) == 0 {
+			continue
+		}
+		localDisksInZoneHeal[i] = localDisksToHeal
+	}
+	return localDisksInZoneHeal
+
+}
+
+func getDrivesToHealCount(localDisksInZoneHeal []Endpoints) int {
+	var drivesToHeal int
+	for _, eps := range localDisksInZoneHeal {
+		for range eps {
+			drivesToHeal++
+		}
+	}
+	return drivesToHeal
+}
+
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	// Run the background healer
+	globalBackgroundHealRoutine = newHealRoutine()
+	go globalBackgroundHealRoutine.run(ctx, objAPI)
+
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence())
+}
+
+// monitorLocalDisksAndHeal - ensures that detected new disks are healed
+//  1. Only the concerned erasure set will be listed and healed
+//  2. Only the node hosting the disk is responsible to perform the heal
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, drivesToHeal int, localDisksInZoneHeal []Endpoints, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-time.After(defaultMonitorNewDiskInterval):
-			// Attempt a heal as the server starts-up first.
-			localDisksInZoneHeal := make([]Endpoints, len(z.zones))
-			var healNewDisks bool
-			for i, ep := range globalEndpoints {
-				localDisksToHeal := Endpoints{}
-				for _, endpoint := range ep.Endpoints {
-					if !endpoint.IsLocal {
-						continue
-					}
-					// Try to connect to the current endpoint
-					// and reformat if the current disk is not formatted
-					_, _, err := connectEndpoint(endpoint)
-					if err == errUnformattedDisk {
-						localDisksToHeal = append(localDisksToHeal, endpoint)
-					}
-				}
-				if len(localDisksToHeal) == 0 {
+			// heal only if new disks found.
+			if drivesToHeal == 0 {
+				localDisksInZoneHeal = getLocalDisksToHeal(z)
+				drivesToHeal = getDrivesToHealCount(localDisksInZoneHeal)
+				if drivesToHeal == 0 {
+					// No drives to heal.
+					globalBackgroundHealState.updateHealLocalDisks(nil)
 					continue
 				}
-				localDisksInZoneHeal[i] = localDisksToHeal
-				healNewDisks = true
+				globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
+
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+					drivesToHeal))
+
+				// Reformat disks
+				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
+
+				// Ensure that reformatting disks is finished
+				bgSeq.sourceCh <- healSource{bucket: nopHeal}
 			}
 
-			// Reformat disks only if needed.
-			if !healNewDisks {
-				continue
-			}
-
-			logger.Info("New unformatted drives detected attempting to heal...")
-			for i, disks := range localDisksInZoneHeal {
-				for _, disk := range disks {
-					logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
-				}
-			}
-
-			// Reformat disks
-			bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
-
-			// Ensure that reformatting disks is finished
-			bgSeq.sourceCh <- healSource{bucket: nopHeal}
-
 			var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
 			// Compute the list of erasure set to heal
 			for i, localDisksToHeal := range localDisksInZoneHeal {
@@ -108,6 +159,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 						printEndpointError(endpoint, err, true)
 						continue
 					}
+
 					// Calculate the set index where the current endpoint belongs
 					setIndex, _, err := findDiskIndex(z.zones[i].format, format)
 					if err != nil {
@@ -120,6 +172,13 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 				erasureSetInZoneToHeal[i] = erasureSetToHeal
 			}
 
+			logger.Info("New unformatted drives detected attempting to heal the content...")
+			for i, disks := range localDisksInZoneHeal {
+				for _, disk := range disks {
+					logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+				}
+			}
+
 			// Heal all erasure sets that need
 			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
 				for _, setIndex := range erasureSetToHeal {
@@ -127,6 +186,11 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 					if err != nil {
 						logger.LogIf(ctx, err)
 					}
+
+					// Only upon success reduce the counter
+					if err == nil {
+						drivesToHeal--
+					}
 				}
 			}
 		}

cmd/erasure-sets.go

@@ -1414,6 +1414,11 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			return madmin.HealResultItem{}, err
 		}
 
+		refFormat, err = getFormatErasureInQuorum(tmpNewFormats)
+		if err != nil {
+			return madmin.HealResultItem{}, err
+		}
+
 		// kill the monitoring loop such that we stop writing
 		// to indicate that we will re-initialize everything
 		// with new format.

cmd/erasure-zones.go

@@ -2032,6 +2032,7 @@ type HealthOptions struct {
 // was queried
 type HealthResult struct {
 	Healthy       bool
+	HealingDrives int
 	ZoneID, SetID int
 	WriteQuorum   int
 }
@@ -2086,8 +2087,27 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 			}
 		}
 	}
+
+	// check if local disks are being healed, if they are being healed
+	// we need to tell healthy status as 'false' so that this server
+	// is not taken down for maintenance
+	aggHealStateResult, err := getAggregatedBackgroundHealState(ctx, true)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to verify global heal status: %w", err))
+		return HealthResult{
+			Healthy: false,
+		}
+	}
+
+	if len(aggHealStateResult.HealDisks) > 0 {
+		logger.LogIf(ctx, fmt.Errorf("Total drives to be healed %d", len(aggHealStateResult.HealDisks)))
+	}
+
+	healthy := len(aggHealStateResult.HealDisks) == 0
+
 	return HealthResult{
-		Healthy: true,
+		Healthy:       healthy,
+		HealingDrives: len(aggHealStateResult.HealDisks),
 	}
 }
 

cmd/global-heal.go

@@ -53,7 +53,9 @@ func newBgHealSequence() *healSequence {
 		respCh:      make(chan healResult),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
-		settings:    hs,
+		// run-background heal with reserved bucket
+		bucket:   minioReservedBucket,
+		settings: hs,
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
@@ -67,17 +69,25 @@ func newBgHealSequence() *healSequence {
 	}
 }
 
-func getLocalBackgroundHealStatus() madmin.BgHealState {
+func getLocalBackgroundHealStatus() (madmin.BgHealState, bool) {
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if !ok {
-		return madmin.BgHealState{}
+		return madmin.BgHealState{}, false
+	}
+
+	var healDisks []string
+	for _, eps := range globalBackgroundHealState.getHealLocalDisks() {
+		for _, ep := range eps {
+			healDisks = append(healDisks, ep.String())
+		}
 	}
 
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.getScannedItemsCount(),
 		LastHealActivity:  bgSeq.lastHealActivity,
+		HealDisks:         healDisks,
 		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
-	}
+	}, true
 }
 
 // healErasureSet lists and heals all objects in a specific erasure set
@@ -164,13 +174,14 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, dr
 // deepHealObject heals given object path in deep to fix bitrot.
 func deepHealObject(bucket, object, versionID string) {
 	// Get background heal sequence to send elements to heal
-	bgSeq, _ := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-
-	bgSeq.sourceCh <- healSource{
-		bucket:    bucket,
-		object:    object,
-		versionID: versionID,
-		opts:      &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
+	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+	if ok {
+		bgSeq.sourceCh <- healSource{
+			bucket:    bucket,
+			object:    object,
+			versionID: versionID,
+			opts:      &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
+		}
 	}
 }
 

cmd/handler-utils.go

@@ -480,6 +480,12 @@ func getHostName(r *http.Request) (hostName string) {
 func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, ep ProxyEndpoint) (success bool) {
 	success = true
 
+	// Make sure we remove any existing headers before
+	// proxying the request to another node.
+	for k := range w.Header() {
+		w.Header().Del(k)
+	}
+
 	f := handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
 		RoundTripper: ep.Transport,

cmd/notification.go

@@ -269,21 +269,26 @@ func (sys *NotificationSys) LoadServiceAccount(accessKey string) []NotificationP
 }
 
 // BackgroundHealStatus - returns background heal status of all peers
-func (sys *NotificationSys) BackgroundHealStatus() []madmin.BgHealState {
+func (sys *NotificationSys) BackgroundHealStatus() ([]madmin.BgHealState, []NotificationPeerErr) {
+	ng := WithNPeers(len(sys.peerClients))
 	states := make([]madmin.BgHealState, len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
 			continue
 		}
-		st, err := client.BackgroundHealStatus()
-		if err != nil {
-			logger.LogIf(GlobalContext, err)
-		} else {
+		idx := idx
+		client := client
+		ng.Go(GlobalContext, func() error {
+			st, err := client.BackgroundHealStatus()
+			if err != nil {
+				return err
+			}
 			states[idx] = st
-		}
+			return nil
+		}, idx, *client.host)
 	}
 
-	return states
+	return states, ng.Wait()
 }
 
 // StartProfiling - start profiling on remote peers, by initiating a remote RPC.

cmd/peer-rest-server.go

@@ -976,7 +976,11 @@ func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *h
 
 	ctx := newContext(r, w, "BackgroundHealStatus")
 
-	state := getLocalBackgroundHealStatus()
+	state, ok := getLocalBackgroundHealStatus()
+	if !ok {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
 
 	defer w.(http.Flusher).Flush()
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(state))

cmd/server-main.go

@@ -221,8 +221,7 @@ func initSafeMode(ctx context.Context, newObject ObjectLayer) (err error) {
 
 	// Enable healing to heal drives if possible
 	if globalIsErasure {
-		initBackgroundHealing(ctx, newObject)
-		initLocalDisksAutoHeal(ctx, newObject)
+		initAutoHeal(ctx, newObject)
 	}
 
 	// ****  WARNING ****

cmd/xl-storage.go

@@ -2103,9 +2103,6 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
 		legacyDataPath := pathJoin(dstVolumeDir, dstPath, legacyDataDir)
 		// legacy data dir means its old content, honor system umask.
 		if err = os.MkdirAll(legacyDataPath, 0777); err != nil {
-			if isSysErrIO(err) {
-				return errFaultyDisk
-			}
 			return osErrToFileErr(err)
 		}
 
@@ -2116,14 +2113,11 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
 
 		for _, entry := range entries {
 			// Skip xl.meta renames further, also ignore any directories such as `legacyDataDir`
-			if entry == xlStorageFormatFile || strings.HasPrefix(entry, SlashSeparator) {
+			if entry == xlStorageFormatFile || strings.HasSuffix(entry, slashSeparator) {
 				continue
 			}
 
 			if err = os.Rename(pathJoin(currentDataPath, entry), pathJoin(legacyDataPath, entry)); err != nil {
-				if isSysErrIO(err) {
-					return errFaultyDisk
-				}
 				return osErrToFileErr(err)
 			}
 		}
@@ -2159,20 +2153,14 @@ func (s *xlStorage) RenameData(srcVolume, srcPath, dataDir, dstVolume, dstPath s
 	}
 
 	if err = renameAll(srcFilePath, dstFilePath); err != nil {
-		if isSysErrIO(err) {
-			return errFaultyDisk
-		}
-		return err
+		return osErrToFileErr(err)
 	}
 
 	if srcDataPath != "" {
 		removeAll(oldDstDataPath)
 		removeAll(dstDataPath)
 		if err = renameAll(srcDataPath, dstDataPath); err != nil {
-			if isSysErrIO(err) {
-				return errFaultyDisk
-			}
-			return err
+			return osErrToFileErr(err)
 		}
 	}
 
@@ -2265,10 +2253,7 @@ func (s *xlStorage) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (e
 	}
 
 	if err = renameAll(srcFilePath, dstFilePath); err != nil {
-		if isSysErrIO(err) {
-			return errFaultyDisk
-		}
-		return err
+		return osErrToFileErr(err)
 	}
 
 	// Remove parent dir of the source file if empty

docs/bucket/replication/README.md

@@ -1,8 +1,8 @@
 # Bucket Replication Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
-Bucket replication is designed to replicate specific objects in buckets that are configured for replication.
+Bucket replication is designed to replicate selected objects in a bucket to a destination bucket.
 
-To replicate objects in a bucket to a destination bucket on a target site either on the same cluster or a different cluster, start by creating version enabled buckets on both `source` and `dest` buckets. Next, the target site and destination bucket need to be configured on MinIO server by setting
+To replicate objects in a bucket to a destination bucket on a target site either in the same cluster or a different cluster, start by enabling [versioning](https://docs.minio.io/docs/minio-bucket-versioning-guide.html) for both source and destination buckets. Finally, the target site and the destination bucket need to be configured on the source MinIO server.
 
 ## Highlights
 - Supports source and destination buckets to have the same name unlike AWS S3, addresses variety of usecases such as *Splunk*, *Veeam* site to site DR.
@@ -65,6 +65,7 @@ Replication configuration applied successfully to myminio/srcbucket.
 
 The replication configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely.
 
+
 When an object is deleted from the source bucket, the replica will not be deleted as per S3 spec.
 
 ![delete](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/DELETE_bucket_replication.png)

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # 9001 through 9004.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-08-05T21-34-13Z
+    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
     hostname: minio4
     volumes:
       - minio4-data:/export

pkg/env/web_env.go

@@ -152,7 +152,7 @@ func LookupEnv(key string) (string, bool) {
 	if ok {
 		u, err := url.Parse(v)
 		if err != nil {
-			return "", false
+			return v, true
 		}
 		if !isValidEnvScheme(u.Scheme) {
 			return v, true

pkg/madmin/heal-commands.go

@@ -294,6 +294,7 @@ type BgHealState struct {
 	ScannedItemsCount int64
 	LastHealActivity  time.Time
 	NextHealRound     time.Time
+	HealDisks         []string
 }
 
 // BackgroundHealStatus returns the background heal status of the