fix: make sure to correctly initialize health checks (#17765)

health checks were missing for drives replaced since

- HealFormat() would replace the drives without a health check
- disconnected drives when they reconnect via connectEndpoint()
  the loop also loses health checks for local disks and merges
  these into a single code.
- other than this separate cleanUp, health check variables to avoid
  overloading them with similar requirements.
- also ensure that we compete via context selector for disk monitoring
  such that the canceled disks don't linger around longer waiting for
  the ticker to trigger.
- allow disabling active monitoring.

cmd/xl-storage-disk-id-check.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/config"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
@@ -176,7 +177,7 @@ func newXLStorageDiskIDCheck(storage *xlStorage, healthCheck bool) *xlStorageDis
 	for i := range xl.apiLatencies[:] {
 		xl.apiLatencies[i] = &lockedLastMinuteLatency{}
 	}
-	if healthCheck {
+	if healthCheck && diskActiveMonitoring {
 		go xl.monitorDiskWritable(xl.diskCtx)
 	}
 	return &xl
@@ -718,6 +719,9 @@ var diskStartChecking = 32
 // offline under active monitoring.
 var diskMaxTimeout = 2 * time.Minute
 
+// diskActiveMonitoring indicates if we have enabled "active" disk monitoring
+var diskActiveMonitoring = true
+
 func init() {
 	s := env.Get("_MINIO_DISK_MAX_CONCURRENT", "")
 	if s != "" {
@@ -727,6 +731,7 @@ func init() {
 			diskMaxConcurrent = 512
 		}
 	}
+
 	d := env.Get("_MINIO_DISK_MAX_TIMEOUT", "")
 	if d != "" {
 		timeoutOperation, _ := time.ParseDuration(d)
@@ -737,6 +742,8 @@ func init() {
 		}
 	}
 
+	diskActiveMonitoring = env.Get("_MINIO_DISK_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn
+
 	diskStartChecking = 16 + diskMaxConcurrent/8
 	if diskStartChecking > diskMaxConcurrent {
 		diskStartChecking = diskMaxConcurrent
@@ -997,41 +1004,48 @@ func (p *xlStorageDiskIDCheck) monitorDiskWritable(ctx context.Context) {
 	toWrite := []byte{xioutil.DirectioAlignSize + 1: 42}
 	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	for range t.C {
+	monitor := func() bool {
 		if contextCanceled(ctx) {
-			break
+			return false
 		}
+
 		if atomic.LoadInt32(&p.health.status) != diskHealthOK {
-			continue
+			return true
 		}
+
 		if time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess))) < skipIfSuccessBefore {
 			// We recently saw a success - no need to check.
-			continue
+			return true
 		}
+
 		goOffline := func(err error, spent time.Duration) {
 			if atomic.CompareAndSwapInt32(&p.health.status, diskHealthOK, diskHealthFaulty) {
 				logger.LogAlwaysIf(ctx, fmt.Errorf("node(%s): taking drive %s offline: %v", globalLocalNodeName, p.storage.String(), err))
 				go p.monitorDiskStatus(spent)
 			}
 		}
+
 		// Offset checks a bit.
 		time.Sleep(time.Duration(rng.Int63n(int64(1 * time.Second))))
-		done := make(chan struct{})
+
+		dctx, dcancel := context.WithCancel(ctx)
 		started := time.Now()
 		go func() {
 			timeout := time.NewTimer(diskMaxTimeout)
 			select {
-			case <-timeout.C:
-				spent := time.Since(started)
-				goOffline(fmt.Errorf("unable to write+read for %v", spent.Round(time.Millisecond)), spent)
-			case <-done:
+			case <-dctx.Done():
 				if !timeout.Stop() {
 					<-timeout.C
 				}
+			case <-timeout.C:
+				spent := time.Since(started)
+				goOffline(fmt.Errorf("unable to write+read for %v", spent.Round(time.Millisecond)), spent)
 			}
 		}()
+
 		func() {
-			defer close(done)
+			defer dcancel()
+
 			err := p.storage.WriteAll(ctx, minioMetaTmpBucket, fn, toWrite)
 			if err != nil {
 				if osErrToFileErr(err) == errFaultyDisk {
@@ -1047,6 +1061,20 @@ func (p *xlStorageDiskIDCheck) monitorDiskWritable(ctx context.Context) {
 				return
 			}
 		}()
+
+		// Continue to monitor
+		return true
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if !monitor() {
+				return
+			}
+		}
 	}
 }